42a922ff71ed72b0802476d236e6b5adb81925ce
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38
39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
40 {
41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
42         unsigned long flags;
43
44         spin_lock_irqsave(&t->tasklet_lock, flags);
45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
46                 smp_processor_id());
47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
49
50         tasklet_schedule(&t->tasklet);
51 }
52
53 /*
54  * Must not be called in parallel with scst_unregister_session_ex() for the
55  * same sess
56  */
57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
58                              const uint8_t *lun, int lun_len,
59                              const uint8_t *cdb, int cdb_len, int atomic)
60 {
61         struct scst_cmd *cmd;
62
63         TRACE_ENTRY();
64
65 #ifdef CONFIG_SCST_EXTRACHECKS
66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
67                 PRINT_CRIT_ERROR("%s",
68                         "New cmd while shutting down the session");
69                 sBUG();
70         }
71 #endif
72
73         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
74         if (cmd == NULL)
75                 goto out;
76
77         cmd->sess = sess;
78         cmd->tgt = sess->tgt;
79         cmd->tgtt = sess->tgt->tgtt;
80
81         /*
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100 EXPORT_SYMBOL(scst_rx_cmd);
101
102 /*
103  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
104  * this command should be stopped.
105  */
106 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
107 {
108         int rc, res = 0;
109
110         TRACE_ENTRY();
111
112         /* See the comment in scst_do_job_init() */
113         if (unlikely(!list_empty(&scst_init_cmd_list))) {
114                 TRACE_MGMT_DBG("%s", "init cmd list busy");
115                 goto out_redirect;
116         }
117         /*
118          * Memory barrier isn't necessary here, because CPU appears to
119          * be self-consistent
120          */
121
122         rc = __scst_init_cmd(cmd);
123         if (unlikely(rc > 0))
124                 goto out_redirect;
125         else if (unlikely(rc != 0))
126                 goto out;
127
128         /* Small context optimization */
129         if (((*context == SCST_CONTEXT_TASKLET) ||
130              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
131              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
132               scst_cmd_is_expected_set(cmd)) {
133                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
134                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
135                                         &cmd->tgt_dev->tgt_dev_flags))
136                                 *context = SCST_CONTEXT_THREAD;
137                 } else {
138                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
139                                         &cmd->tgt_dev->tgt_dev_flags))
140                                 *context = SCST_CONTEXT_THREAD;
141                 }
142         }
143
144 out:
145         TRACE_EXIT_RES(res);
146         return res;
147
148 out_redirect:
149         if (cmd->preprocessing_only) {
150                 /*
151                  * Poor man solution for single threaded targets, where
152                  * blocking receiver at least sometimes means blocking all.
153                  */
154                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
155                 scst_set_busy(cmd);
156                 scst_set_cmd_abnormal_done_state(cmd);
157                 /* Keep initiator away from too many BUSY commands */
158                 msleep(50);
159         } else {
160                 unsigned long flags;
161                 spin_lock_irqsave(&scst_init_lock, flags);
162                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
163                         "%d)", cmd, atomic_read(&scst_cmd_count));
164                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
165                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
166                         scst_init_poll_cnt++;
167                 spin_unlock_irqrestore(&scst_init_lock, flags);
168                 wake_up(&scst_init_cmd_list_waitQ);
169                 res = -1;
170         }
171         goto out;
172 }
173
174 #ifdef CONFIG_SCST_MEASURE_LATENCY
175 static inline uint64_t scst_sec_to_nsec(time_t sec)
176 {
177         return (uint64_t)sec * 1000000000;
178 }
179 #endif
180
181 void scst_cmd_init_done(struct scst_cmd *cmd,
182         enum scst_exec_context pref_context)
183 {
184         unsigned long flags;
185         struct scst_session *sess = cmd->sess;
186         int rc;
187
188         TRACE_ENTRY();
189
190 #ifdef CONFIG_SCST_MEASURE_LATENCY
191         {
192                 struct timespec ts;
193                 getnstimeofday(&ts);
194                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
195                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
196                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
197                         ts.tv_nsec);
198         }
199 #endif
200
201         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
202         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d",
203               (long long unsigned int)cmd->tag,
204               (long long unsigned int)cmd->lun,
205               cmd->cdb_len);
206         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
207                 cmd->cdb, cmd->cdb_len);
208
209 #ifdef CONFIG_SCST_EXTRACHECKS
210         if (unlikely((in_irq() || irqs_disabled())) &&
211             ((pref_context == SCST_CONTEXT_DIRECT) ||
212              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
213                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
214                         "SCST_CONTEXT_THREAD instead\n", pref_context,
215                         cmd->tgtt->name);
216                 pref_context = SCST_CONTEXT_THREAD;
217         }
218 #endif
219
220         atomic_inc(&sess->sess_cmd_count);
221
222         spin_lock_irqsave(&sess->sess_list_lock, flags);
223
224         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
225                 /*
226                  * We have to always keep command in the search list from the
227                  * very beginning, because otherwise it can be missed during
228                  * TM processing. This check is needed because there might be
229                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
230                  */
231                 if (cmd->search_cmd_list_entry.next == NULL)
232                         list_add_tail(&cmd->search_cmd_list_entry,
233                                 &sess->search_cmd_list);
234                 switch (sess->init_phase) {
235                 case SCST_SESS_IPH_SUCCESS:
236                         break;
237                 case SCST_SESS_IPH_INITING:
238                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
239                                   cmd);
240                         list_add_tail(&cmd->cmd_list_entry,
241                                 &sess->init_deferred_cmd_list);
242                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
243                         goto out;
244                 case SCST_SESS_IPH_FAILED:
245                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
246                         scst_set_busy(cmd);
247                         scst_set_cmd_abnormal_done_state(cmd);
248                         goto active;
249                 default:
250                         sBUG();
251                 }
252         } else
253                 list_add_tail(&cmd->search_cmd_list_entry,
254                               &sess->search_cmd_list);
255
256         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
257
258         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
259                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
260                 scst_set_cmd_error(cmd,
261                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
262                 scst_set_cmd_abnormal_done_state(cmd);
263                 goto active;
264         }
265
266         if (unlikely(cmd->cdb_len == 0)) {
267                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
268                 scst_set_cmd_error(cmd,
269                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
270                 scst_set_cmd_abnormal_done_state(cmd);
271                 goto active;
272         }
273
274         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
275                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
276                 scst_set_cmd_error(cmd,
277                         SCST_LOAD_SENSE(scst_sense_invalid_message));
278                 scst_set_cmd_abnormal_done_state(cmd);
279                 goto active;
280         }
281
282         cmd->state = SCST_CMD_STATE_INIT;
283         /* cmd must be inited here to preserve the order */
284         rc = scst_init_cmd(cmd, &pref_context);
285         if (unlikely(rc < 0))
286                 goto out;
287
288 active:
289         /* Here cmd must not be in any cmd list, no locks */
290         switch (pref_context) {
291         case SCST_CONTEXT_TASKLET:
292                 scst_schedule_tasklet(cmd);
293                 break;
294
295         case SCST_CONTEXT_DIRECT:
296                 scst_process_active_cmd(cmd, false);
297                 /* For *NEED_THREAD wake_up() is already done */
298                 break;
299
300         case SCST_CONTEXT_DIRECT_ATOMIC:
301                 scst_process_active_cmd(cmd, true);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         default:
306                 PRINT_ERROR("Context %x is undefined, using the thread one",
307                         pref_context);
308                 /* go through */
309         case SCST_CONTEXT_THREAD:
310                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
311                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
312                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
313                         list_add(&cmd->cmd_list_entry,
314                                 &cmd->cmd_lists->active_cmd_list);
315                 else
316                         list_add_tail(&cmd->cmd_list_entry,
317                                 &cmd->cmd_lists->active_cmd_list);
318                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
319                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
320                 break;
321         }
322
323 out:
324         TRACE_EXIT();
325         return;
326 }
327 EXPORT_SYMBOL(scst_cmd_init_done);
328
329 static int scst_pre_parse(struct scst_cmd *cmd)
330 {
331         int res = SCST_CMD_STATE_RES_CONT_SAME;
332         struct scst_device *dev = cmd->dev;
333         int rc;
334
335         TRACE_ENTRY();
336
337         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
338              (!dev->has_own_order_mgmt &&
339               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
340                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
341
342         /*
343          * Expected transfer data supplied by the SCSI transport via the
344          * target driver are untrusted, so we prefer to fetch them from CDB.
345          * Additionally, not all transports support supplying the expected
346          * transfer data.
347          */
348
349         rc = scst_get_cdb_info(cmd);
350         if (unlikely(rc != 0)) {
351                 if (rc > 0) {
352                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
353                         goto out_xmit;
354                 }
355                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
356                         "Should you update scst_scsi_op_table?",
357                         cmd->cdb[0], dev->handler->name);
358                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
359 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
360                 if (scst_cmd_is_expected_set(cmd)) {
361                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
362                                 "direction %d, transfer_len %d",
363                                 cmd->expected_data_direction,
364                                 cmd->expected_transfer_len);
365                         cmd->data_direction = cmd->expected_data_direction;
366
367                         cmd->bufflen = cmd->expected_transfer_len;
368                         /* Restore (likely) lost CDB length */
369                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
370                         if (cmd->cdb_len == -1) {
371                                 PRINT_ERROR("Unable to get CDB length for "
372                                         "opcode 0x%02x. Returning INVALID "
373                                         "OPCODE", cmd->cdb[0]);
374                                 scst_set_cmd_error(cmd,
375                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
376                                 goto out_xmit;
377                         }
378                 } else {
379                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
380                              "target %s not supplied expected values",
381                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
382                         scst_set_cmd_error(cmd,
383                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
384                         goto out_xmit;
385                 }
386 #else
387                 scst_set_cmd_error(cmd,
388                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                 goto out_xmit;
390 #endif
391         } else {
392                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
393                         "set %s), transfer_len=%d (expected len %d), flags=%d",
394                         cmd->op_name, cmd->data_direction,
395                         cmd->expected_data_direction,
396                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
397                         cmd->bufflen, cmd->expected_transfer_len,
398                         cmd->op_flags);
399
400                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
401                         if (scst_cmd_is_expected_set(cmd)) {
402                                 /*
403                                  * Command data length can't be easily
404                                  * determined from the CDB. ToDo, all such
405                                  * commands processing should be fixed. Until
406                                  * it's done, get the length from the supplied
407                                  * expected value, but limit it to some
408                                  * reasonable value (15MB).
409                                  */
410                                 cmd->bufflen = min(cmd->expected_transfer_len,
411                                                         15*1024*1024);
412                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
413                         } else
414                                 cmd->bufflen = 0;
415                 }
416         }
417
418         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
419                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
420                             "(opcode 0x%02x)", cmd->cdb[0]);
421                 scst_set_cmd_error(cmd,
422                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
423                 goto out_xmit;
424         }
425
426         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
427                 PRINT_ERROR("Linked commands are not supported "
428                             "(opcode 0x%02x)", cmd->cdb[0]);
429                 scst_set_cmd_error(cmd,
430                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
431                 goto out_xmit;
432         }
433
434         cmd->state = SCST_CMD_STATE_DEV_PARSE;
435
436 out:
437         TRACE_EXIT_RES(res);
438         return res;
439
440 out_xmit:
441         scst_set_cmd_abnormal_done_state(cmd);
442         res = SCST_CMD_STATE_RES_CONT_SAME;
443         goto out;
444 }
445
446 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
447 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
448 {
449         bool res = false;
450
451         switch (cmd->cdb[0]) {
452         case TEST_UNIT_READY:
453                 /* Crazy VMware people sometimes do TUR with READ direction */
454                 res = true;
455                 break;
456         case VERIFY:
457         case VERIFY_6:
458         case VERIFY_12:
459         case VERIFY_16:
460                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
461                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
462                     (cmd->cdb[1] & BYTCHK) == 0)
463                         res = true;
464                 break;
465         }
466
467         return res;
468 }
469 #endif
470
471 static int scst_parse_cmd(struct scst_cmd *cmd)
472 {
473         int res = SCST_CMD_STATE_RES_CONT_SAME;
474         int state;
475         struct scst_device *dev = cmd->dev;
476         int orig_bufflen = cmd->bufflen;
477
478         TRACE_ENTRY();
479
480         if (likely(!scst_is_cmd_local(cmd))) {
481                 if (unlikely(!dev->handler->parse_atomic &&
482                              scst_cmd_atomic(cmd))) {
483                         /*
484                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
485                          * optimization.
486                          */
487                         TRACE_DBG("Dev handler %s parse() needs thread "
488                                 "context, rescheduling", dev->handler->name);
489                         res = SCST_CMD_STATE_RES_NEED_THREAD;
490                         goto out;
491                 }
492
493                 TRACE_DBG("Calling dev handler %s parse(%p)",
494                       dev->handler->name, cmd);
495                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
496                                 cmd->cdb, cmd->cdb_len);
497                 state = dev->handler->parse(cmd);
498                 /* Caution: cmd can be already dead here */
499                 TRACE_DBG("Dev handler %s parse() returned %d",
500                         dev->handler->name, state);
501
502                 switch (state) {
503                 case SCST_CMD_STATE_NEED_THREAD_CTX:
504                         TRACE_DBG("Dev handler %s parse() requested thread "
505                               "context, rescheduling", dev->handler->name);
506                         res = SCST_CMD_STATE_RES_NEED_THREAD;
507                         goto out;
508
509                 case SCST_CMD_STATE_STOP:
510                         TRACE_DBG("Dev handler %s parse() requested stop "
511                                 "processing", dev->handler->name);
512                         res = SCST_CMD_STATE_RES_CONT_NEXT;
513                         goto out;
514                 }
515
516                 if (state == SCST_CMD_STATE_DEFAULT)
517                         state = SCST_CMD_STATE_PREPARE_SPACE;
518         } else
519                 state = SCST_CMD_STATE_PREPARE_SPACE;
520
521         if (cmd->data_len == -1)
522                 cmd->data_len = cmd->bufflen;
523
524         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
525                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
526                         "is less, than required (size %d)", cmd->bufflen,
527                         orig_bufflen);
528                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
529                 goto out_error;
530         }
531
532         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
533                 goto set_res;
534
535         if (unlikely((cmd->bufflen == 0) &&
536                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
537                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
538                         "(handler %s, target %s)", cmd->cdb[0],
539                         dev->handler->name, cmd->tgtt->name);
540                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
541                 goto out_error;
542         }
543
544 #ifdef CONFIG_SCST_EXTRACHECKS
545         if ((cmd->bufflen != 0) &&
546             ((cmd->data_direction == SCST_DATA_NONE) ||
547              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
548                 PRINT_ERROR("Dev handler %s parse() returned "
549                         "invalid cmd data_direction %d, bufflen %d, state %d "
550                         "or sg %p (opcode 0x%x)", dev->handler->name,
551                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
552                         cmd->cdb[0]);
553                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
554                 goto out_error;
555         }
556 #endif
557
558         if (scst_cmd_is_expected_set(cmd)) {
559 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
560 #       ifdef CONFIG_SCST_EXTRACHECKS
561                 if ((cmd->data_direction != cmd->expected_data_direction) ||
562                     (cmd->bufflen != cmd->expected_transfer_len)) {
563                         PRINT_WARNING("Expected values don't match decoded "
564                                 "ones: data_direction %d, "
565                                 "expected_data_direction %d, "
566                                 "bufflen %d, expected_transfer_len %d",
567                                 cmd->data_direction,
568                                 cmd->expected_data_direction,
569                                 cmd->bufflen, cmd->expected_transfer_len);
570                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
571                 }
572 #       endif
573                 cmd->data_direction = cmd->expected_data_direction;
574                 cmd->bufflen = cmd->expected_transfer_len;
575 #else
576                 if (unlikely(cmd->data_direction !=
577                                 cmd->expected_data_direction)) {
578                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
579                              (cmd->bufflen != 0)) &&
580                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
581                                 PRINT_ERROR("Expected data direction %d for "
582                                         "opcode 0x%02x (handler %s, target %s) "
583                                         "doesn't match "
584                                         "decoded value %d",
585                                         cmd->expected_data_direction,
586                                         cmd->cdb[0], dev->handler->name,
587                                         cmd->tgtt->name, cmd->data_direction);
588                                 PRINT_BUFFER("Failed CDB",
589                                         cmd->cdb, cmd->cdb_len);
590                                 scst_set_cmd_error(cmd,
591                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
592                                 goto out_dev_done;
593                         }
594                 }
595                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
596                         static int repd;
597                         if (repd < 100) {
598                                 /*
599                                  * Intentionally unlocked. Few messages more
600                                  * or less don't matter.
601                                  */
602                                 repd++;
603                                 TRACE(TRACE_MINOR, "Warning: expected transfer "
604                                         "length %d for opcode 0x%02x (handler "
605                                         "%s, target %s) doesn't match decoded "
606                                         "value %d. Faulty initiator (e.g. "
607                                         "VMware is known to be such) or "
608                                         "scst_scsi_op_table should be updated?",
609                                         cmd->expected_transfer_len, cmd->cdb[0],
610                                         dev->handler->name, cmd->tgtt->name,
611                                         cmd->bufflen);
612                                 PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
613                                         cmd->cdb, cmd->cdb_len);
614                         }
615                 }
616 #endif
617         }
618
619         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
620                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
621                         "target %s", cmd->cdb[0], dev->handler->name,
622                         cmd->tgtt->name);
623                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
624                 goto out_error;
625         }
626
627 set_res:
628         switch (state) {
629         case SCST_CMD_STATE_PREPARE_SPACE:
630         case SCST_CMD_STATE_PRE_PARSE:
631         case SCST_CMD_STATE_DEV_PARSE:
632         case SCST_CMD_STATE_RDY_TO_XFER:
633         case SCST_CMD_STATE_TGT_PRE_EXEC:
634         case SCST_CMD_STATE_SEND_FOR_EXEC:
635         case SCST_CMD_STATE_LOCAL_EXEC:
636         case SCST_CMD_STATE_REAL_EXEC:
637         case SCST_CMD_STATE_PRE_DEV_DONE:
638         case SCST_CMD_STATE_DEV_DONE:
639         case SCST_CMD_STATE_PRE_XMIT_RESP:
640         case SCST_CMD_STATE_XMIT_RESP:
641         case SCST_CMD_STATE_FINISHED:
642                 cmd->state = state;
643                 res = SCST_CMD_STATE_RES_CONT_SAME;
644                 break;
645
646         default:
647                 if (state >= 0) {
648                         PRINT_ERROR("Dev handler %s parse() returned "
649                              "invalid cmd state %d (opcode %d)",
650                              dev->handler->name, state, cmd->cdb[0]);
651                 } else {
652                         PRINT_ERROR("Dev handler %s parse() returned "
653                                 "error %d (opcode %d)", dev->handler->name,
654                                 state, cmd->cdb[0]);
655                 }
656                 goto out_error;
657         }
658
659         if (cmd->resp_data_len == -1) {
660                 if (cmd->data_direction == SCST_DATA_READ)
661                         cmd->resp_data_len = cmd->bufflen;
662                 else
663                          cmd->resp_data_len = 0;
664         }
665
666 out:
667         TRACE_EXIT_HRES(res);
668         return res;
669
670 out_error:
671         /* dev_done() will be called as part of the regular cmd's finish */
672         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
673
674 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
675 out_dev_done:
676 #endif
677         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
678         res = SCST_CMD_STATE_RES_CONT_SAME;
679         goto out;
680 }
681
682 static int scst_prepare_space(struct scst_cmd *cmd)
683 {
684         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
685
686         TRACE_ENTRY();
687
688         if (cmd->data_direction == SCST_DATA_NONE)
689                 goto prep_done;
690
691         if (cmd->data_buf_tgt_alloc) {
692                 int orig_bufflen = cmd->bufflen;
693
694                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
695
696                 r = cmd->tgtt->alloc_data_buf(cmd);
697                 if (r > 0)
698                         goto alloc;
699                 else if (r == 0) {
700                         if (unlikely(cmd->bufflen == 0)) {
701                                 /* See comment in scst_alloc_space() */
702                                 if (cmd->sg == NULL)
703                                         goto alloc;
704                         }
705                         cmd->data_buf_alloced = 1;
706                         if (unlikely(orig_bufflen < cmd->bufflen)) {
707                                 PRINT_ERROR("Target driver allocated data "
708                                         "buffer (size %d), is less, than "
709                                         "required (size %d)", orig_bufflen,
710                                         cmd->bufflen);
711                                 goto out_error;
712                         }
713                         TRACE_MEM("%s", "data_buf_alloced, returning");
714                 }
715                 goto check;
716         }
717
718 alloc:
719         if (!cmd->data_buf_alloced)
720                 r = scst_alloc_space(cmd);
721         else
722                 TRACE_MEM("%s", "data_buf_alloced set, returning");
723
724 check:
725         if (r != 0) {
726                 if (scst_cmd_atomic(cmd)) {
727                         TRACE_MEM("%s", "Atomic memory allocation failed, "
728                               "rescheduling to the thread");
729                         res = SCST_CMD_STATE_RES_NEED_THREAD;
730                         goto out;
731                 } else
732                         goto out_no_space;
733         }
734
735 prep_done:
736         if (cmd->preprocessing_only) {
737                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
738                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
739                                 "cmd %p", cmd);
740                         scst_set_cmd_abnormal_done_state(cmd);
741                         res = SCST_CMD_STATE_RES_CONT_SAME;
742                         goto out;
743                 }
744
745                 res = SCST_CMD_STATE_RES_CONT_NEXT;
746                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
747
748                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
749                 cmd->tgtt->preprocessing_done(cmd);
750                 TRACE_DBG("%s", "preprocessing_done() returned");
751                 goto out;
752
753         }
754
755         switch (cmd->data_direction) {
756         case SCST_DATA_WRITE:
757                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
758                 break;
759
760         default:
761                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
762                 break;
763         }
764
765 out:
766         TRACE_EXIT_HRES(res);
767         return res;
768
769 out_no_space:
770         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
771                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
772         scst_set_busy(cmd);
773         scst_set_cmd_abnormal_done_state(cmd);
774         res = SCST_CMD_STATE_RES_CONT_SAME;
775         goto out;
776
777 out_error:
778         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
779         scst_set_cmd_abnormal_done_state(cmd);
780         res = SCST_CMD_STATE_RES_CONT_SAME;
781         goto out;
782 }
783
784 void scst_restart_cmd(struct scst_cmd *cmd, int status, 
785         enum scst_exec_context pref_context)
786 {
787         TRACE_ENTRY();
788
789         TRACE_DBG("Preferred context: %d", pref_context);
790         TRACE_DBG("tag=%llu, status=%#x",
791                   (long long unsigned int)scst_cmd_get_tag(cmd),
792                   status);
793
794 #ifdef CONFIG_SCST_EXTRACHECKS
795         if ((in_irq() || irqs_disabled()) &&
796             ((pref_context == SCST_CONTEXT_DIRECT) ||
797              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
798                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
799                         "SCST_CONTEXT_THREAD instead\n", pref_context,
800                         cmd->tgtt->name);
801                 pref_context = SCST_CONTEXT_THREAD;
802         }
803 #endif
804
805         switch (status) {
806         case SCST_PREPROCESS_STATUS_SUCCESS:
807                 switch (cmd->data_direction) {
808                 case SCST_DATA_WRITE:
809                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
810                         break;
811                 default:
812                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
813                         break;
814                 }
815                 if (cmd->set_sn_on_restart_cmd)
816                         scst_cmd_set_sn(cmd);
817                 /* Small context optimization */
818                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
819                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
820                     ((pref_context == SCST_CONTEXT_SAME) &&
821                      scst_cmd_atomic(cmd))) {
822                         if (cmd->data_direction == SCST_DATA_WRITE) {
823                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
824                                                 &cmd->tgt_dev->tgt_dev_flags))
825                                         pref_context = SCST_CONTEXT_THREAD;
826                         } else {
827                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
828                                                 &cmd->tgt_dev->tgt_dev_flags))
829                                         pref_context = SCST_CONTEXT_THREAD;
830                         }
831                 }
832                 break;
833
834         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
835                 scst_set_cmd_abnormal_done_state(cmd);
836                 break;
837
838         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
839                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
840                 /* go through */
841         case SCST_PREPROCESS_STATUS_ERROR:
842                 scst_set_cmd_error(cmd,
843                            SCST_LOAD_SENSE(scst_sense_hardw_error));
844                 scst_set_cmd_abnormal_done_state(cmd);
845                 break;
846
847         default:
848                 PRINT_ERROR("%s() received unknown status %x", __func__,
849                         status);
850                 scst_set_cmd_abnormal_done_state(cmd);
851                 break;
852         }
853
854         scst_proccess_redirect_cmd(cmd, pref_context, 1);
855
856         TRACE_EXIT();
857         return;
858 }
859 EXPORT_SYMBOL(scst_restart_cmd);
860
861 /* No locks */
862 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
863 {
864         struct scst_tgt *tgt = cmd->sess->tgt;
865         int res = 0;
866         unsigned long flags;
867
868         TRACE_ENTRY();
869
870         spin_lock_irqsave(&tgt->tgt_lock, flags);
871         tgt->retry_cmds++;
872         smp_mb();
873         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
874               tgt->retry_cmds);
875         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
876                 /* At least one cmd finished, so try again */
877                 tgt->retry_cmds--;
878                 TRACE_RETRY("Some command(s) finished, direct retry "
879                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
880                       "retry_cmds=%d)", finished_cmds,
881                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
882                 res = -1;
883                 goto out_unlock_tgt;
884         }
885
886         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
887         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
888
889         if (!tgt->retry_timer_active) {
890                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
891                 add_timer(&tgt->retry_timer);
892                 tgt->retry_timer_active = 1;
893         }
894
895 out_unlock_tgt:
896         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
897
898         TRACE_EXIT_RES(res);
899         return res;
900 }
901
902 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
903 {
904         int res, rc;
905
906         TRACE_ENTRY();
907
908         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
909                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
910                 goto out_dev_done;
911         }
912
913         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
914                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
915                 res = SCST_CMD_STATE_RES_CONT_SAME;
916                 goto out;
917         }
918
919         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
920                 /*
921                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
922                  * optimization.
923                  */
924                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
925                               "context, rescheduling", cmd->tgtt->name);
926                 res = SCST_CMD_STATE_RES_NEED_THREAD;
927                 goto out;
928         }
929
930         while (1) {
931                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
932
933                 res = SCST_CMD_STATE_RES_CONT_NEXT;
934                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
935
936                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
937 #ifdef CONFIG_SCST_DEBUG_RETRY
938                 if (((scst_random() % 100) == 75))
939                         rc = SCST_TGT_RES_QUEUE_FULL;
940                 else
941 #endif
942                         rc = cmd->tgtt->rdy_to_xfer(cmd);
943                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
944
945                 if (likely(rc == SCST_TGT_RES_SUCCESS))
946                         goto out;
947
948                 /* Restore the previous state */
949                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
950
951                 switch (rc) {
952                 case SCST_TGT_RES_QUEUE_FULL:
953                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
954                                 break;
955                         else
956                                 continue;
957
958                 case SCST_TGT_RES_NEED_THREAD_CTX:
959                         TRACE_DBG("Target driver %s "
960                               "rdy_to_xfer() requested thread "
961                               "context, rescheduling", cmd->tgtt->name);
962                         res = SCST_CMD_STATE_RES_NEED_THREAD;
963                         break;
964
965                 default:
966                         goto out_error_rc;
967                 }
968                 break;
969         }
970
971 out:
972         TRACE_EXIT_HRES(res);
973         return res;
974
975 out_error_rc:
976         if (rc == SCST_TGT_RES_FATAL_ERROR) {
977                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
978                      "fatal error", cmd->tgtt->name);
979         } else {
980                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
981                             "value %d", cmd->tgtt->name, rc);
982         }
983         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
984
985 out_dev_done:
986         scst_set_cmd_abnormal_done_state(cmd);
987         res = SCST_CMD_STATE_RES_CONT_SAME;
988         goto out;
989 }
990
991 /* No locks, but might be in IRQ */
992 void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
993         enum scst_exec_context context, int check_retries)
994 {
995         unsigned long flags;
996
997         TRACE_ENTRY();
998
999         TRACE_DBG("Context: %x", context);
1000
1001         if (context == SCST_CONTEXT_SAME)
1002                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1003                                                  SCST_CONTEXT_DIRECT;
1004
1005         switch (context) {
1006         case SCST_CONTEXT_DIRECT_ATOMIC:
1007                 scst_process_active_cmd(cmd, true);
1008                 break;
1009
1010         case SCST_CONTEXT_DIRECT:
1011                 if (check_retries)
1012                         scst_check_retries(cmd->tgt);
1013                 scst_process_active_cmd(cmd, false);
1014                 break;
1015
1016         default:
1017                 PRINT_ERROR("Context %x is unknown, using the thread one",
1018                             context);
1019                 /* go through */
1020         case SCST_CONTEXT_THREAD:
1021                 if (check_retries)
1022                         scst_check_retries(cmd->tgt);
1023                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1024                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1025                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1026                         list_add(&cmd->cmd_list_entry,
1027                                 &cmd->cmd_lists->active_cmd_list);
1028                 else
1029                         list_add_tail(&cmd->cmd_list_entry,
1030                                 &cmd->cmd_lists->active_cmd_list);
1031                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1032                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1033                 break;
1034
1035         case SCST_CONTEXT_TASKLET:
1036                 if (check_retries)
1037                         scst_check_retries(cmd->tgt);
1038                 scst_schedule_tasklet(cmd);
1039                 break;
1040         }
1041
1042         TRACE_EXIT();
1043         return;
1044 }
1045
1046 void scst_rx_data(struct scst_cmd *cmd, int status,
1047         enum scst_exec_context pref_context)
1048 {
1049         TRACE_ENTRY();
1050
1051         TRACE_DBG("Preferred context: %d", pref_context);
1052         TRACE(TRACE_SCSI, "tag=%llu status=%#x",
1053               (long long unsigned int)scst_cmd_get_tag(cmd),
1054               status);
1055
1056 #ifdef CONFIG_SCST_EXTRACHECKS
1057         if ((in_irq() || irqs_disabled()) &&
1058             ((pref_context == SCST_CONTEXT_DIRECT) ||
1059              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1060                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1061                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1062                         cmd->tgtt->name);
1063                 pref_context = SCST_CONTEXT_THREAD;
1064         }
1065 #endif
1066
1067         switch (status) {
1068         case SCST_RX_STATUS_SUCCESS:
1069                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1070                 /* Small context optimization */
1071                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1072                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1073                     ((pref_context == SCST_CONTEXT_SAME) &&
1074                      scst_cmd_atomic(cmd))) {
1075                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1076                                         &cmd->tgt_dev->tgt_dev_flags))
1077                                 pref_context = SCST_CONTEXT_THREAD;
1078                 }
1079                 break;
1080
1081         case SCST_RX_STATUS_ERROR_SENSE_SET:
1082                 scst_set_cmd_abnormal_done_state(cmd);
1083                 break;
1084
1085         case SCST_RX_STATUS_ERROR_FATAL:
1086                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1087                 /* go through */
1088         case SCST_RX_STATUS_ERROR:
1089                 scst_set_cmd_error(cmd,
1090                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1091                 scst_set_cmd_abnormal_done_state(cmd);
1092                 break;
1093
1094         default:
1095                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1096                         status);
1097                 scst_set_cmd_abnormal_done_state(cmd);
1098                 break;
1099         }
1100
1101         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1102
1103         TRACE_EXIT();
1104         return;
1105 }
1106 EXPORT_SYMBOL(scst_rx_data);
1107
1108 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1109 {
1110         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1111
1112         TRACE_ENTRY();
1113
1114         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1115
1116         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1117                 goto out;
1118
1119         TRACE_DBG("Calling pre_exec(%p)", cmd);
1120         rc = cmd->tgtt->pre_exec(cmd);
1121         TRACE_DBG("pre_exec() returned %d", rc);
1122
1123         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1124                 switch (rc) {
1125                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1126                         scst_set_cmd_abnormal_done_state(cmd);
1127                         break;
1128                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1129                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1130                         /* go through */
1131                 case SCST_PREPROCESS_STATUS_ERROR:
1132                         scst_set_cmd_error(cmd,
1133                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1134                         scst_set_cmd_abnormal_done_state(cmd);
1135                         break;
1136                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1137                         TRACE_DBG("Target driver's %s pre_exec() requested "
1138                                 "thread context, rescheduling",
1139                                 cmd->tgtt->name);
1140                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1141                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1142                         break;
1143                 default:
1144                         sBUG();
1145                         break;
1146                 }
1147         }
1148
1149 out:
1150         TRACE_EXIT_RES(res);
1151         return res;
1152 }
1153
1154 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1155         const uint8_t *rq_sense, int rq_sense_len, int resid)
1156 {
1157         TRACE_ENTRY();
1158
1159 #ifdef CONFIG_SCST_MEASURE_LATENCY
1160         {
1161                 struct timespec ts;
1162                 getnstimeofday(&ts);
1163                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1164                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1165                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1166                         ts.tv_sec, ts.tv_nsec);
1167         }
1168 #endif
1169
1170         cmd->status = result & 0xff;
1171         cmd->msg_status = msg_byte(result);
1172         cmd->host_status = host_byte(result);
1173         cmd->driver_status = driver_byte(result);
1174         if (unlikely(resid != 0)) {
1175 #ifdef CONFIG_SCST_EXTRACHECKS
1176                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1177                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1178                                 "op %x)", resid, cmd->resp_data_len,
1179                                 cmd->cdb[0]);
1180                 } else
1181 #endif
1182                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1183         }
1184
1185         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1186                 /* We might have double reset UA here */
1187                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1188                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1189
1190                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1191         }
1192
1193         TRACE(TRACE_SCSI, "cmd%p, result=%x, cmd->status=%x, resid=%d, "
1194               "cmd->msg_status=%x, cmd->host_status=%x, "
1195               "cmd->driver_status=%x", cmd, result, cmd->status, resid,
1196               cmd->msg_status, cmd->host_status, cmd->driver_status);
1197
1198         cmd->completed = 1;
1199
1200         TRACE_EXIT();
1201         return;
1202 }
1203
1204 /* For small context optimization */
1205 static inline enum scst_exec_context scst_optimize_post_exec_context(
1206         struct scst_cmd *cmd, enum scst_exec_context context)
1207 {
1208         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1209             (context == SCST_CONTEXT_TASKLET) ||
1210             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1211                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1212                                 &cmd->tgt_dev->tgt_dev_flags))
1213                         context = SCST_CONTEXT_THREAD;
1214         }
1215         return context;
1216 }
1217
1218 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1219 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1220                                             struct scsi_request **req)
1221 {
1222         struct scst_cmd *cmd = NULL;
1223
1224         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1225                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1226
1227         if (cmd == NULL) {
1228                 PRINT_ERROR("%s", "Request with NULL cmd");
1229                 if (*req)
1230                         scsi_release_request(*req);
1231         }
1232
1233         return cmd;
1234 }
1235
1236 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1237 {
1238         struct scsi_request *req = NULL;
1239         struct scst_cmd *cmd;
1240         enum scst_exec_context context;
1241
1242         TRACE_ENTRY();
1243
1244         cmd = scst_get_cmd(scsi_cmd, &req);
1245         if (cmd == NULL)
1246                 goto out;
1247
1248         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1249                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1250
1251         /* Clear out request structure */
1252         req->sr_use_sg = 0;
1253         req->sr_sglist_len = 0;
1254         req->sr_bufflen = 0;
1255         req->sr_buffer = NULL;
1256         req->sr_underflow = 0;
1257         req->sr_request->rq_disk = NULL; /* disown request blk */
1258
1259         scst_release_request(cmd);
1260
1261         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1262
1263         scst_proccess_redirect_cmd(cmd,
1264                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1265                                                 0);
1266
1267 out:
1268         TRACE_EXIT();
1269         return;
1270 }
1271 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1272 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1273 {
1274         struct scst_cmd *cmd;
1275
1276         TRACE_ENTRY();
1277
1278         cmd = (struct scst_cmd *)data;
1279         if (cmd == NULL)
1280                 goto out;
1281
1282         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1283
1284         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1285
1286         scst_proccess_redirect_cmd(cmd,
1287                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1288                                                 0);
1289
1290 out:
1291         TRACE_EXIT();
1292         return;
1293 }
1294 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1295
1296 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1297         enum scst_exec_context pref_context)
1298 {
1299         TRACE_ENTRY();
1300
1301 #ifdef CONFIG_SCST_MEASURE_LATENCY
1302         {
1303                 struct timespec ts;
1304                 getnstimeofday(&ts);
1305                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1306                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1307                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1308                         ts.tv_sec, ts.tv_nsec);
1309         }
1310 #endif
1311
1312         if (next_state == SCST_CMD_STATE_DEFAULT)
1313                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1314
1315 #if defined(CONFIG_SCST_DEBUG)
1316         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1317                 if (cmd->sg) {
1318                         int i;
1319                         struct scatterlist *sg = cmd->sg;
1320                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1321                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1322                         for (i = 0; i < cmd->sg_cnt; ++i) {
1323                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1324                                         "Exec'd sg", sg_virt(&sg[i]),
1325                                         sg[i].length);
1326                         }
1327                 }
1328         }
1329 #endif
1330
1331         cmd->state = next_state;
1332
1333 #ifdef CONFIG_SCST_EXTRACHECKS
1334         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1335             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1336             (next_state != SCST_CMD_STATE_FINISHED)) {
1337                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1338                         __func__, next_state, cmd->cdb[0]);
1339                 scst_set_cmd_error(cmd,
1340                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1341                 scst_set_cmd_abnormal_done_state(cmd);
1342         }
1343 #endif
1344         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1345         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1346
1347         TRACE_EXIT();
1348         return;
1349 }
1350
1351 static int scst_report_luns_local(struct scst_cmd *cmd)
1352 {
1353         int rc;
1354         int dev_cnt = 0;
1355         int buffer_size;
1356         int i;
1357         struct scst_tgt_dev *tgt_dev = NULL;
1358         uint8_t *buffer;
1359         int offs, overflow = 0;
1360
1361         TRACE_ENTRY();
1362
1363         rc = scst_check_local_events(cmd);
1364         if (unlikely(rc != 0))
1365                 goto out_done;
1366
1367         cmd->status = 0;
1368         cmd->msg_status = 0;
1369         cmd->host_status = DID_OK;
1370         cmd->driver_status = 0;
1371
1372         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1373                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1374                         "LUNS command", cmd->cdb[2]);
1375                 goto out_err;
1376         }
1377
1378         buffer_size = scst_get_buf_first(cmd, &buffer);
1379         if (unlikely(buffer_size == 0))
1380                 goto out_compl;
1381         else if (unlikely(buffer_size < 0))
1382                 goto out_err;
1383
1384         if (buffer_size < 16)
1385                 goto out_put_err;
1386
1387         memset(buffer, 0, buffer_size);
1388         offs = 8;
1389
1390         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1391         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1392                 struct list_head *sess_tgt_dev_list_head =
1393                         &cmd->sess->sess_tgt_dev_list_hash[i];
1394                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1395                                 sess_tgt_dev_list_entry) {
1396                         if (!overflow) {
1397                                 if (offs >= buffer_size) {
1398                                         scst_put_buf(cmd, buffer);
1399                                         buffer_size = scst_get_buf_next(cmd,
1400                                                                        &buffer);
1401                                         if (buffer_size > 0) {
1402                                                 memset(buffer, 0, buffer_size);
1403                                                 offs = 0;
1404                                         } else {
1405                                                 overflow = 1;
1406                                                 goto inc_dev_cnt;
1407                                         }
1408                                 }
1409                                 if ((buffer_size - offs) < 8) {
1410                                         PRINT_ERROR("Buffer allocated for "
1411                                                 "REPORT LUNS command doesn't "
1412                                                 "allow to fit 8 byte entry "
1413                                                 "(buffer_size=%d)",
1414                                                 buffer_size);
1415                                         goto out_put_hw_err;
1416                                 }
1417                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1418                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1419                                 offs += 8;
1420                         }
1421 inc_dev_cnt:
1422                         dev_cnt++;
1423                 }
1424         }
1425         if (!overflow)
1426                 scst_put_buf(cmd, buffer);
1427
1428         /* Set the response header */
1429         buffer_size = scst_get_buf_first(cmd, &buffer);
1430         if (unlikely(buffer_size == 0))
1431                 goto out_compl;
1432         else if (unlikely(buffer_size < 0))
1433                 goto out_err;
1434
1435         dev_cnt *= 8;
1436         buffer[0] = (dev_cnt >> 24) & 0xff;
1437         buffer[1] = (dev_cnt >> 16) & 0xff;
1438         buffer[2] = (dev_cnt >> 8) & 0xff;
1439         buffer[3] = dev_cnt & 0xff;
1440
1441         scst_put_buf(cmd, buffer);
1442
1443         dev_cnt += 8;
1444         if (dev_cnt < cmd->resp_data_len)
1445                 scst_set_resp_data_len(cmd, dev_cnt);
1446
1447 out_compl:
1448         cmd->completed = 1;
1449
1450 out_done:
1451         /* Report the result */
1452         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1453
1454         TRACE_EXIT();
1455         return SCST_EXEC_COMPLETED;
1456
1457 out_put_err:
1458         scst_put_buf(cmd, buffer);
1459
1460 out_err:
1461         scst_set_cmd_error(cmd,
1462                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1463         goto out_compl;
1464
1465 out_put_hw_err:
1466         scst_put_buf(cmd, buffer);
1467         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1468         goto out_compl;
1469 }
1470
1471 static int scst_pre_select(struct scst_cmd *cmd)
1472 {
1473         int res = SCST_EXEC_NOT_COMPLETED;
1474
1475         TRACE_ENTRY();
1476
1477         if (scst_cmd_atomic(cmd)) {
1478                 res = SCST_EXEC_NEED_THREAD;
1479                 goto out;
1480         }
1481
1482         scst_block_dev_cmd(cmd, 1);
1483
1484         /* Check for local events will be done when cmd will be executed */
1485
1486 out:
1487         TRACE_EXIT_RES(res);
1488         return res;
1489 }
1490
1491 static int scst_reserve_local(struct scst_cmd *cmd)
1492 {
1493         int res = SCST_EXEC_NOT_COMPLETED, rc;
1494         struct scst_device *dev;
1495         struct scst_tgt_dev *tgt_dev_tmp;
1496
1497         TRACE_ENTRY();
1498
1499         if (scst_cmd_atomic(cmd)) {
1500                 res = SCST_EXEC_NEED_THREAD;
1501                 goto out;
1502         }
1503
1504         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1505                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1506                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1507                 scst_set_cmd_error(cmd,
1508                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1509                 goto out_done;
1510         }
1511
1512         dev = cmd->dev;
1513
1514         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1515                 scst_block_dev_cmd(cmd, 1);
1516
1517         rc = scst_check_local_events(cmd);
1518         if (unlikely(rc != 0))
1519                 goto out_done;
1520
1521         spin_lock_bh(&dev->dev_lock);
1522
1523         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1524                 spin_unlock_bh(&dev->dev_lock);
1525                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1526                 goto out_done;
1527         }
1528
1529         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1530                             dev_tgt_dev_list_entry) {
1531                 if (cmd->tgt_dev != tgt_dev_tmp)
1532                         set_bit(SCST_TGT_DEV_RESERVED,
1533                                 &tgt_dev_tmp->tgt_dev_flags);
1534         }
1535         dev->dev_reserved = 1;
1536
1537         spin_unlock_bh(&dev->dev_lock);
1538
1539 out:
1540         TRACE_EXIT_RES(res);
1541         return res;
1542
1543 out_done:
1544         /* Report the result */
1545         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1546         res = SCST_EXEC_COMPLETED;
1547         goto out;
1548 }
1549
1550 static int scst_release_local(struct scst_cmd *cmd)
1551 {
1552         int res = SCST_EXEC_NOT_COMPLETED, rc;
1553         struct scst_tgt_dev *tgt_dev_tmp;
1554         struct scst_device *dev;
1555
1556         TRACE_ENTRY();
1557
1558         if (scst_cmd_atomic(cmd)) {
1559                 res = SCST_EXEC_NEED_THREAD;
1560                 goto out;
1561         }
1562
1563         dev = cmd->dev;
1564
1565         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1566                 scst_block_dev_cmd(cmd, 1);
1567
1568         rc = scst_check_local_events(cmd);
1569         if (unlikely(rc != 0))
1570                 goto out_done;
1571
1572         spin_lock_bh(&dev->dev_lock);
1573
1574         /*
1575          * The device could be RELEASED behind us, if RESERVING session
1576          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1577          * matter, so use lock and no retest for DEV_RESERVED bits again
1578          */
1579         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1580                 res = SCST_EXEC_COMPLETED;
1581                 cmd->status = 0;
1582                 cmd->msg_status = 0;
1583                 cmd->host_status = DID_OK;
1584                 cmd->driver_status = 0;
1585                 cmd->completed = 1;
1586         } else {
1587                 list_for_each_entry(tgt_dev_tmp,
1588                                     &dev->dev_tgt_dev_list,
1589                                     dev_tgt_dev_list_entry) {
1590                         clear_bit(SCST_TGT_DEV_RESERVED,
1591                                 &tgt_dev_tmp->tgt_dev_flags);
1592                 }
1593                 dev->dev_reserved = 0;
1594         }
1595
1596         spin_unlock_bh(&dev->dev_lock);
1597
1598         if (res == SCST_EXEC_COMPLETED)
1599                 goto out_done;
1600
1601 out:
1602         TRACE_EXIT_RES(res);
1603         return res;
1604
1605 out_done:
1606         res = SCST_EXEC_COMPLETED;
1607         /* Report the result */
1608         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1609         goto out;
1610 }
1611
1612 /* No locks, no IRQ or IRQ-safe context allowed */
1613 int scst_check_local_events(struct scst_cmd *cmd)
1614 {
1615         int res, rc;
1616         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1617         struct scst_device *dev = cmd->dev;
1618
1619         TRACE_ENTRY();
1620
1621         /*
1622          * There's no race here, because we need to trace commands sent
1623          * *after* dev_double_ua_possible flag was set.
1624          */
1625         if (unlikely(dev->dev_double_ua_possible))
1626                 cmd->double_ua_possible = 1;
1627
1628         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1629                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1630                 goto out_uncomplete;
1631         }
1632
1633         /* Reserve check before Unit Attention */
1634         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1635                               &tgt_dev->tgt_dev_flags))) {
1636                 if (cmd->cdb[0] != INQUIRY &&
1637                     cmd->cdb[0] != REPORT_LUNS &&
1638                     cmd->cdb[0] != RELEASE &&
1639                     cmd->cdb[0] != RELEASE_10 &&
1640                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1641                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1642                      (cmd->cdb[4] & 3)) &&
1643                     cmd->cdb[0] != LOG_SENSE &&
1644                     cmd->cdb[0] != REQUEST_SENSE) {
1645                         scst_set_cmd_error_status(cmd,
1646                                 SAM_STAT_RESERVATION_CONFLICT);
1647                         goto out_complete;
1648                 }
1649         }
1650
1651         /* If we had internal bus reset, set the command error unit attention */
1652         if ((dev->scsi_dev != NULL) &&
1653             unlikely(dev->scsi_dev->was_reset)) {
1654                 if (scst_is_ua_command(cmd)) {
1655                         int done = 0;
1656                         /*
1657                          * Prevent more than 1 cmd to be triggered by
1658                          * was_reset.
1659                          */
1660                         spin_lock_bh(&dev->dev_lock);
1661                         barrier(); /* to reread was_reset */
1662                         if (dev->scsi_dev->was_reset) {
1663                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1664                                 scst_set_cmd_error(cmd,
1665                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1666                                 /*
1667                                  * It looks like it is safe to clear was_reset
1668                                  * here.
1669                                  */
1670                                 dev->scsi_dev->was_reset = 0;
1671                                 done = 1;
1672                         }
1673                         spin_unlock_bh(&dev->dev_lock);
1674
1675                         if (done)
1676                                 goto out_complete;
1677                 }
1678         }
1679
1680         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1681                         &cmd->tgt_dev->tgt_dev_flags))) {
1682                 if (scst_is_ua_command(cmd)) {
1683                         rc = scst_set_pending_UA(cmd);
1684                         if (rc == 0)
1685                                 goto out_complete;
1686                 }
1687         }
1688
1689         res = 0;
1690
1691 out:
1692         TRACE_EXIT_RES(res);
1693         return res;
1694
1695 out_complete:
1696         res = 1;
1697         sBUG_ON(!cmd->completed);
1698         goto out;
1699
1700 out_uncomplete:
1701         res = -1;
1702         goto out;
1703 }
1704 EXPORT_SYMBOL(scst_check_local_events);
1705
1706 /* No locks */
1707 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1708 {
1709         if (slot == NULL)
1710                 goto inc;
1711
1712         /* Optimized for lockless fast path */
1713
1714         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1715                 atomic_read(slot));
1716
1717         if (!atomic_dec_and_test(slot))
1718                 goto out;
1719
1720         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1721                 tgt_dev->num_free_sn_slots);
1722         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1723                 spin_lock_irq(&tgt_dev->sn_lock);
1724                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1725                         if (tgt_dev->num_free_sn_slots < 0)
1726                                 tgt_dev->cur_sn_slot = slot;
1727                         /*
1728                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1729                          */
1730                         smp_mb();
1731                         tgt_dev->num_free_sn_slots++;
1732                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1733                                 tgt_dev->num_free_sn_slots);
1734
1735                 }
1736                 spin_unlock_irq(&tgt_dev->sn_lock);
1737         }
1738
1739 inc:
1740         /*
1741          * No locks is needed, because only one thread at time can
1742          * be here (serialized by sn). Also it is supposed that there
1743          * could not be half-incremented halves.
1744          */
1745         tgt_dev->expected_sn++;
1746         smp_mb(); /* write must be before def_cmd_count read */
1747         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1748
1749 out:
1750         return;
1751 }
1752
1753 /* No locks */
1754 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1755         bool make_active)
1756 {
1757         /* For HQ commands SN is not set */
1758         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1759                                cmd->sn_set && !cmd->retry;
1760         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1761         struct scst_cmd *res;
1762
1763         TRACE_ENTRY();
1764
1765         if (inc_expected_sn)
1766                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1767
1768         if (make_active) {
1769                 scst_make_deferred_commands_active(tgt_dev);
1770                 res = NULL;
1771         } else
1772                 res = scst_check_deferred_commands(tgt_dev);
1773
1774         TRACE_EXIT_HRES(res);
1775         return res;
1776 }
1777
1778 /* cmd must be additionally referenced to not die inside */
1779 static int scst_do_real_exec(struct scst_cmd *cmd)
1780 {
1781         int res = SCST_EXEC_NOT_COMPLETED, rc;
1782         struct scst_device *dev = cmd->dev;
1783         struct scst_dev_type *handler = dev->handler;
1784
1785         TRACE_ENTRY();
1786
1787         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1788
1789         if (handler->exec) {
1790                 if (unlikely(!dev->handler->exec_atomic &&
1791                              scst_cmd_atomic(cmd))) {
1792                         /*
1793                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1794                          * optimization.
1795                          */
1796                         TRACE_DBG("Dev handler %s exec() needs thread "
1797                                 "context, rescheduling", dev->handler->name);
1798                         res = SCST_EXEC_NEED_THREAD;
1799                         goto out_restore;
1800                 }
1801
1802                 TRACE_DBG("Calling dev handler %s exec(%p)",
1803                       handler->name, cmd);
1804                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1805                         cmd->cdb_len);
1806                 res = handler->exec(cmd);
1807                 TRACE_DBG("Dev handler %s exec() returned %d",
1808                       handler->name, res);
1809
1810                 if (res == SCST_EXEC_COMPLETED)
1811                         goto out_complete;
1812                 else if (res == SCST_EXEC_NEED_THREAD)
1813                         goto out_restore;
1814
1815                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1816         }
1817
1818         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1819
1820         if (unlikely(dev->scsi_dev == NULL)) {
1821                 PRINT_ERROR("Command for virtual device must be "
1822                         "processed by device handler (lun %lld)!",
1823                         (long long unsigned int)cmd->lun);
1824                 goto out_error;
1825         }
1826
1827         res = scst_check_local_events(cmd);
1828         if (unlikely(res != 0))
1829                 goto out_done;
1830
1831 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1832         if (unlikely(scst_cmd_atomic(cmd))) {
1833                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1834                         "context, rescheduling to the thread (handler %s)",
1835                         handler->name);
1836                 res = SCST_EXEC_NEED_THREAD;
1837                 goto out_restore;
1838         }
1839 #endif
1840
1841 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1842         if (unlikely(scst_alloc_request(cmd) != 0)) {
1843                 if (scst_cmd_atomic(cmd)) {
1844                         res = SCST_EXEC_NEED_THREAD;
1845                         goto out_restore;
1846                 } else {
1847                         PRINT_INFO("%s", "Unable to allocate request, "
1848                                 "sending BUSY status");
1849                         goto out_busy;
1850                 }
1851         }
1852
1853         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1854                     (void *)cmd->scsi_req->sr_buffer,
1855                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1856                     cmd->retries);
1857 #else
1858         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1859                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1860                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1861                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1862         if (unlikely(rc != 0)) {
1863                 if (scst_cmd_atomic(cmd)) {
1864                         res = SCST_EXEC_NEED_THREAD;
1865                         goto out_restore;
1866                 } else {
1867                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1868                         goto out_error;
1869                 }
1870         }
1871 #endif
1872
1873 out_complete:
1874         res = SCST_EXEC_COMPLETED;
1875
1876 out:
1877         TRACE_EXIT();
1878         return res;
1879
1880 out_restore:
1881         /* Restore the state */
1882         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1883         goto out;
1884
1885 out_error:
1886         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1887         goto out_done;
1888
1889 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1890 out_busy:
1891         scst_set_busy(cmd);
1892         /* go through */
1893 #endif
1894
1895 out_done:
1896         res = SCST_EXEC_COMPLETED;
1897         /* Report the result */
1898         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1899         goto out_complete;
1900 }
1901
1902 static inline int scst_real_exec(struct scst_cmd *cmd)
1903 {
1904         int res;
1905
1906         TRACE_ENTRY();
1907
1908         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1909         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1910         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1911
1912         __scst_cmd_get(cmd);
1913
1914         res = scst_do_real_exec(cmd);
1915
1916         if (likely(res == SCST_EXEC_COMPLETED)) {
1917                 scst_post_exec_sn(cmd, true);
1918                 if (cmd->dev->scsi_dev != NULL)
1919                         generic_unplug_device(
1920                                 cmd->dev->scsi_dev->request_queue);
1921         } else
1922                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1923
1924         __scst_cmd_put(cmd);
1925
1926         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1927
1928         TRACE_EXIT_RES(res);
1929         return res;
1930 }
1931
1932 static int scst_do_local_exec(struct scst_cmd *cmd)
1933 {
1934         int res;
1935         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1936
1937         TRACE_ENTRY();
1938
1939         /* Check READ_ONLY device status */
1940         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1941             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1942              cmd->cdb[0] == WRITE_10 ||
1943              cmd->cdb[0] == WRITE_12 ||
1944              cmd->cdb[0] == WRITE_16 ||
1945              cmd->cdb[0] == WRITE_VERIFY ||
1946              cmd->cdb[0] == WRITE_VERIFY_12 ||
1947              cmd->cdb[0] == WRITE_VERIFY_16 ||
1948              (cmd->dev->handler->type == TYPE_TAPE &&
1949               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1950                 scst_set_cmd_error(cmd,
1951                            SCST_LOAD_SENSE(scst_sense_data_protect));
1952                 goto out_done;
1953         }
1954
1955         /*
1956          * Adding new commands here don't forget to update
1957          * scst_is_cmd_local() in scst.h, if necessary
1958          */
1959
1960         switch (cmd->cdb[0]) {
1961         case MODE_SELECT:
1962         case MODE_SELECT_10:
1963         case LOG_SELECT:
1964                 res = scst_pre_select(cmd);
1965                 break;
1966         case RESERVE:
1967         case RESERVE_10:
1968                 res = scst_reserve_local(cmd);
1969                 break;
1970         case RELEASE:
1971         case RELEASE_10:
1972                 res = scst_release_local(cmd);
1973                 break;
1974         case REPORT_LUNS:
1975                 res = scst_report_luns_local(cmd);
1976                 break;
1977         default:
1978                 res = SCST_EXEC_NOT_COMPLETED;
1979                 break;
1980         }
1981
1982 out:
1983         TRACE_EXIT_RES(res);
1984         return res;
1985
1986 out_done:
1987         /* Report the result */
1988         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1989         res = SCST_EXEC_COMPLETED;
1990         goto out;
1991 }
1992
1993 static int scst_local_exec(struct scst_cmd *cmd)
1994 {
1995         int res;
1996
1997         TRACE_ENTRY();
1998
1999         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2000         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2001         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2002
2003         __scst_cmd_get(cmd);
2004
2005         res = scst_do_local_exec(cmd);
2006         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2007                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2008         else if (res == SCST_EXEC_COMPLETED)
2009                 scst_post_exec_sn(cmd, true);
2010         else
2011                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2012
2013         __scst_cmd_put(cmd);
2014
2015         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2016         TRACE_EXIT_RES(res);
2017         return res;
2018 }
2019
2020 static int scst_exec(struct scst_cmd **active_cmd)
2021 {
2022         struct scst_cmd *cmd = *active_cmd;
2023         struct scst_cmd *ref_cmd;
2024         struct scst_device *dev = cmd->dev;
2025         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2026
2027         TRACE_ENTRY();
2028
2029         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2030                 goto out;
2031
2032         /* To protect tgt_dev */
2033         ref_cmd = cmd;
2034         __scst_cmd_get(ref_cmd);
2035
2036         count = 0;
2037         while (1) {
2038                 int rc;
2039
2040                 cmd->sent_for_exec = 1;
2041                 cmd->scst_cmd_done = scst_cmd_done_local;
2042                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2043
2044                 rc = scst_do_local_exec(cmd);
2045                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2046                         /* Nothing to do */;
2047                 else if (rc == SCST_EXEC_NEED_THREAD) {
2048                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2049                                 "thread context, rescheduling");
2050                         scst_dec_on_dev_cmd(cmd);
2051                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2052                         break;
2053                 } else {
2054                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2055                         goto done;
2056                 }
2057
2058                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2059
2060                 rc = scst_do_real_exec(cmd);
2061                 if (likely(rc == SCST_EXEC_COMPLETED))
2062                         /* Nothing to do */;
2063                 else if (rc == SCST_EXEC_NEED_THREAD) {
2064                         TRACE_DBG("scst_real_exec() requested thread "
2065                                 "context, rescheduling (cmd %p)", cmd);
2066                         scst_dec_on_dev_cmd(cmd);
2067                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2068                         break;
2069                 } else
2070                         sBUG();
2071
2072 done:
2073                 count++;
2074
2075                 cmd = scst_post_exec_sn(cmd, false);
2076                 if (cmd == NULL)
2077                         break;
2078
2079                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2080                         break;
2081
2082                 __scst_cmd_put(ref_cmd);
2083                 ref_cmd = cmd;
2084                 __scst_cmd_get(ref_cmd);
2085         }
2086
2087         *active_cmd = cmd;
2088
2089         if (count == 0)
2090                 goto out_put;
2091
2092         if (dev->scsi_dev != NULL)
2093                 generic_unplug_device(dev->scsi_dev->request_queue);
2094
2095 out_put:
2096         __scst_cmd_put(ref_cmd);
2097         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2098
2099 out:
2100         TRACE_EXIT_RES(res);
2101         return res;
2102 }
2103
2104 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2105 {
2106         int res;
2107         struct scst_cmd *cmd = *active_cmd;
2108         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2109         typeof(tgt_dev->expected_sn) expected_sn;
2110
2111         TRACE_ENTRY();
2112
2113 #ifdef CONFIG_SCST_MEASURE_LATENCY
2114         if (cmd->pre_exec_finish == 0) {
2115                 struct timespec ts;
2116                 getnstimeofday(&ts);
2117                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2118                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2119                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2120                         ts.tv_sec, ts.tv_nsec);
2121         }
2122 #endif
2123
2124         if (unlikely(cmd->internal))
2125                 goto exec;
2126
2127         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2128                 goto exec;
2129
2130         sBUG_ON(!cmd->sn_set);
2131
2132         expected_sn = tgt_dev->expected_sn;
2133         /* Optimized for lockless fast path */
2134         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2135                 spin_lock_irq(&tgt_dev->sn_lock);
2136
2137                 tgt_dev->def_cmd_count++;
2138                 smp_mb();
2139
2140                 expected_sn = tgt_dev->expected_sn;
2141                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2142                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2143                                               &cmd->cmd_flags))) {
2144                                 /* Necessary to allow aborting out of sn cmds */
2145                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2146                                         "(tag %llu, sn %lu)", cmd,
2147                                         (long long unsigned)cmd->tag, cmd->sn);
2148                                 tgt_dev->def_cmd_count--;
2149                                 scst_set_cmd_abnormal_done_state(cmd);
2150                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2151                         } else {
2152                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2153                                         "expected_sn=%ld)", cmd, cmd->sn,
2154                                         cmd->sn_set, expected_sn);
2155                                 list_add_tail(&cmd->sn_cmd_list_entry,
2156                                               &tgt_dev->deferred_cmd_list);
2157                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2158                         }
2159                         spin_unlock_irq(&tgt_dev->sn_lock);
2160                         goto out;
2161                 } else {
2162                         TRACE_SN("Somebody incremented expected_sn %ld, "
2163                                 "continuing", expected_sn);
2164                         tgt_dev->def_cmd_count--;
2165                         spin_unlock_irq(&tgt_dev->sn_lock);
2166                 }
2167         }
2168
2169 exec:
2170         res = scst_exec(active_cmd);
2171
2172 out:
2173         TRACE_EXIT_HRES(res);
2174         return res;
2175 }
2176
2177 /* No locks supposed to be held */
2178 static int scst_check_sense(struct scst_cmd *cmd)
2179 {
2180         int res = 0;
2181         struct scst_device *dev = cmd->dev;
2182
2183         TRACE_ENTRY();
2184
2185         if (unlikely(cmd->ua_ignore))
2186                 goto out;
2187
2188         /* If we had internal bus reset behind us, set the command error UA */
2189         if ((dev->scsi_dev != NULL) &&
2190             unlikely(cmd->host_status == DID_RESET) &&
2191             scst_is_ua_command(cmd)) {
2192                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2193                       dev->scsi_dev->was_reset, cmd->host_status);
2194                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2195                 /* It looks like it is safe to clear was_reset here */
2196                 dev->scsi_dev->was_reset = 0;
2197         }
2198
2199         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2200             SCST_SENSE_VALID(cmd->sense)) {
2201                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2202                         SCST_SENSE_BUFFERSIZE);
2203
2204                 /* Check Unit Attention Sense Key */
2205                 if (scst_is_ua_sense(cmd->sense)) {
2206                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2207                                 if (cmd->double_ua_possible) {
2208                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2209                                                 "detected for device %p", dev);
2210                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2211                                                 " %p (tag %llu)", cmd,
2212                                                 (long long unsigned)cmd->tag);
2213
2214                                         cmd->status = 0;
2215                                         cmd->msg_status = 0;
2216                                         cmd->host_status = DID_OK;
2217                                         cmd->driver_status = 0;
2218
2219                                         mempool_free(cmd->sense,
2220                                                      scst_sense_mempool);
2221                                         cmd->sense = NULL;
2222
2223                                         scst_check_restore_sg_buff(cmd);
2224
2225                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2226                                         cmd->data_direction =
2227                                                 cmd->dbl_ua_orig_data_direction;
2228                                         cmd->resp_data_len =
2229                                                 cmd->dbl_ua_orig_resp_data_len;
2230
2231                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2232                                         cmd->retry = 1;
2233                                         res = 1;
2234                                         goto out;
2235                                 }
2236                         }
2237                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2238                                 SCST_SENSE_BUFFERSIZE);
2239                 }
2240         }
2241
2242         if (unlikely(cmd->double_ua_possible)) {
2243                 if (scst_is_ua_command(cmd)) {
2244                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2245                                 "cmd %p)", dev, cmd);
2246                         /*
2247                          * Lock used to protect other flags in the bitfield
2248                          * (just in case, actually). Those 2 flags can't be
2249                          * changed in parallel, because the device is
2250                          * serialized.
2251                          */
2252                         spin_lock_bh(&dev->dev_lock);
2253                         dev->dev_double_ua_possible = 0;
2254                         dev->dev_serialized = 0;
2255                         spin_unlock_bh(&dev->dev_lock);
2256                 }
2257         }
2258
2259 out:
2260         TRACE_EXIT_RES(res);
2261         return res;
2262 }
2263
2264 static int scst_check_auto_sense(struct scst_cmd *cmd)
2265 {
2266         int res = 0;
2267
2268         TRACE_ENTRY();
2269
2270         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2271             (!SCST_SENSE_VALID(cmd->sense) ||
2272              SCST_NO_SENSE(cmd->sense))) {
2273                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2274                       "cmd->status=%x, cmd->msg_status=%x, "
2275                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2276                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2277                 res = 1;
2278         } else if (unlikely(cmd->host_status)) {
2279                 if ((cmd->host_status == DID_REQUEUE) ||
2280                     (cmd->host_status == DID_IMM_RETRY) ||
2281                     (cmd->host_status == DID_SOFT_ERROR) ||
2282                     (cmd->host_status == DID_ABORT)) {
2283                         scst_set_busy(cmd);
2284                 } else {
2285                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2286                                 "received, returning HARDWARE ERROR instead",
2287                                 cmd->host_status);
2288                         scst_set_cmd_error(cmd,
2289                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2290                 }
2291         }
2292
2293         TRACE_EXIT_RES(res);
2294         return res;
2295 }
2296
2297 static int scst_pre_dev_done(struct scst_cmd *cmd)
2298 {
2299         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2300
2301         TRACE_ENTRY();
2302
2303         if (unlikely(scst_check_auto_sense(cmd))) {
2304                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2305                             "without sense data (opcode 0x%x), issuing "
2306                             "REQUEST SENSE", cmd->cdb[0]);
2307                 rc = scst_prepare_request_sense(cmd);
2308                 if (rc == 0)
2309                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2310                 else {
2311                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2312                                     "returning HARDWARE ERROR");
2313                         scst_set_cmd_error(cmd,
2314                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2315                 }
2316                 goto out;
2317         } else if (unlikely(scst_check_sense(cmd)))
2318                 goto out;
2319
2320         if (likely(scsi_status_is_good(cmd->status))) {
2321                 unsigned char type = cmd->dev->handler->type;
2322                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2323                               cmd->cdb[0] == MODE_SENSE_10)) &&
2324                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2325                     (type == TYPE_DISK ||
2326                      type == TYPE_WORM ||
2327                      type == TYPE_MOD ||
2328                      type == TYPE_TAPE)) {
2329                         int32_t length;
2330                         uint8_t *address;
2331                         bool err = false;
2332
2333                         length = scst_get_buf_first(cmd, &address);
2334                         if (length < 0) {
2335                                 PRINT_ERROR("%s", "Unable to get "
2336                                         "MODE_SENSE buffer");
2337                                 scst_set_cmd_error(cmd,
2338                                         SCST_LOAD_SENSE(
2339                                                 scst_sense_hardw_error));
2340                                 err = true;
2341                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2342                                 address[2] |= 0x80;   /* Write Protect*/
2343                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2344                                 address[3] |= 0x80;   /* Write Protect*/
2345                         scst_put_buf(cmd, address);
2346
2347                         if (err)
2348                                 goto out;
2349                 }
2350
2351                 /*
2352                  * Check and clear NormACA option for the device, if necessary,
2353                  * since we don't support ACA
2354                  */
2355                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2356                     /* Std INQUIRY data (no EVPD) */
2357                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2358                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2359                         uint8_t *buffer;
2360                         int buflen;
2361                         bool err = false;
2362
2363                         /* ToDo: all pages ?? */
2364                         buflen = scst_get_buf_first(cmd, &buffer);
2365                         if (buflen > SCST_INQ_BYTE3) {
2366 #ifdef CONFIG_SCST_EXTRACHECKS
2367                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2368                                         PRINT_INFO("NormACA set for device: "
2369                                             "lun=%lld, type 0x%02x. Clear it, "
2370                                             "since it's unsupported.",
2371                                             (long long unsigned int)cmd->lun,
2372                                             buffer[0]);
2373                                 }
2374 #endif
2375                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2376                         } else if (buflen != 0) {
2377                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2378                                     "buffer");
2379                                 scst_set_cmd_error(cmd,
2380                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2381                                 err = true;
2382                         }
2383                         if (buflen > 0)
2384                                 scst_put_buf(cmd, buffer);
2385
2386                         if (err)
2387                                 goto out;
2388                 }
2389
2390                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2391                     (cmd->cdb[0] == MODE_SELECT_10) ||
2392                     (cmd->cdb[0] == LOG_SELECT))) {
2393                         TRACE(TRACE_SCSI,
2394                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2395                                 (long long unsigned int)cmd->lun);
2396                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2397                         goto out;
2398                 }
2399         } else {
2400                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2401                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2402                                         &cmd->tgt_dev->tgt_dev_flags)) {
2403                                 struct scst_tgt_dev *tgt_dev_tmp;
2404                                 struct scst_device *dev = cmd->dev;
2405
2406                                 TRACE(TRACE_SCSI,
2407                                         "Real RESERVE failed lun=%lld, "
2408                                         "status=%x",
2409                                         (long long unsigned int)cmd->lun,
2410                                         cmd->status);
2411                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2412                                         SCST_SENSE_BUFFERSIZE);
2413
2414                                 /* Clearing the reservation */
2415                                 spin_lock_bh(&dev->dev_lock);
2416                                 list_for_each_entry(tgt_dev_tmp,
2417                                                     &dev->dev_tgt_dev_list,
2418                                                     dev_tgt_dev_list_entry) {
2419                                         clear_bit(SCST_TGT_DEV_RESERVED,
2420                                                 &tgt_dev_tmp->tgt_dev_flags);
2421                                 }
2422                                 dev->dev_reserved = 0;
2423                                 spin_unlock_bh(&dev->dev_lock);
2424                         }
2425                 }
2426
2427                 /* Check for MODE PARAMETERS CHANGED UA */
2428                 if ((cmd->dev->scsi_dev != NULL) &&
2429                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2430                     SCST_SENSE_VALID(cmd->sense) &&
2431                     scst_is_ua_sense(cmd->sense) &&
2432                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2433                         TRACE(TRACE_SCSI,
2434                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2435                               (long long unsigned int)cmd->lun);
2436                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2437                         goto out;
2438                 }
2439         }
2440
2441         cmd->state = SCST_CMD_STATE_DEV_DONE;
2442
2443 out:
2444         TRACE_EXIT_RES(res);
2445         return res;
2446 }
2447
2448 static int scst_mode_select_checks(struct scst_cmd *cmd)
2449 {
2450         int res = SCST_CMD_STATE_RES_CONT_SAME;
2451         int atomic = scst_cmd_atomic(cmd);
2452
2453         TRACE_ENTRY();
2454
2455         if (likely(scsi_status_is_good(cmd->status))) {
2456                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2457                     (cmd->cdb[0] == MODE_SELECT_10) ||
2458                     (cmd->cdb[0] == LOG_SELECT))) {
2459                         struct scst_device *dev = cmd->dev;
2460                         if (atomic && (dev->scsi_dev != NULL)) {
2461                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2462                                         "context required");
2463                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2464                                 goto out;
2465                         }
2466
2467                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2468                                 "setting the SELECT UA (lun=%lld)",
2469                                 (long long unsigned int)cmd->lun);
2470
2471                         spin_lock_bh(&dev->dev_lock);
2472                         spin_lock(&scst_temp_UA_lock);
2473                         if (cmd->cdb[0] == LOG_SELECT) {
2474                                 scst_set_sense(scst_temp_UA,
2475                                         sizeof(scst_temp_UA),
2476                                         UNIT_ATTENTION, 0x2a, 0x02);
2477                         } else {
2478                                 scst_set_sense(scst_temp_UA,
2479                                         sizeof(scst_temp_UA),
2480                                         UNIT_ATTENTION, 0x2a, 0x01);
2481                         }
2482                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2483                                 sizeof(scst_temp_UA));
2484                         spin_unlock(&scst_temp_UA_lock);
2485                         spin_unlock_bh(&dev->dev_lock);
2486
2487                         if (dev->scsi_dev != NULL)
2488                                 scst_obtain_device_parameters(dev);
2489                 }
2490         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2491                     SCST_SENSE_VALID(cmd->sense) &&
2492                     scst_is_ua_sense(cmd->sense) &&
2493                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2494                      (cmd->sense[12] == 0x29) /* reset */ ||
2495                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2496                      /* cleared by another ini (just in case) */
2497                      (cmd->sense[12] == 0x2F))) {
2498                 if (atomic) {
2499                         TRACE_DBG("Possible parameters changed UA %x: "
2500                                 "thread context required", cmd->sense[12]);
2501                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2502                         goto out;
2503                 }
2504
2505                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2506                         "(lun %lld): getting new parameters", cmd->sense[12],
2507                         (long long unsigned int)cmd->lun);
2508
2509                 scst_obtain_device_parameters(cmd->dev);
2510         } else
2511                 sBUG();
2512
2513         cmd->state = SCST_CMD_STATE_DEV_DONE;
2514
2515 out:
2516         TRACE_EXIT_HRES(res);
2517         return res;
2518 }
2519
2520 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2521 {
2522         if (likely(cmd->sn_set))
2523                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2524
2525         scst_make_deferred_commands_active(cmd->tgt_dev);
2526 }
2527
2528 static int scst_dev_done(struct scst_cmd **pcmd)
2529 {
2530         int res = SCST_CMD_STATE_RES_CONT_SAME;
2531         struct scst_cmd *cmd = *pcmd;
2532         int state;
2533         struct scst_device *dev = cmd->dev;
2534
2535         TRACE_ENTRY();
2536
2537         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2538
2539         if (likely(!scst_is_cmd_local(cmd)) &&
2540             likely(dev->handler->dev_done != NULL)) {
2541                 int rc;
2542
2543                 if (unlikely(!dev->handler->dev_done_atomic &&
2544                              scst_cmd_atomic(cmd))) {
2545                         /*
2546                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2547                          * optimization.
2548                          */
2549                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2550                               "context, rescheduling", dev->handler->name);
2551                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2552                         goto out;
2553                 }
2554
2555                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2556                       dev->handler->name, cmd);
2557                 rc = dev->handler->dev_done(cmd);
2558                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2559                       dev->handler->name, rc);
2560                 if (rc != SCST_CMD_STATE_DEFAULT)
2561                         state = rc;
2562         }
2563
2564         switch (state) {
2565         case SCST_CMD_STATE_PRE_XMIT_RESP:
2566         case SCST_CMD_STATE_DEV_PARSE:
2567         case SCST_CMD_STATE_PRE_PARSE:
2568         case SCST_CMD_STATE_PREPARE_SPACE:
2569         case SCST_CMD_STATE_RDY_TO_XFER:
2570         case SCST_CMD_STATE_TGT_PRE_EXEC:
2571         case SCST_CMD_STATE_SEND_FOR_EXEC:
2572         case SCST_CMD_STATE_LOCAL_EXEC:
2573         case SCST_CMD_STATE_REAL_EXEC:
2574         case SCST_CMD_STATE_PRE_DEV_DONE:
2575         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2576         case SCST_CMD_STATE_DEV_DONE:
2577         case SCST_CMD_STATE_XMIT_RESP:
2578         case SCST_CMD_STATE_FINISHED:
2579                 cmd->state = state;
2580                 break;
2581
2582         case SCST_CMD_STATE_NEED_THREAD_CTX:
2583                 TRACE_DBG("Dev handler %s dev_done() requested "
2584                       "thread context, rescheduling",
2585                       dev->handler->name);
2586                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2587                 break;
2588
2589         default:
2590                 if (state >= 0) {
2591                         PRINT_ERROR("Dev handler %s dev_done() returned "
2592                                 "invalid cmd state %d",
2593                                 dev->handler->name, state);
2594                 } else {
2595                         PRINT_ERROR("Dev handler %s dev_done() returned "
2596                                 "error %d", dev->handler->name,
2597                                 state);
2598                 }
2599                 scst_set_cmd_error(cmd,
2600                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2601                 scst_set_cmd_abnormal_done_state(cmd);
2602                 break;
2603         }
2604
2605         if (cmd->needs_unblocking)
2606                 scst_unblock_dev_cmd(cmd);
2607
2608         if (likely(cmd->dec_on_dev_needed))
2609                 scst_dec_on_dev_cmd(cmd);
2610
2611         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2612                 scst_inc_check_expected_sn(cmd);
2613
2614         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2615                 *pcmd = scst_complete_request_sense(cmd);
2616
2617 out:
2618         TRACE_EXIT_HRES(res);
2619         return res;
2620 }
2621
2622 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2623 {
2624         int res;
2625
2626         TRACE_ENTRY();
2627
2628 #ifdef CONFIG_SCST_DEBUG_TM
2629         if (cmd->tm_dbg_delayed &&
2630                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2631                 if (scst_cmd_atomic(cmd)) {
2632                         TRACE_MGMT_DBG("%s",
2633                                 "DEBUG_TM delayed cmd needs a thread");
2634                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2635                         return res;
2636                 }
2637                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2638                         cmd, cmd->tag);
2639                 schedule_timeout_uninterruptible(HZ);
2640         }
2641 #endif
2642
2643         if (likely(cmd->tgt_dev != NULL)) {
2644                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2645                 atomic_dec(&cmd->dev->dev_cmd_count);
2646                 /* If expected values not set, expected direction is UNKNOWN */
2647                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2648                         atomic_dec(&cmd->dev->write_cmd_count);
2649
2650                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2651                         scst_on_hq_cmd_response(cmd);
2652
2653                 if (unlikely(!cmd->sent_for_exec)) {
2654                         TRACE_SN("cmd %p was not sent to mid-lev"
2655                                 " (sn %ld, set %d)",
2656                                 cmd, cmd->sn, cmd->sn_set);
2657                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2658                         cmd->sent_for_exec = 1;
2659                 }
2660         }
2661
2662         /*
2663          * If we don't remove cmd from the search list here, before
2664          * submitting it for transmittion, we will have a race, when for
2665          * some reason cmd's release is delayed after transmittion and
2666          * initiator sends cmd with the same tag => it is possible that
2667          * a wrong cmd will be found by find() functions.
2668          */
2669         spin_lock_irq(&cmd->sess->sess_list_lock);
2670         list_del(&cmd->search_cmd_list_entry);
2671         spin_unlock_irq(&cmd->sess->sess_list_lock);
2672
2673         cmd->done = 1;
2674         smp_mb(); /* to sync with scst_abort_cmd() */
2675
2676         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2677                 scst_xmit_process_aborted_cmd(cmd);
2678
2679         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2680                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2681                                 " skipping",
2682                                 cmd, (long long unsigned int)cmd->tag);
2683                 cmd->state = SCST_CMD_STATE_FINISHED;
2684                 res = SCST_CMD_STATE_RES_CONT_SAME;
2685                 goto out;
2686         }
2687
2688         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2689         res = SCST_CMD_STATE_RES_CONT_SAME;
2690
2691 out:
2692 #ifdef CONFIG_SCST_MEASURE_LATENCY
2693         {
2694                 struct timespec ts;
2695                 uint64_t finish, scst_time, proc_time;
2696                 struct scst_session *sess = cmd->sess;
2697
2698                 getnstimeofday(&ts);
2699                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2700
2701                 spin_lock_bh(&sess->meas_lock);
2702
2703                 scst_time = cmd->pre_exec_finish - cmd->start;
2704                 scst_time += finish - cmd->post_exec_start;
2705                 proc_time = finish - cmd->start;
2706
2707                 sess->scst_time += scst_time;
2708                 sess->processing_time += proc_time;
2709                 sess->processed_cmds++;
2710
2711                 spin_unlock_bh(&sess->meas_lock);
2712
2713                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2714                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2715                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2716                         proc_time);
2717         }
2718 #endif
2719         TRACE_EXIT_HRES(res);
2720         return res;
2721 }
2722
2723 static int scst_xmit_response(struct scst_cmd *cmd)
2724 {
2725         int res, rc;
2726
2727         TRACE_ENTRY();
2728
2729         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2730                      scst_cmd_atomic(cmd))) {
2731                 /*
2732                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2733                  * optimization.
2734                  */
2735                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2736                               "context, rescheduling", cmd->tgtt->name);
2737                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2738                 goto out;
2739         }
2740
2741         while (1) {
2742                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2743
2744                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2745                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2746
2747                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2748
2749 #if defined(CONFIG_SCST_DEBUG)
2750                 if (cmd->sg) {
2751                         int i;
2752                         struct scatterlist *sg = cmd->sg;
2753                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at "
2754                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
2755                         for (i = 0; i < cmd->sg_cnt; ++i) {
2756                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2757                                     "Xmitting sg", sg_virt(&sg[i]),
2758                                     sg[i].length);
2759                         }
2760                 }
2761 #endif
2762
2763 #ifdef CONFIG_SCST_DEBUG_RETRY
2764                 if (((scst_random() % 100) == 77))
2765                         rc = SCST_TGT_RES_QUEUE_FULL;
2766                 else
2767 #endif
2768                         rc = cmd->tgtt->xmit_response(cmd);
2769                 TRACE_DBG("xmit_response() returned %d", rc);
2770
2771                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2772                         goto out;
2773
2774                 /* Restore the previous state */
2775                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2776
2777                 switch (rc) {
2778                 case SCST_TGT_RES_QUEUE_FULL:
2779                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2780                                 break;
2781                         else
2782                                 continue;
2783
2784                 case SCST_TGT_RES_NEED_THREAD_CTX:
2785                         TRACE_DBG("Target driver %s xmit_response() "
2786                               "requested thread context, rescheduling",
2787                               cmd->tgtt->name);
2788                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2789                         break;
2790
2791                 default:
2792                         goto out_error;
2793                 }
2794                 break;
2795         }
2796
2797 out:
2798         /* Caution: cmd can be already dead here */
2799         TRACE_EXIT_HRES(res);
2800         return res;
2801
2802 out_error:
2803         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2804                 PRINT_ERROR("Target driver %s xmit_response() returned "
2805                         "fatal error", cmd->tgtt->name);
2806         } else {
2807                 PRINT_ERROR("Target driver %s xmit_response() returned "
2808                         "invalid value %d", cmd->tgtt->name, rc);
2809         }
2810         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2811         cmd->state = SCST_CMD_STATE_FINISHED;
2812         res = SCST_CMD_STATE_RES_CONT_SAME;
2813         goto out;
2814 }
2815
2816 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2817         enum scst_exec_context pref_context)
2818 {
2819         TRACE_ENTRY();
2820
2821         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2822
2823         cmd->state = SCST_CMD_STATE_FINISHED;
2824         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2825
2826         TRACE_EXIT();
2827         return;
2828 }
2829 EXPORT_SYMBOL(scst_tgt_cmd_done);
2830
2831 static int scst_finish_cmd(struct scst_cmd *cmd)
2832 {
2833         int res;
2834
2835         TRACE_ENTRY();
2836
2837         atomic_dec(&cmd->sess->sess_cmd_count);
2838
2839         cmd->finished = 1;
2840         smp_mb(); /* to sync with scst_abort_cmd() */
2841
2842         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2843                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2844                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2845                         atomic_read(&scst_cmd_count));
2846
2847                 scst_finish_cmd_mgmt(cmd);
2848         }
2849
2850         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2851                 if ((cmd->tgt_dev != NULL) &&
2852                     scst_is_ua_sense(cmd->sense)) {
2853                         /* This UA delivery failed, so requeue it */
2854                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2855                                 "%p", cmd);
2856                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2857                                         SCST_SENSE_BUFFERSIZE, 1);
2858                 }
2859         }
2860
2861         __scst_cmd_put(cmd);
2862
2863         res = SCST_CMD_STATE_RES_CONT_NEXT;
2864
2865         TRACE_EXIT_HRES(res);
2866         return res;
2867 }
2868
2869 /*
2870  * No locks, but it must be externally serialized (see comment for
2871  * scst_cmd_init_done() in scst.h)
2872  */
2873 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2874 {
2875         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2876         unsigned long flags;
2877
2878         TRACE_ENTRY();
2879
2880         if (scst_is_implicit_hq(cmd)) {
2881                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING,
2882                         "Implicit HQ cmd %p", cmd);
2883                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2884         }
2885
2886         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2887
2888         /* Optimized for lockless fast path */
2889
2890         scst_check_debug_sn(cmd);
2891
2892         if (cmd->dev->queue_alg ==
2893                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2894                 /*
2895                  * Not the best way, but well enough until there will be a
2896                  * possibility to specify queue type during pass-through
2897                  * commands submission.
2898                  */
2899                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2900         }
2901
2902         switch (cmd->queue_type) {
2903         case SCST_CMD_QUEUE_SIMPLE:
2904         case SCST_CMD_QUEUE_UNTAGGED:
2905 #if 1 /* temporary, ToDo */
2906                 if (scst_cmd_is_expected_set(cmd)) {
2907                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2908                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2909                                 goto ordered;
2910                 } else
2911                         goto ordered;
2912 #endif
2913                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2914                         /*
2915                          * atomic_inc_return() implies memory barrier to sync
2916                          * with scst_inc_expected_sn()
2917                          */
2918                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2919                                 tgt_dev->curr_sn++;
2920                                 TRACE_SN("Incremented curr_sn %ld",
2921                                         tgt_dev->curr_sn);
2922                         }
2923                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2924                         cmd->sn = tgt_dev->curr_sn;
2925
2926                         tgt_dev->prev_cmd_ordered = 0;
2927                 } else {
2928                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2929                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2930                         goto ordered;
2931                 }
2932                 break;
2933
2934         case SCST_CMD_QUEUE_ORDERED:
2935                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2936                         "(op %x)", cmd, cmd->cdb[0]);
2937 ordered:
2938                 if (!tgt_dev->prev_cmd_ordered) {
2939                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2940                         if (tgt_dev->num_free_sn_slots >= 0) {
2941                                 tgt_dev->num_free_sn_slots--;
2942                                 if (tgt_dev->num_free_sn_slots >= 0) {
2943                                         int i = 0;
2944                                         /* Commands can finish in any order, so
2945                                          * we don't know which slot is empty.
2946                                          */
2947                                         while (1) {
2948                                                 tgt_dev->cur_sn_slot++;
2949                                                 if (tgt_dev->cur_sn_slot ==
2950                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
2951                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2952
2953                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2954                                                         break;
2955
2956                                                 i++;
2957                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2958                                         }
2959                                         TRACE_SN("New cur SN slot %zd",
2960                                                 tgt_dev->cur_sn_slot -
2961                                                 tgt_dev->sn_slots);
2962                                 }
2963                         }
2964                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2965                 }
2966                 tgt_dev->prev_cmd_ordered = 1;
2967                 tgt_dev->curr_sn++;
2968                 cmd->sn = tgt_dev->curr_sn;
2969                 break;
2970
2971         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2972                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2973                         "(op %x)", cmd, cmd->cdb[0]);
2974                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2975                 tgt_dev->hq_cmd_count++;
2976                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2977                 cmd->hq_cmd_inced = 1;
2978                 goto out;
2979
2980         default:
2981                 sBUG();
2982         }
2983
2984         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2985                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2986                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2987                 atomic_read(tgt_dev->cur_sn_slot),
2988                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2989                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2990
2991         cmd->sn_set = 1;
2992
2993 out:
2994         TRACE_EXIT();
2995         return;
2996 }
2997
2998 /*
2999  * Returns 0 on success, > 0 when we need to wait for unblock,
3000  * < 0 if there is no device (lun) or device type handler.
3001  *
3002  * No locks, but might be on IRQ, protection is done by the
3003  * suspended activity.
3004  */
3005 static int scst_translate_lun(struct scst_cmd *cmd)
3006 {
3007         struct scst_tgt_dev *tgt_dev = NULL;
3008         int res;
3009
3010         TRACE_ENTRY();
3011
3012         __scst_get(1);
3013
3014         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3015                 struct list_head *sess_tgt_dev_list_head =
3016                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3017                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3018                         (long long unsigned int)cmd->lun);
3019                 res = -1;
3020                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3021                                 sess_tgt_dev_list_entry) {
3022                         if (tgt_dev->lun == cmd->lun) {
3023                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3024
3025                                 if (unlikely(tgt_dev->dev->handler ==
3026                                                 &scst_null_devtype)) {
3027                                         PRINT_INFO("Dev handler for device "
3028                                           "%lld is NULL, the device will not "
3029                                           "be visible remotely",
3030                                            (long long unsigned int)cmd->lun);
3031                                         break;
3032                                 }
3033
3034                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3035                                 cmd->tgt_dev = tgt_dev;
3036                                 cmd->dev = tgt_dev->dev;
3037
3038                                 res = 0;
3039                                 break;
3040                         }
3041                 }
3042                 if (res != 0) {
3043                         TRACE(TRACE_MINOR,
3044                                 "tgt_dev for lun %lld not found, command to "
3045                                 "unexisting LU?",
3046                                 (long long unsigned int)cmd->lun);
3047                         __scst_put();
3048                 }
3049         } else {
3050                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3051                 __scst_put();
3052                 res = 1;
3053         }
3054
3055         TRACE_EXIT_RES(res);
3056         return res;
3057 }
3058
3059 /*
3060  * No locks, but might be on IRQ
3061  *
3062  * Returns 0 on success, > 0 when we need to wait for unblock,
3063  * < 0 if there is no device (lun) or device type handler.
3064  */
3065 static int __scst_init_cmd(struct scst_cmd *cmd)
3066 {
3067         int res = 0;
3068
3069         TRACE_ENTRY();
3070
3071         res = scst_translate_lun(cmd);
3072         if (likely(res == 0)) {
3073                 int cnt;
3074                 bool failure = false;
3075
3076                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3077
3078                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3079                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3080                         TRACE(TRACE_MGMT_MINOR,
3081                                 "Too many pending commands (%d) in "
3082                                 "session, returning BUSY to initiator \"%s\"",
3083                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3084                                   "Anonymous" : cmd->sess->initiator_name);
3085                         failure = true;
3086                 }
3087
3088                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3089                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3090                         if (!failure) {
3091                                 TRACE(TRACE_MGMT_MINOR,
3092                                         "Too many pending device "
3093                                         "commands (%d), returning BUSY to "
3094                                         "initiator \"%s\"", cnt,
3095                                         (cmd->sess->initiator_name[0] == '\0') ?
3096                                                 "Anonymous" :
3097                                                 cmd->sess->initiator_name);
3098                                 failure = true;
3099                         }
3100                 }
3101
3102                 /* If expected values not set, expected direction is UNKNOWN */
3103                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3104                         atomic_inc(&cmd->dev->write_cmd_count);
3105
3106                 if (unlikely(failure))
3107                         goto out_busy;
3108
3109                 if (!cmd->set_sn_on_restart_cmd)
3110                         scst_cmd_set_sn(cmd);
3111         } else if (res < 0) {
3112                 TRACE_DBG("Finishing cmd %p", cmd);
3113                 scst_set_cmd_error(cmd,
3114                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3115                 scst_set_cmd_abnormal_done_state(cmd);
3116         } else
3117                 goto out;
3118
3119 out:
3120         TRACE_EXIT_RES(res);
3121         return res;
3122
3123 out_busy:
3124         scst_set_busy(cmd);
3125         scst_set_cmd_abnormal_done_state(cmd);
3126         goto out;
3127 }
3128
3129 /* Called under scst_init_lock and IRQs disabled */
3130 static void scst_do_job_init(void)
3131 {
3132         struct scst_cmd *cmd;
3133         int susp;
3134
3135         TRACE_ENTRY();
3136
3137 restart:
3138         /*
3139          * There is no need for read barrier here, because we don't care where
3140          * this check will be done.
3141          */
3142         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3143         if (scst_init_poll_cnt > 0)
3144                 scst_init_poll_cnt--;
3145
3146         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3147                 int rc;
3148                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3149                         continue;
3150                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3151                         spin_unlock_irq(&scst_init_lock);
3152                         rc = __scst_init_cmd(cmd);
3153                         spin_lock_irq(&scst_init_lock);
3154                         if (rc > 0) {
3155                                 TRACE_MGMT_DBG("%s",
3156                                         "FLAG SUSPENDED set, restarting");
3157                                 goto restart;
3158                         }
3159                 } else {
3160                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3161                                        cmd, (long long unsigned int)cmd->tag);
3162                         scst_set_cmd_abnormal_done_state(cmd);
3163                 }
3164
3165                 /*
3166                  * Deleting cmd from init cmd list after __scst_init_cmd()
3167                  * is necessary to keep the check in scst_init_cmd() correct
3168                  * to preserve the commands order.
3169                  *
3170                  * We don't care about the race, when init cmd list is empty
3171                  * and one command detected that it just was not empty, so
3172                  * it's inserting to it, but another command at the same time
3173                  * seeing init cmd list empty and goes directly, because it
3174                  * could affect only commands from the same initiator to the
3175                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3176                  * in case of simultaneous such calls anyway.
3177                  */
3178                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3179                 smp_wmb();
3180                 list_del(&cmd->cmd_list_entry);
3181                 spin_unlock(&scst_init_lock);
3182
3183                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3184                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3185                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3186                         list_add(&cmd->cmd_list_entry,
3187                                 &cmd->cmd_lists->active_cmd_list);
3188                 else
3189                         list_add_tail(&cmd->cmd_list_entry,
3190                                 &cmd->cmd_lists->active_cmd_list);
3191                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3192                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3193
3194                 spin_lock(&scst_init_lock);
3195                 goto restart;
3196         }
3197
3198         /* It isn't really needed, but let's keep it */
3199         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3200                 goto restart;
3201
3202         TRACE_EXIT();
3203         return;
3204 }
3205
3206 static inline int test_init_cmd_list(void)
3207 {
3208         int res = (!list_empty(&scst_init_cmd_list) &&
3209                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3210                   unlikely(kthread_should_stop()) ||
3211                   (scst_init_poll_cnt > 0);
3212         return res;
3213 }
3214
3215 int scst_init_cmd_thread(void *arg)
3216 {
3217         TRACE_ENTRY();
3218
3219         PRINT_INFO("Init thread started, PID %d", current->pid);
3220
3221         current->flags |= PF_NOFREEZE;
3222
3223         set_user_nice(current, -10);
3224
3225         spin_lock_irq(&scst_init_lock);
3226         while (!kthread_should_stop()) {
3227                 wait_queue_t wait;
3228                 init_waitqueue_entry(&wait, current);
3229
3230                 if (!test_init_cmd_list()) {
3231                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3232                                                  &wait);
3233                         for (;;) {
3234                                 set_current_state(TASK_INTERRUPTIBLE);
3235                                 if (test_init_cmd_list())
3236                                         break;
3237                                 spin_unlock_irq(&scst_init_lock);
3238                                 schedule();
3239                                 spin_lock_irq(&scst_init_lock);
3240                         }
3241                         set_current_state(TASK_RUNNING);
3242                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3243                 }
3244                 scst_do_job_init();
3245         }
3246         spin_unlock_irq(&scst_init_lock);
3247
3248         /*
3249          * If kthread_should_stop() is true, we are guaranteed to be
3250          * on the module unload, so scst_init_cmd_list must be empty.
3251          */
3252         sBUG_ON(!list_empty(&scst_init_cmd_list));
3253
3254         PRINT_INFO("Init thread PID %d finished", current->pid);
3255
3256         TRACE_EXIT();
3257         return 0;
3258 }
3259
3260 /* Called with no locks held */
3261 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3262 {
3263         int res;
3264
3265         TRACE_ENTRY();
3266
3267         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3268
3269         cmd->atomic = atomic;
3270
3271         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3272
3273         do {
3274                 switch (cmd->state) {
3275                 case SCST_CMD_STATE_PRE_PARSE:
3276                         res = scst_pre_parse(cmd);
3277                         EXTRACHECKS_BUG_ON(res ==
3278                                 SCST_CMD_STATE_RES_NEED_THREAD);
3279                         break;
3280
3281                 case SCST_CMD_STATE_DEV_PARSE:
3282                         res = scst_parse_cmd(cmd);
3283                         break;
3284
3285                 case SCST_CMD_STATE_PREPARE_SPACE:
3286                         res = scst_prepare_space(cmd);
3287                         break;
3288
3289                 case SCST_CMD_STATE_RDY_TO_XFER:
3290                         res = scst_rdy_to_xfer(cmd);
3291                         break;
3292
3293                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3294                         res = scst_tgt_pre_exec(cmd);
3295                         break;
3296
3297                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3298                         if (tm_dbg_check_cmd(cmd) != 0) {
3299                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3300                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3301                                         "because of TM DBG delay", cmd,
3302                                         (long long unsigned int)cmd->tag);
3303                                 break;
3304                         }
3305                         res = scst_send_for_exec(&cmd);
3306                         /*
3307                          * !! At this point cmd, sess & tgt_dev can already be
3308                          * freed !!
3309                          */
3310                         break;
3311
3312                 case SCST_CMD_STATE_LOCAL_EXEC:
3313                         res = scst_local_exec(cmd);
3314                         /*
3315                          * !! At this point cmd, sess & tgt_dev can already be
3316                          * freed !!
3317                          */
3318                         break;
3319
3320                 case SCST_CMD_STATE_REAL_EXEC:
3321                         res = scst_real_exec(cmd);
3322                         /*
3323                          * !! At this point cmd, sess & tgt_dev can already be
3324                          * freed !!
3325                          */
3326                         break;
3327
3328                 case SCST_CMD_STATE_PRE_DEV_DONE:
3329                         res = scst_pre_dev_done(cmd);
3330                         EXTRACHECKS_BUG_ON(res ==
3331                                 SCST_CMD_STATE_RES_NEED_THREAD);
3332                         break;
3333
3334                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3335                         res = scst_mode_select_checks(cmd);
3336                         break;
3337
3338                 case SCST_CMD_STATE_DEV_DONE:
3339                         res = scst_dev_done(&cmd);
3340                         break;
3341
3342                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3343                         res = scst_pre_xmit_response(cmd);
3344                         EXTRACHECKS_BUG_ON(res ==
3345                                 SCST_CMD_STATE_RES_NEED_THREAD);
3346                         break;
3347
3348                 case SCST_CMD_STATE_XMIT_RESP:
3349                         res = scst_xmit_response(cmd);
3350                         break;
3351
3352                 case SCST_CMD_STATE_FINISHED:
3353                         res = scst_finish_cmd(cmd);
3354                         EXTRACHECKS_BUG_ON(res ==
3355                                 SCST_CMD_STATE_RES_NEED_THREAD);
3356                         break;
3357
3358                 default:
3359                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3360                                 "be", cmd, cmd->state);
3361                         sBUG();
3362                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3363                         break;
3364                 }
3365         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3366
3367         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3368                 /* None */
3369         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3370                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3371                 switch (cmd->state) {
3372                 case SCST_CMD_STATE_PRE_PARSE:
3373                 case SCST_CMD_STATE_DEV_PARSE:
3374                 case SCST_CMD_STATE_PREPARE_SPACE:
3375                 case SCST_CMD_STATE_RDY_TO_XFER:
3376                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3377                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3378                 case SCST_CMD_STATE_LOCAL_EXEC:
3379                 case SCST_CMD_STATE_REAL_EXEC:
3380                 case SCST_CMD_STATE_PRE_DEV_DONE:
3381                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3382                 case SCST_CMD_STATE_DEV_DONE:
3383                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3384                 case SCST_CMD_STATE_XMIT_RESP:
3385                 case SCST_CMD_STATE_FINISHED:
3386                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3387                                   cmd);
3388                         list_add(&cmd->cmd_list_entry,
3389                                 &cmd->cmd_lists->active_cmd_list);
3390                         break;
3391 #ifdef CONFIG_SCST_EXTRACHECKS
3392                 /* not very valid commands */
3393                 case SCST_CMD_STATE_DEFAULT:
3394                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3395                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3396                                 cmd->state);
3397                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3398                         sBUG();
3399                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3400                         break;
3401 #endif
3402                 default:
3403                         break;
3404                 }
3405                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3406                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3407         } else
3408                 sBUG();
3409
3410         TRACE_EXIT();
3411         return;
3412 }
3413 EXPORT_SYMBOL(scst_process_active_cmd);
3414
3415 /* Called under cmd_list_lock and IRQs disabled */
3416 static void scst_do_job_active(struct list_head *cmd_list,
3417         spinlock_t *cmd_list_lock, bool atomic)
3418 {
3419         TRACE_ENTRY();
3420
3421         while (!list_empty(cmd_list)) {
3422                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3423                                         cmd_list_entry);
3424                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3425                 list_del(&cmd->cmd_list_entry);
3426                 spin_unlock_irq(cmd_list_lock);
3427                 scst_process_active_cmd(cmd, atomic);
3428                 spin_lock_irq(cmd_list_lock);
3429         }
3430
3431         TRACE_EXIT();
3432         return;
3433 }
3434
3435 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3436 {
3437         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3438             unlikely(kthread_should_stop()) ||
3439             tm_dbg_is_release();
3440         return res;
3441 }
3442
3443 int scst_cmd_thread(void *arg)
3444 {
3445         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3446
3447         TRACE_ENTRY();
3448
3449         PRINT_INFO("Processing thread started, PID %d", current->pid);
3450
3451 #if 0
3452         set_user_nice(current, 10);
3453 #endif
3454         current->flags |= PF_NOFREEZE;
3455
3456         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3457         while (!kthread_should_stop()) {
3458                 wait_queue_t wait;
3459                 init_waitqueue_entry(&wait, current);
3460
3461                 if (!test_cmd_lists(p_cmd_lists)) {
3462                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3463                                 &wait);
3464                         for (;;) {
3465                                 set_current_state(TASK_INTERRUPTIBLE);
3466                                 if (test_cmd_lists(p_cmd_lists))
3467                                         break;
3468                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3469                                 schedule();
3470                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3471                         }
3472                         set_current_state(TASK_RUNNING);
3473                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3474                 }
3475
3476                 if (tm_dbg_is_release()) {
3477                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3478                         tm_dbg_check_released_cmds();
3479                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3480                 }
3481
3482                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3483                         &p_cmd_lists->cmd_list_lock, false);
3484         }
3485         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3486
3487 #ifdef CONFIG_SCST_EXTRACHECKS
3488         /*
3489          * If kthread_should_stop() is true, we are guaranteed to be either
3490          * on the module unload, or there must be at least one other thread to
3491          * process the commands lists.
3492          */
3493         if (p_cmd_lists == &scst_main_cmd_lists) {
3494                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3495                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3496         }
3497 #