Added possibility to coexist if both target driver and dev handler need custom memory...
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38
39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
40 {
41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
42         unsigned long flags;
43
44         spin_lock_irqsave(&t->tasklet_lock, flags);
45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
46                 smp_processor_id());
47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
49
50         tasklet_schedule(&t->tasklet);
51 }
52
53 /*
54  * Must not be called in parallel with scst_unregister_session_ex() for the
55  * same sess
56  */
57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
58                              const uint8_t *lun, int lun_len,
59                              const uint8_t *cdb, int cdb_len, int atomic)
60 {
61         struct scst_cmd *cmd;
62
63         TRACE_ENTRY();
64
65 #ifdef CONFIG_SCST_EXTRACHECKS
66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
67                 PRINT_CRIT_ERROR("%s",
68                         "New cmd while shutting down the session");
69                 sBUG();
70         }
71 #endif
72
73         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
74         if (cmd == NULL)
75                 goto out;
76
77         cmd->sess = sess;
78         cmd->tgt = sess->tgt;
79         cmd->tgtt = sess->tgt->tgtt;
80
81         /*
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100 EXPORT_SYMBOL(scst_rx_cmd);
101
102 /*
103  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
104  * this command should be stopped.
105  */
106 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
107 {
108         int rc, res = 0;
109
110         TRACE_ENTRY();
111
112         /* See the comment in scst_do_job_init() */
113         if (unlikely(!list_empty(&scst_init_cmd_list))) {
114                 TRACE_MGMT_DBG("%s", "init cmd list busy");
115                 goto out_redirect;
116         }
117         /*
118          * Memory barrier isn't necessary here, because CPU appears to
119          * be self-consistent
120          */
121
122         rc = __scst_init_cmd(cmd);
123         if (unlikely(rc > 0))
124                 goto out_redirect;
125         else if (unlikely(rc != 0))
126                 goto out;
127
128         /* Small context optimization */
129         if (((*context == SCST_CONTEXT_TASKLET) ||
130              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
131              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
132               scst_cmd_is_expected_set(cmd)) {
133                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
134                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
135                                         &cmd->tgt_dev->tgt_dev_flags))
136                                 *context = SCST_CONTEXT_THREAD;
137                 } else {
138                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
139                                         &cmd->tgt_dev->tgt_dev_flags))
140                                 *context = SCST_CONTEXT_THREAD;
141                 }
142         }
143
144 out:
145         TRACE_EXIT_RES(res);
146         return res;
147
148 out_redirect:
149         if (cmd->preprocessing_only) {
150                 /*
151                  * Poor man solution for single threaded targets, where
152                  * blocking receiver at least sometimes means blocking all.
153                  */
154                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
155                 scst_set_busy(cmd);
156                 scst_set_cmd_abnormal_done_state(cmd);
157                 /* Keep initiator away from too many BUSY commands */
158                 msleep(50);
159         } else {
160                 unsigned long flags;
161                 spin_lock_irqsave(&scst_init_lock, flags);
162                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
163                         "%d)", cmd, atomic_read(&scst_cmd_count));
164                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
165                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
166                         scst_init_poll_cnt++;
167                 spin_unlock_irqrestore(&scst_init_lock, flags);
168                 wake_up(&scst_init_cmd_list_waitQ);
169                 res = -1;
170         }
171         goto out;
172 }
173
174 #ifdef CONFIG_SCST_MEASURE_LATENCY
175 static inline uint64_t scst_sec_to_nsec(time_t sec)
176 {
177         return (uint64_t)sec * 1000000000;
178 }
179 #endif
180
181 void scst_cmd_init_done(struct scst_cmd *cmd,
182         enum scst_exec_context pref_context)
183 {
184         unsigned long flags;
185         struct scst_session *sess = cmd->sess;
186         int rc;
187
188         TRACE_ENTRY();
189
190 #ifdef CONFIG_SCST_MEASURE_LATENCY
191         {
192                 struct timespec ts;
193                 getnstimeofday(&ts);
194                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
195                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
196                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
197                         ts.tv_nsec);
198         }
199 #endif
200
201         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
202         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
203                 "(cmd %p)", (long long unsigned int)cmd->tag,
204                 (long long unsigned int)cmd->lun, cmd->cdb_len,
205                 cmd->queue_type, cmd);
206         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
207                 cmd->cdb, cmd->cdb_len);
208
209 #ifdef CONFIG_SCST_EXTRACHECKS
210         if (unlikely((in_irq() || irqs_disabled())) &&
211             ((pref_context == SCST_CONTEXT_DIRECT) ||
212              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
213                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
214                         "SCST_CONTEXT_THREAD instead\n", pref_context,
215                         cmd->tgtt->name);
216                 pref_context = SCST_CONTEXT_THREAD;
217         }
218 #endif
219
220         atomic_inc(&sess->sess_cmd_count);
221
222         spin_lock_irqsave(&sess->sess_list_lock, flags);
223
224         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
225                 /*
226                  * We have to always keep command in the search list from the
227                  * very beginning, because otherwise it can be missed during
228                  * TM processing. This check is needed because there might be
229                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
230                  */
231                 if (cmd->search_cmd_list_entry.next == NULL)
232                         list_add_tail(&cmd->search_cmd_list_entry,
233                                 &sess->search_cmd_list);
234                 switch (sess->init_phase) {
235                 case SCST_SESS_IPH_SUCCESS:
236                         break;
237                 case SCST_SESS_IPH_INITING:
238                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
239                                   cmd);
240                         list_add_tail(&cmd->cmd_list_entry,
241                                 &sess->init_deferred_cmd_list);
242                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
243                         goto out;
244                 case SCST_SESS_IPH_FAILED:
245                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
246                         scst_set_busy(cmd);
247                         scst_set_cmd_abnormal_done_state(cmd);
248                         goto active;
249                 default:
250                         sBUG();
251                 }
252         } else
253                 list_add_tail(&cmd->search_cmd_list_entry,
254                               &sess->search_cmd_list);
255
256         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
257
258         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
259                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
260                 scst_set_cmd_error(cmd,
261                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
262                 scst_set_cmd_abnormal_done_state(cmd);
263                 goto active;
264         }
265
266         if (unlikely(cmd->cdb_len == 0)) {
267                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
268                 scst_set_cmd_error(cmd,
269                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
270                 scst_set_cmd_abnormal_done_state(cmd);
271                 goto active;
272         }
273
274         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
275                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
276                 scst_set_cmd_error(cmd,
277                         SCST_LOAD_SENSE(scst_sense_invalid_message));
278                 scst_set_cmd_abnormal_done_state(cmd);
279                 goto active;
280         }
281
282         cmd->state = SCST_CMD_STATE_INIT;
283         /* cmd must be inited here to preserve the order */
284         rc = scst_init_cmd(cmd, &pref_context);
285         if (unlikely(rc < 0))
286                 goto out;
287
288 active:
289         /* Here cmd must not be in any cmd list, no locks */
290         switch (pref_context) {
291         case SCST_CONTEXT_TASKLET:
292                 scst_schedule_tasklet(cmd);
293                 break;
294
295         case SCST_CONTEXT_DIRECT:
296                 scst_process_active_cmd(cmd, false);
297                 /* For *NEED_THREAD wake_up() is already done */
298                 break;
299
300         case SCST_CONTEXT_DIRECT_ATOMIC:
301                 scst_process_active_cmd(cmd, true);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         default:
306                 PRINT_ERROR("Context %x is undefined, using the thread one",
307                         pref_context);
308                 /* go through */
309         case SCST_CONTEXT_THREAD:
310                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
311                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
312                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
313                         list_add(&cmd->cmd_list_entry,
314                                 &cmd->cmd_lists->active_cmd_list);
315                 else
316                         list_add_tail(&cmd->cmd_list_entry,
317                                 &cmd->cmd_lists->active_cmd_list);
318                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
319                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
320                 break;
321         }
322
323 out:
324         TRACE_EXIT();
325         return;
326 }
327 EXPORT_SYMBOL(scst_cmd_init_done);
328
329 static int scst_pre_parse(struct scst_cmd *cmd)
330 {
331         int res = SCST_CMD_STATE_RES_CONT_SAME;
332         struct scst_device *dev = cmd->dev;
333         int rc;
334
335         TRACE_ENTRY();
336
337         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
338              (!dev->has_own_order_mgmt &&
339               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
340                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
341
342         /*
343          * Expected transfer data supplied by the SCSI transport via the
344          * target driver are untrusted, so we prefer to fetch them from CDB.
345          * Additionally, not all transports support supplying the expected
346          * transfer data.
347          */
348
349         rc = scst_get_cdb_info(cmd);
350         if (unlikely(rc != 0)) {
351                 if (rc > 0) {
352                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
353                         goto out_xmit;
354                 }
355                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
356                         "Should you update scst_scsi_op_table?",
357                         cmd->cdb[0], dev->handler->name);
358                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
359 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
360                 if (scst_cmd_is_expected_set(cmd)) {
361                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
362                                 "direction %d, transfer_len %d",
363                                 cmd->expected_data_direction,
364                                 cmd->expected_transfer_len);
365                         cmd->data_direction = cmd->expected_data_direction;
366
367                         cmd->bufflen = cmd->expected_transfer_len;
368                         /* Restore (likely) lost CDB length */
369                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
370                         if (cmd->cdb_len == -1) {
371                                 PRINT_ERROR("Unable to get CDB length for "
372                                         "opcode 0x%02x. Returning INVALID "
373                                         "OPCODE", cmd->cdb[0]);
374                                 scst_set_cmd_error(cmd,
375                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
376                                 goto out_xmit;
377                         }
378                 } else {
379                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
380                              "target %s not supplied expected values",
381                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
382                         scst_set_cmd_error(cmd,
383                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
384                         goto out_xmit;
385                 }
386 #else
387                 scst_set_cmd_error(cmd,
388                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                 goto out_xmit;
390 #endif
391         } else {
392                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
393                         "(expected %d, set %s), transfer_len=%d (expected "
394                         "len %d), flags=%d", cmd->op_name, cmd,
395                         cmd->data_direction, cmd->expected_data_direction,
396                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
397                         cmd->bufflen, cmd->expected_transfer_len,
398                         cmd->op_flags);
399
400                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
401                         if (scst_cmd_is_expected_set(cmd)) {
402                                 /*
403                                  * Command data length can't be easily
404                                  * determined from the CDB. ToDo, all such
405                                  * commands processing should be fixed. Until
406                                  * it's done, get the length from the supplied
407                                  * expected value, but limit it to some
408                                  * reasonable value (15MB).
409                                  */
410                                 cmd->bufflen = min(cmd->expected_transfer_len,
411                                                         15*1024*1024);
412                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
413                         } else
414                                 cmd->bufflen = 0;
415                 }
416         }
417
418         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
419                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
420                             "(opcode 0x%02x)", cmd->cdb[0]);
421                 scst_set_cmd_error(cmd,
422                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
423                 goto out_xmit;
424         }
425
426         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
427                 PRINT_ERROR("Linked commands are not supported "
428                             "(opcode 0x%02x)", cmd->cdb[0]);
429                 scst_set_cmd_error(cmd,
430                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
431                 goto out_xmit;
432         }
433
434         cmd->state = SCST_CMD_STATE_DEV_PARSE;
435
436 out:
437         TRACE_EXIT_RES(res);
438         return res;
439
440 out_xmit:
441         scst_set_cmd_abnormal_done_state(cmd);
442         res = SCST_CMD_STATE_RES_CONT_SAME;
443         goto out;
444 }
445
446 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
447 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
448 {
449         bool res = false;
450
451         switch (cmd->cdb[0]) {
452         case TEST_UNIT_READY:
453                 /* Crazy VMware people sometimes do TUR with READ direction */
454                 res = true;
455                 break;
456         case VERIFY:
457         case VERIFY_6:
458         case VERIFY_12:
459         case VERIFY_16:
460                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
461                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
462                     (cmd->cdb[1] & BYTCHK) == 0)
463                         res = true;
464                 break;
465         }
466
467         return res;
468 }
469 #endif
470
471 static int scst_parse_cmd(struct scst_cmd *cmd)
472 {
473         int res = SCST_CMD_STATE_RES_CONT_SAME;
474         int state;
475         struct scst_device *dev = cmd->dev;
476         int orig_bufflen = cmd->bufflen;
477
478         TRACE_ENTRY();
479
480         if (likely(!scst_is_cmd_local(cmd))) {
481                 if (unlikely(!dev->handler->parse_atomic &&
482                              scst_cmd_atomic(cmd))) {
483                         /*
484                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
485                          * optimization.
486                          */
487                         TRACE_DBG("Dev handler %s parse() needs thread "
488                                 "context, rescheduling", dev->handler->name);
489                         res = SCST_CMD_STATE_RES_NEED_THREAD;
490                         goto out;
491                 }
492
493                 TRACE_DBG("Calling dev handler %s parse(%p)",
494                       dev->handler->name, cmd);
495                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
496                                 cmd->cdb, cmd->cdb_len);
497                 state = dev->handler->parse(cmd);
498                 /* Caution: cmd can be already dead here */
499                 TRACE_DBG("Dev handler %s parse() returned %d",
500                         dev->handler->name, state);
501
502                 switch (state) {
503                 case SCST_CMD_STATE_NEED_THREAD_CTX:
504                         TRACE_DBG("Dev handler %s parse() requested thread "
505                               "context, rescheduling", dev->handler->name);
506                         res = SCST_CMD_STATE_RES_NEED_THREAD;
507                         goto out;
508
509                 case SCST_CMD_STATE_STOP:
510                         TRACE_DBG("Dev handler %s parse() requested stop "
511                                 "processing", dev->handler->name);
512                         res = SCST_CMD_STATE_RES_CONT_NEXT;
513                         goto out;
514                 }
515
516                 if (state == SCST_CMD_STATE_DEFAULT)
517                         state = SCST_CMD_STATE_PREPARE_SPACE;
518         } else
519                 state = SCST_CMD_STATE_PREPARE_SPACE;
520
521         if (cmd->data_len == -1)
522                 cmd->data_len = cmd->bufflen;
523
524         if (cmd->dh_data_buf_alloced &&
525             unlikely((orig_bufflen > cmd->bufflen))) {
526                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
527                         "is less, than required (size %d)", cmd->bufflen,
528                         orig_bufflen);
529                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
530                 goto out_error;
531         }
532
533         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
534                 goto set_res;
535
536         if (unlikely((cmd->bufflen == 0) &&
537                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
538                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
539                         "(handler %s, target %s)", cmd->cdb[0],
540                         dev->handler->name, cmd->tgtt->name);
541                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
542                 goto out_error;
543         }
544
545 #ifdef CONFIG_SCST_EXTRACHECKS
546         if ((cmd->bufflen != 0) &&
547             ((cmd->data_direction == SCST_DATA_NONE) ||
548              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
549                 PRINT_ERROR("Dev handler %s parse() returned "
550                         "invalid cmd data_direction %d, bufflen %d, state %d "
551                         "or sg %p (opcode 0x%x)", dev->handler->name,
552                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
553                         cmd->cdb[0]);
554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
555                 goto out_error;
556         }
557 #endif
558
559         if (scst_cmd_is_expected_set(cmd)) {
560 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
561 #       ifdef CONFIG_SCST_EXTRACHECKS
562                 if ((cmd->data_direction != cmd->expected_data_direction) ||
563                     (cmd->bufflen != cmd->expected_transfer_len)) {
564                         PRINT_WARNING("Expected values don't match decoded "
565                                 "ones: data_direction %d, "
566                                 "expected_data_direction %d, "
567                                 "bufflen %d, expected_transfer_len %d",
568                                 cmd->data_direction,
569                                 cmd->expected_data_direction,
570                                 cmd->bufflen, cmd->expected_transfer_len);
571                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
572                 }
573 #       endif
574                 cmd->data_direction = cmd->expected_data_direction;
575                 cmd->bufflen = cmd->expected_transfer_len;
576 #else
577                 if (unlikely(cmd->data_direction !=
578                                 cmd->expected_data_direction)) {
579                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
580                              (cmd->bufflen != 0)) &&
581                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
582                                 PRINT_ERROR("Expected data direction %d for "
583                                         "opcode 0x%02x (handler %s, target %s) "
584                                         "doesn't match "
585                                         "decoded value %d",
586                                         cmd->expected_data_direction,
587                                         cmd->cdb[0], dev->handler->name,
588                                         cmd->tgtt->name, cmd->data_direction);
589                                 PRINT_BUFFER("Failed CDB",
590                                         cmd->cdb, cmd->cdb_len);
591                                 scst_set_cmd_error(cmd,
592                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
593                                 goto out_dev_done;
594                         }
595                 }
596                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
597                         static int repd;
598                         if (repd < 100) {
599                                 /*
600                                  * Intentionally unlocked. Few messages more
601                                  * or less don't matter.
602                                  */
603                                 repd++;
604                                 TRACE(TRACE_MINOR, "Warning: expected transfer "
605                                         "length %d for opcode 0x%02x (handler "
606                                         "%s, target %s) doesn't match decoded "
607                                         "value %d. Faulty initiator (e.g. "
608                                         "VMware is known to be such) or "
609                                         "scst_scsi_op_table should be updated?",
610                                         cmd->expected_transfer_len, cmd->cdb[0],
611                                         dev->handler->name, cmd->tgtt->name,
612                                         cmd->bufflen);
613                                 PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
614                                         cmd->cdb, cmd->cdb_len);
615                         }
616                 }
617 #endif
618         }
619
620         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
621                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
622                         "target %s", cmd->cdb[0], dev->handler->name,
623                         cmd->tgtt->name);
624                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
625                 goto out_error;
626         }
627
628 set_res:
629         switch (state) {
630         case SCST_CMD_STATE_PREPARE_SPACE:
631         case SCST_CMD_STATE_PRE_PARSE:
632         case SCST_CMD_STATE_DEV_PARSE:
633         case SCST_CMD_STATE_RDY_TO_XFER:
634         case SCST_CMD_STATE_TGT_PRE_EXEC:
635         case SCST_CMD_STATE_SEND_FOR_EXEC:
636         case SCST_CMD_STATE_LOCAL_EXEC:
637         case SCST_CMD_STATE_REAL_EXEC:
638         case SCST_CMD_STATE_PRE_DEV_DONE:
639         case SCST_CMD_STATE_DEV_DONE:
640         case SCST_CMD_STATE_PRE_XMIT_RESP:
641         case SCST_CMD_STATE_XMIT_RESP:
642         case SCST_CMD_STATE_FINISHED:
643                 cmd->state = state;
644                 res = SCST_CMD_STATE_RES_CONT_SAME;
645                 break;
646
647         default:
648                 if (state >= 0) {
649                         PRINT_ERROR("Dev handler %s parse() returned "
650                              "invalid cmd state %d (opcode %d)",
651                              dev->handler->name, state, cmd->cdb[0]);
652                 } else {
653                         PRINT_ERROR("Dev handler %s parse() returned "
654                                 "error %d (opcode %d)", dev->handler->name,
655                                 state, cmd->cdb[0]);
656                 }
657                 goto out_error;
658         }
659
660         if (cmd->resp_data_len == -1) {
661                 if (cmd->data_direction == SCST_DATA_READ)
662                         cmd->resp_data_len = cmd->bufflen;
663                 else
664                          cmd->resp_data_len = 0;
665         }
666
667 out:
668         TRACE_EXIT_HRES(res);
669         return res;
670
671 out_error:
672         /* dev_done() will be called as part of the regular cmd's finish */
673         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
674
675 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
676 out_dev_done:
677 #endif
678         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
679         res = SCST_CMD_STATE_RES_CONT_SAME;
680         goto out;
681 }
682
683 static int scst_prepare_space(struct scst_cmd *cmd)
684 {
685         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
686
687         TRACE_ENTRY();
688
689         if (cmd->data_direction == SCST_DATA_NONE)
690                 goto prep_done;
691
692         if (cmd->tgt_need_alloc_data_buf) {
693                 int orig_bufflen = cmd->bufflen;
694
695                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
696                         cmd);
697
698                 r = cmd->tgtt->alloc_data_buf(cmd);
699                 if (r > 0)
700                         goto alloc;
701                 else if (r == 0) {
702                         if (unlikely(cmd->bufflen == 0)) {
703                                 /* See comment in scst_alloc_space() */
704                                 if (cmd->sg == NULL)
705                                         goto alloc;
706                         }
707
708                         cmd->tgt_data_buf_alloced = 1;
709
710                         if (unlikely(orig_bufflen < cmd->bufflen)) {
711                                 PRINT_ERROR("Target driver allocated data "
712                                         "buffer (size %d), is less, than "
713                                         "required (size %d)", orig_bufflen,
714                                         cmd->bufflen);
715                                 goto out_error;
716                         }
717                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
718                 } else
719                         goto check;
720         }
721
722 alloc:
723         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
724                 r = scst_alloc_space(cmd);
725                 cmd->tgt_sg = cmd->sg;
726                 cmd->tgt_sg_cnt = cmd->sg_cnt;
727         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
728                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
729                 cmd->tgt_sg = cmd->sg;
730                 cmd->tgt_sg_cnt = cmd->sg_cnt;
731                 r = 0;
732         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
733                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
734                 cmd->sg = cmd->tgt_sg;
735                 cmd->sg_cnt = cmd->tgt_sg_cnt;
736                 r = 0;
737         } else {
738                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
739                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
740                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
741                 r = 0;
742         }
743
744 check:
745         if (r != 0) {
746                 if (scst_cmd_atomic(cmd)) {
747                         TRACE_MEM("%s", "Atomic memory allocation failed, "
748                               "rescheduling to the thread");
749                         res = SCST_CMD_STATE_RES_NEED_THREAD;
750                         goto out;
751                 } else
752                         goto out_no_space;
753         }
754
755 prep_done:
756         if (cmd->preprocessing_only) {
757                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
758                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
759                                 "cmd %p", cmd);
760                         scst_set_cmd_abnormal_done_state(cmd);
761                         res = SCST_CMD_STATE_RES_CONT_SAME;
762                         goto out;
763                 }
764
765                 res = SCST_CMD_STATE_RES_CONT_NEXT;
766                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
767
768                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
769                 cmd->tgtt->preprocessing_done(cmd);
770                 TRACE_DBG("%s", "preprocessing_done() returned");
771                 goto out;
772
773         }
774
775         switch (cmd->data_direction) {
776         case SCST_DATA_WRITE:
777                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
778                 break;
779
780         default:
781                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
782                 break;
783         }
784
785 out:
786         TRACE_EXIT_HRES(res);
787         return res;
788
789 out_no_space:
790         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
791                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
792         scst_set_busy(cmd);
793         scst_set_cmd_abnormal_done_state(cmd);
794         res = SCST_CMD_STATE_RES_CONT_SAME;
795         goto out;
796
797 out_error:
798         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
799         scst_set_cmd_abnormal_done_state(cmd);
800         res = SCST_CMD_STATE_RES_CONT_SAME;
801         goto out;
802 }
803
804 void scst_restart_cmd(struct scst_cmd *cmd, int status, 
805         enum scst_exec_context pref_context)
806 {
807         TRACE_ENTRY();
808
809         TRACE_DBG("Preferred context: %d", pref_context);
810         TRACE_DBG("tag=%llu, status=%#x",
811                   (long long unsigned int)scst_cmd_get_tag(cmd),
812                   status);
813
814 #ifdef CONFIG_SCST_EXTRACHECKS
815         if ((in_irq() || irqs_disabled()) &&
816             ((pref_context == SCST_CONTEXT_DIRECT) ||
817              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
818                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
819                         "SCST_CONTEXT_THREAD instead\n", pref_context,
820                         cmd->tgtt->name);
821                 pref_context = SCST_CONTEXT_THREAD;
822         }
823 #endif
824
825         switch (status) {
826         case SCST_PREPROCESS_STATUS_SUCCESS:
827                 switch (cmd->data_direction) {
828                 case SCST_DATA_WRITE:
829                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
830                         break;
831                 default:
832                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
833                         break;
834                 }
835                 if (cmd->set_sn_on_restart_cmd)
836                         scst_cmd_set_sn(cmd);
837                 /* Small context optimization */
838                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
839                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
840                     ((pref_context == SCST_CONTEXT_SAME) &&
841                      scst_cmd_atomic(cmd))) {
842                         if (cmd->data_direction == SCST_DATA_WRITE) {
843                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
844                                                 &cmd->tgt_dev->tgt_dev_flags))
845                                         pref_context = SCST_CONTEXT_THREAD;
846                         } else {
847                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
848                                                 &cmd->tgt_dev->tgt_dev_flags))
849                                         pref_context = SCST_CONTEXT_THREAD;
850                         }
851                 }
852                 break;
853
854         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
855                 scst_set_cmd_abnormal_done_state(cmd);
856                 break;
857
858         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
859                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
860                 /* go through */
861         case SCST_PREPROCESS_STATUS_ERROR:
862                 scst_set_cmd_error(cmd,
863                            SCST_LOAD_SENSE(scst_sense_hardw_error));
864                 scst_set_cmd_abnormal_done_state(cmd);
865                 break;
866
867         default:
868                 PRINT_ERROR("%s() received unknown status %x", __func__,
869                         status);
870                 scst_set_cmd_abnormal_done_state(cmd);
871                 break;
872         }
873
874         scst_proccess_redirect_cmd(cmd, pref_context, 1);
875
876         TRACE_EXIT();
877         return;
878 }
879 EXPORT_SYMBOL(scst_restart_cmd);
880
881 /* No locks */
882 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
883 {
884         struct scst_tgt *tgt = cmd->sess->tgt;
885         int res = 0;
886         unsigned long flags;
887
888         TRACE_ENTRY();
889
890         spin_lock_irqsave(&tgt->tgt_lock, flags);
891         tgt->retry_cmds++;
892         smp_mb();
893         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
894               tgt->retry_cmds);
895         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
896                 /* At least one cmd finished, so try again */
897                 tgt->retry_cmds--;
898                 TRACE_RETRY("Some command(s) finished, direct retry "
899                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
900                       "retry_cmds=%d)", finished_cmds,
901                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
902                 res = -1;
903                 goto out_unlock_tgt;
904         }
905
906         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
907         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
908
909         if (!tgt->retry_timer_active) {
910                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
911                 add_timer(&tgt->retry_timer);
912                 tgt->retry_timer_active = 1;
913         }
914
915 out_unlock_tgt:
916         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
917
918         TRACE_EXIT_RES(res);
919         return res;
920 }
921
922 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
923 {
924         int res, rc;
925
926         TRACE_ENTRY();
927
928         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
929                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
930                 goto out_dev_done;
931         }
932
933         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
934                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
935                 res = SCST_CMD_STATE_RES_CONT_SAME;
936                 goto out;
937         }
938
939         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
940                 /*
941                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
942                  * optimization.
943                  */
944                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
945                               "context, rescheduling", cmd->tgtt->name);
946                 res = SCST_CMD_STATE_RES_NEED_THREAD;
947                 goto out;
948         }
949
950         while (1) {
951                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
952
953                 res = SCST_CMD_STATE_RES_CONT_NEXT;
954                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
955
956                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
957 #ifdef CONFIG_SCST_DEBUG_RETRY
958                 if (((scst_random() % 100) == 75))
959                         rc = SCST_TGT_RES_QUEUE_FULL;
960                 else
961 #endif
962                         rc = cmd->tgtt->rdy_to_xfer(cmd);
963                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
964
965                 if (likely(rc == SCST_TGT_RES_SUCCESS))
966                         goto out;
967
968                 /* Restore the previous state */
969                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
970
971                 switch (rc) {
972                 case SCST_TGT_RES_QUEUE_FULL:
973                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
974                                 break;
975                         else
976                                 continue;
977
978                 case SCST_TGT_RES_NEED_THREAD_CTX:
979                         TRACE_DBG("Target driver %s "
980                               "rdy_to_xfer() requested thread "
981                               "context, rescheduling", cmd->tgtt->name);
982                         res = SCST_CMD_STATE_RES_NEED_THREAD;
983                         break;
984
985                 default:
986                         goto out_error_rc;
987                 }
988                 break;
989         }
990
991 out:
992         TRACE_EXIT_HRES(res);
993         return res;
994
995 out_error_rc:
996         if (rc == SCST_TGT_RES_FATAL_ERROR) {
997                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
998                      "fatal error", cmd->tgtt->name);
999         } else {
1000                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1001                             "value %d", cmd->tgtt->name, rc);
1002         }
1003         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1004
1005 out_dev_done:
1006         scst_set_cmd_abnormal_done_state(cmd);
1007         res = SCST_CMD_STATE_RES_CONT_SAME;
1008         goto out;
1009 }
1010
1011 /* No locks, but might be in IRQ */
1012 void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1013         enum scst_exec_context context, int check_retries)
1014 {
1015         unsigned long flags;
1016
1017         TRACE_ENTRY();
1018
1019         TRACE_DBG("Context: %x", context);
1020
1021         if (context == SCST_CONTEXT_SAME)
1022                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1023                                                  SCST_CONTEXT_DIRECT;
1024
1025         switch (context) {
1026         case SCST_CONTEXT_DIRECT_ATOMIC:
1027                 scst_process_active_cmd(cmd, true);
1028                 break;
1029
1030         case SCST_CONTEXT_DIRECT:
1031                 if (check_retries)
1032                         scst_check_retries(cmd->tgt);
1033                 scst_process_active_cmd(cmd, false);
1034                 break;
1035
1036         default:
1037                 PRINT_ERROR("Context %x is unknown, using the thread one",
1038                             context);
1039                 /* go through */
1040         case SCST_CONTEXT_THREAD:
1041                 if (check_retries)
1042                         scst_check_retries(cmd->tgt);
1043                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1044                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1045                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1046                         list_add(&cmd->cmd_list_entry,
1047                                 &cmd->cmd_lists->active_cmd_list);
1048                 else
1049                         list_add_tail(&cmd->cmd_list_entry,
1050                                 &cmd->cmd_lists->active_cmd_list);
1051                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1052                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1053                 break;
1054
1055         case SCST_CONTEXT_TASKLET:
1056                 if (check_retries)
1057                         scst_check_retries(cmd->tgt);
1058                 scst_schedule_tasklet(cmd);
1059                 break;
1060         }
1061
1062         TRACE_EXIT();
1063         return;
1064 }
1065
1066 void scst_rx_data(struct scst_cmd *cmd, int status,
1067         enum scst_exec_context pref_context)
1068 {
1069         TRACE_ENTRY();
1070
1071         TRACE_DBG("Preferred context: %d", pref_context);
1072         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1073
1074 #ifdef CONFIG_SCST_EXTRACHECKS
1075         if ((in_irq() || irqs_disabled()) &&
1076             ((pref_context == SCST_CONTEXT_DIRECT) ||
1077              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1078                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1079                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1080                         cmd->tgtt->name);
1081                 pref_context = SCST_CONTEXT_THREAD;
1082         }
1083 #endif
1084
1085         switch (status) {
1086         case SCST_RX_STATUS_SUCCESS:
1087 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1088                 if (cmd->tgt_sg) {
1089                         int i;
1090                         struct scatterlist *sg = cmd->tgt_sg;
1091                         TRACE_RECV_BOT("RX data for cmd %p "
1092                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1093                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1094                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1095                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1096                                         sg_virt(&sg[i]), sg[i].length);
1097                         }
1098                 }
1099 #endif
1100                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1101                 /* Small context optimization */
1102                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1103                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1104                     ((pref_context == SCST_CONTEXT_SAME) &&
1105                      scst_cmd_atomic(cmd))) {
1106                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1107                                         &cmd->tgt_dev->tgt_dev_flags))
1108                                 pref_context = SCST_CONTEXT_THREAD;
1109                 }
1110                 break;
1111
1112         case SCST_RX_STATUS_ERROR_SENSE_SET:
1113                 scst_set_cmd_abnormal_done_state(cmd);
1114                 break;
1115
1116         case SCST_RX_STATUS_ERROR_FATAL:
1117                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1118                 /* go through */
1119         case SCST_RX_STATUS_ERROR:
1120                 scst_set_cmd_error(cmd,
1121                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1122                 scst_set_cmd_abnormal_done_state(cmd);
1123                 break;
1124
1125         default:
1126                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1127                         status);
1128                 scst_set_cmd_abnormal_done_state(cmd);
1129                 break;
1130         }
1131
1132         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1133
1134         TRACE_EXIT();
1135         return;
1136 }
1137 EXPORT_SYMBOL(scst_rx_data);
1138
1139 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1140 {
1141         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1142
1143         TRACE_ENTRY();
1144
1145         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1146
1147         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1148                 goto out;
1149
1150         TRACE_DBG("Calling pre_exec(%p)", cmd);
1151         rc = cmd->tgtt->pre_exec(cmd);
1152         TRACE_DBG("pre_exec() returned %d", rc);
1153
1154         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1155                 switch (rc) {
1156                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1157                         scst_set_cmd_abnormal_done_state(cmd);
1158                         break;
1159                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1160                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1161                         /* go through */
1162                 case SCST_PREPROCESS_STATUS_ERROR:
1163                         scst_set_cmd_error(cmd,
1164                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1165                         scst_set_cmd_abnormal_done_state(cmd);
1166                         break;
1167                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1168                         TRACE_DBG("Target driver's %s pre_exec() requested "
1169                                 "thread context, rescheduling",
1170                                 cmd->tgtt->name);
1171                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1172                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1173                         break;
1174                 default:
1175                         sBUG();
1176                         break;
1177                 }
1178         }
1179
1180 out:
1181         TRACE_EXIT_RES(res);
1182         return res;
1183 }
1184
1185 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1186         const uint8_t *rq_sense, int rq_sense_len, int resid)
1187 {
1188         TRACE_ENTRY();
1189
1190 #ifdef CONFIG_SCST_MEASURE_LATENCY
1191         {
1192                 struct timespec ts;
1193                 getnstimeofday(&ts);
1194                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1195                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1196                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1197                         ts.tv_sec, ts.tv_nsec);
1198         }
1199 #endif
1200
1201         cmd->status = result & 0xff;
1202         cmd->msg_status = msg_byte(result);
1203         cmd->host_status = host_byte(result);
1204         cmd->driver_status = driver_byte(result);
1205         if (unlikely(resid != 0)) {
1206 #ifdef CONFIG_SCST_EXTRACHECKS
1207                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1208                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1209                                 "op %x)", resid, cmd->resp_data_len,
1210                                 cmd->cdb[0]);
1211                 } else
1212 #endif
1213                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1214         }
1215
1216         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1217                 /* We might have double reset UA here */
1218                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1219                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1220
1221                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1222         }
1223
1224         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1225               "cmd->msg_status=%x, cmd->host_status=%x, "
1226               "cmd->driver_status=%x (cmd %p)",cmd,  result, cmd->status, resid,
1227               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1228
1229         cmd->completed = 1;
1230
1231         TRACE_EXIT();
1232         return;
1233 }
1234
1235 /* For small context optimization */
1236 static inline enum scst_exec_context scst_optimize_post_exec_context(
1237         struct scst_cmd *cmd, enum scst_exec_context context)
1238 {
1239         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1240             (context == SCST_CONTEXT_TASKLET) ||
1241             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1242                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1243                                 &cmd->tgt_dev->tgt_dev_flags))
1244                         context = SCST_CONTEXT_THREAD;
1245         }
1246         return context;
1247 }
1248
1249 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1250 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1251                                             struct scsi_request **req)
1252 {
1253         struct scst_cmd *cmd = NULL;
1254
1255         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1256                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1257
1258         if (cmd == NULL) {
1259                 PRINT_ERROR("%s", "Request with NULL cmd");
1260                 if (*req)
1261                         scsi_release_request(*req);
1262         }
1263
1264         return cmd;
1265 }
1266
1267 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1268 {
1269         struct scsi_request *req = NULL;
1270         struct scst_cmd *cmd;
1271         enum scst_exec_context context;
1272
1273         TRACE_ENTRY();
1274
1275         cmd = scst_get_cmd(scsi_cmd, &req);
1276         if (cmd == NULL)
1277                 goto out;
1278
1279         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1280                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1281
1282         /* Clear out request structure */
1283         req->sr_use_sg = 0;
1284         req->sr_sglist_len = 0;
1285         req->sr_bufflen = 0;
1286         req->sr_buffer = NULL;
1287         req->sr_underflow = 0;
1288         req->sr_request->rq_disk = NULL; /* disown request blk */
1289
1290         scst_release_request(cmd);
1291
1292         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1293
1294         scst_proccess_redirect_cmd(cmd,
1295                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1296                                                 0);
1297
1298 out:
1299         TRACE_EXIT();
1300         return;
1301 }
1302 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1303 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1304 {
1305         struct scst_cmd *cmd;
1306
1307         TRACE_ENTRY();
1308
1309         cmd = (struct scst_cmd *)data;
1310         if (cmd == NULL)
1311                 goto out;
1312
1313         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1314
1315         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1316
1317         scst_proccess_redirect_cmd(cmd,
1318                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1319                                                 0);
1320
1321 out:
1322         TRACE_EXIT();
1323         return;
1324 }
1325 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1326
1327 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1328         enum scst_exec_context pref_context)
1329 {
1330         TRACE_ENTRY();
1331
1332 #ifdef CONFIG_SCST_MEASURE_LATENCY
1333         {
1334                 struct timespec ts;
1335                 getnstimeofday(&ts);
1336                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1337                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1338                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1339                         ts.tv_sec, ts.tv_nsec);
1340         }
1341 #endif
1342
1343         if (next_state == SCST_CMD_STATE_DEFAULT)
1344                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1345
1346 #if defined(CONFIG_SCST_DEBUG)
1347         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1348                 if (cmd->sg) {
1349                         int i;
1350                         struct scatterlist *sg = cmd->sg;
1351                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1352                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1353                         for (i = 0; i < cmd->sg_cnt; ++i) {
1354                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1355                                         "Exec'd sg", sg_virt(&sg[i]),
1356                                         sg[i].length);
1357                         }
1358                 }
1359         }
1360 #endif
1361
1362         cmd->state = next_state;
1363
1364 #ifdef CONFIG_SCST_EXTRACHECKS
1365         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1366             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1367             (next_state != SCST_CMD_STATE_FINISHED)) {
1368                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1369                         __func__, next_state, cmd->cdb[0]);
1370                 scst_set_cmd_error(cmd,
1371                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1372                 scst_set_cmd_abnormal_done_state(cmd);
1373         }
1374 #endif
1375         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1376         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1377
1378         TRACE_EXIT();
1379         return;
1380 }
1381
1382 static int scst_report_luns_local(struct scst_cmd *cmd)
1383 {
1384         int rc;
1385         int dev_cnt = 0;
1386         int buffer_size;
1387         int i;
1388         struct scst_tgt_dev *tgt_dev = NULL;
1389         uint8_t *buffer;
1390         int offs, overflow = 0;
1391
1392         TRACE_ENTRY();
1393
1394         rc = scst_check_local_events(cmd);
1395         if (unlikely(rc != 0))
1396                 goto out_done;
1397
1398         cmd->status = 0;
1399         cmd->msg_status = 0;
1400         cmd->host_status = DID_OK;
1401         cmd->driver_status = 0;
1402
1403         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1404                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1405                         "LUNS command", cmd->cdb[2]);
1406                 goto out_err;
1407         }
1408
1409         buffer_size = scst_get_buf_first(cmd, &buffer);
1410         if (unlikely(buffer_size == 0))
1411                 goto out_compl;
1412         else if (unlikely(buffer_size < 0))
1413                 goto out_err;
1414
1415         if (buffer_size < 16)
1416                 goto out_put_err;
1417
1418         memset(buffer, 0, buffer_size);
1419         offs = 8;
1420
1421         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1422         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1423                 struct list_head *sess_tgt_dev_list_head =
1424                         &cmd->sess->sess_tgt_dev_list_hash[i];
1425                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1426                                 sess_tgt_dev_list_entry) {
1427                         if (!overflow) {
1428                                 if (offs >= buffer_size) {
1429                                         scst_put_buf(cmd, buffer);
1430                                         buffer_size = scst_get_buf_next(cmd,
1431                                                                        &buffer);
1432                                         if (buffer_size > 0) {
1433                                                 memset(buffer, 0, buffer_size);
1434                                                 offs = 0;
1435                                         } else {
1436                                                 overflow = 1;
1437                                                 goto inc_dev_cnt;
1438                                         }
1439                                 }
1440                                 if ((buffer_size - offs) < 8) {
1441                                         PRINT_ERROR("Buffer allocated for "
1442                                                 "REPORT LUNS command doesn't "
1443                                                 "allow to fit 8 byte entry "
1444                                                 "(buffer_size=%d)",
1445                                                 buffer_size);
1446                                         goto out_put_hw_err;
1447                                 }
1448                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1449                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1450                                 offs += 8;
1451                         }
1452 inc_dev_cnt:
1453                         dev_cnt++;
1454                 }
1455         }
1456         if (!overflow)
1457                 scst_put_buf(cmd, buffer);
1458
1459         /* Set the response header */
1460         buffer_size = scst_get_buf_first(cmd, &buffer);
1461         if (unlikely(buffer_size == 0))
1462                 goto out_compl;
1463         else if (unlikely(buffer_size < 0))
1464                 goto out_err;
1465
1466         dev_cnt *= 8;
1467         buffer[0] = (dev_cnt >> 24) & 0xff;
1468         buffer[1] = (dev_cnt >> 16) & 0xff;
1469         buffer[2] = (dev_cnt >> 8) & 0xff;
1470         buffer[3] = dev_cnt & 0xff;
1471
1472         scst_put_buf(cmd, buffer);
1473
1474         dev_cnt += 8;
1475         if (dev_cnt < cmd->resp_data_len)
1476                 scst_set_resp_data_len(cmd, dev_cnt);
1477
1478 out_compl:
1479         cmd->completed = 1;
1480
1481 out_done:
1482         /* Report the result */
1483         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1484
1485         TRACE_EXIT();
1486         return SCST_EXEC_COMPLETED;
1487
1488 out_put_err:
1489         scst_put_buf(cmd, buffer);
1490
1491 out_err:
1492         scst_set_cmd_error(cmd,
1493                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1494         goto out_compl;
1495
1496 out_put_hw_err:
1497         scst_put_buf(cmd, buffer);
1498         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1499         goto out_compl;
1500 }
1501
1502 static int scst_pre_select(struct scst_cmd *cmd)
1503 {
1504         int res = SCST_EXEC_NOT_COMPLETED;
1505
1506         TRACE_ENTRY();
1507
1508         if (scst_cmd_atomic(cmd)) {
1509                 res = SCST_EXEC_NEED_THREAD;
1510                 goto out;
1511         }
1512
1513         scst_block_dev_cmd(cmd, 1);
1514
1515         /* Check for local events will be done when cmd will be executed */
1516
1517 out:
1518         TRACE_EXIT_RES(res);
1519         return res;
1520 }
1521
1522 static int scst_reserve_local(struct scst_cmd *cmd)
1523 {
1524         int res = SCST_EXEC_NOT_COMPLETED, rc;
1525         struct scst_device *dev;
1526         struct scst_tgt_dev *tgt_dev_tmp;
1527
1528         TRACE_ENTRY();
1529
1530         if (scst_cmd_atomic(cmd)) {
1531                 res = SCST_EXEC_NEED_THREAD;
1532                 goto out;
1533         }
1534
1535         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1536                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1537                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1538                 scst_set_cmd_error(cmd,
1539                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1540                 goto out_done;
1541         }
1542
1543         dev = cmd->dev;
1544
1545         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1546                 scst_block_dev_cmd(cmd, 1);
1547
1548         rc = scst_check_local_events(cmd);
1549         if (unlikely(rc != 0))
1550                 goto out_done;
1551
1552         spin_lock_bh(&dev->dev_lock);
1553
1554         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1555                 spin_unlock_bh(&dev->dev_lock);
1556                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1557                 goto out_done;
1558         }
1559
1560         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1561                             dev_tgt_dev_list_entry) {
1562                 if (cmd->tgt_dev != tgt_dev_tmp)
1563                         set_bit(SCST_TGT_DEV_RESERVED,
1564                                 &tgt_dev_tmp->tgt_dev_flags);
1565         }
1566         dev->dev_reserved = 1;
1567
1568         spin_unlock_bh(&dev->dev_lock);
1569
1570 out:
1571         TRACE_EXIT_RES(res);
1572         return res;
1573
1574 out_done:
1575         /* Report the result */
1576         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1577         res = SCST_EXEC_COMPLETED;
1578         goto out;
1579 }
1580
1581 static int scst_release_local(struct scst_cmd *cmd)
1582 {
1583         int res = SCST_EXEC_NOT_COMPLETED, rc;
1584         struct scst_tgt_dev *tgt_dev_tmp;
1585         struct scst_device *dev;
1586
1587         TRACE_ENTRY();
1588
1589         if (scst_cmd_atomic(cmd)) {
1590                 res = SCST_EXEC_NEED_THREAD;
1591                 goto out;
1592         }
1593
1594         dev = cmd->dev;
1595
1596         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1597                 scst_block_dev_cmd(cmd, 1);
1598
1599         rc = scst_check_local_events(cmd);
1600         if (unlikely(rc != 0))
1601                 goto out_done;
1602
1603         spin_lock_bh(&dev->dev_lock);
1604
1605         /*
1606          * The device could be RELEASED behind us, if RESERVING session
1607          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1608          * matter, so use lock and no retest for DEV_RESERVED bits again
1609          */
1610         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1611                 res = SCST_EXEC_COMPLETED;
1612                 cmd->status = 0;
1613                 cmd->msg_status = 0;
1614                 cmd->host_status = DID_OK;
1615                 cmd->driver_status = 0;
1616                 cmd->completed = 1;
1617         } else {
1618                 list_for_each_entry(tgt_dev_tmp,
1619                                     &dev->dev_tgt_dev_list,
1620                                     dev_tgt_dev_list_entry) {
1621                         clear_bit(SCST_TGT_DEV_RESERVED,
1622                                 &tgt_dev_tmp->tgt_dev_flags);
1623                 }
1624                 dev->dev_reserved = 0;
1625         }
1626
1627         spin_unlock_bh(&dev->dev_lock);
1628
1629         if (res == SCST_EXEC_COMPLETED)
1630                 goto out_done;
1631
1632 out:
1633         TRACE_EXIT_RES(res);
1634         return res;
1635
1636 out_done:
1637         res = SCST_EXEC_COMPLETED;
1638         /* Report the result */
1639         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1640         goto out;
1641 }
1642
1643 /* No locks, no IRQ or IRQ-safe context allowed */
1644 int scst_check_local_events(struct scst_cmd *cmd)
1645 {
1646         int res, rc;
1647         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1648         struct scst_device *dev = cmd->dev;
1649
1650         TRACE_ENTRY();
1651
1652         /*
1653          * There's no race here, because we need to trace commands sent
1654          * *after* dev_double_ua_possible flag was set.
1655          */
1656         if (unlikely(dev->dev_double_ua_possible))
1657                 cmd->double_ua_possible = 1;
1658
1659         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1660                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1661                 goto out_uncomplete;
1662         }
1663
1664         /* Reserve check before Unit Attention */
1665         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1666                               &tgt_dev->tgt_dev_flags))) {
1667                 if (cmd->cdb[0] != INQUIRY &&
1668                     cmd->cdb[0] != REPORT_LUNS &&
1669                     cmd->cdb[0] != RELEASE &&
1670                     cmd->cdb[0] != RELEASE_10 &&
1671                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1672                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1673                      (cmd->cdb[4] & 3)) &&
1674                     cmd->cdb[0] != LOG_SENSE &&
1675                     cmd->cdb[0] != REQUEST_SENSE) {
1676                         scst_set_cmd_error_status(cmd,
1677                                 SAM_STAT_RESERVATION_CONFLICT);
1678                         goto out_complete;
1679                 }
1680         }
1681
1682         /* If we had internal bus reset, set the command error unit attention */
1683         if ((dev->scsi_dev != NULL) &&
1684             unlikely(dev->scsi_dev->was_reset)) {
1685                 if (scst_is_ua_command(cmd)) {
1686                         int done = 0;
1687                         /*
1688                          * Prevent more than 1 cmd to be triggered by
1689                          * was_reset.
1690                          */
1691                         spin_lock_bh(&dev->dev_lock);
1692                         barrier(); /* to reread was_reset */
1693                         if (dev->scsi_dev->was_reset) {
1694                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1695                                 scst_set_cmd_error(cmd,
1696                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1697                                 /*
1698                                  * It looks like it is safe to clear was_reset
1699                                  * here.
1700                                  */
1701                                 dev->scsi_dev->was_reset = 0;
1702                                 done = 1;
1703                         }
1704                         spin_unlock_bh(&dev->dev_lock);
1705
1706                         if (done)
1707                                 goto out_complete;
1708                 }
1709         }
1710
1711         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1712                         &cmd->tgt_dev->tgt_dev_flags))) {
1713                 if (scst_is_ua_command(cmd)) {
1714                         rc = scst_set_pending_UA(cmd);
1715                         if (rc == 0)
1716                                 goto out_complete;
1717                 }
1718         }
1719
1720         res = 0;
1721
1722 out:
1723         TRACE_EXIT_RES(res);
1724         return res;
1725
1726 out_complete:
1727         res = 1;
1728         sBUG_ON(!cmd->completed);
1729         goto out;
1730
1731 out_uncomplete:
1732         res = -1;
1733         goto out;
1734 }
1735 EXPORT_SYMBOL(scst_check_local_events);
1736
1737 /* No locks */
1738 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1739 {
1740         if (slot == NULL)
1741                 goto inc;
1742
1743         /* Optimized for lockless fast path */
1744
1745         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1746                 atomic_read(slot));
1747
1748         if (!atomic_dec_and_test(slot))
1749                 goto out;
1750
1751         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1752                 tgt_dev->num_free_sn_slots);
1753         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1754                 spin_lock_irq(&tgt_dev->sn_lock);
1755                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1756                         if (tgt_dev->num_free_sn_slots < 0)
1757                                 tgt_dev->cur_sn_slot = slot;
1758                         /*
1759                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1760                          */
1761                         smp_mb();
1762                         tgt_dev->num_free_sn_slots++;
1763                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1764                                 tgt_dev->num_free_sn_slots);
1765
1766                 }
1767                 spin_unlock_irq(&tgt_dev->sn_lock);
1768         }
1769
1770 inc:
1771         /*
1772          * No locks is needed, because only one thread at time can
1773          * be here (serialized by sn). Also it is supposed that there
1774          * could not be half-incremented halves.
1775          */
1776         tgt_dev->expected_sn++;
1777         smp_mb(); /* write must be before def_cmd_count read */
1778         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1779
1780 out:
1781         return;
1782 }
1783
1784 /* No locks */
1785 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1786         bool make_active)
1787 {
1788         /* For HQ commands SN is not set */
1789         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1790                                cmd->sn_set && !cmd->retry;
1791         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1792         struct scst_cmd *res;
1793
1794         TRACE_ENTRY();
1795
1796         if (inc_expected_sn)
1797                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1798
1799         if (make_active) {
1800                 scst_make_deferred_commands_active(tgt_dev);
1801                 res = NULL;
1802         } else
1803                 res = scst_check_deferred_commands(tgt_dev);
1804
1805         TRACE_EXIT_HRES(res);
1806         return res;
1807 }
1808
1809 /* cmd must be additionally referenced to not die inside */
1810 static int scst_do_real_exec(struct scst_cmd *cmd)
1811 {
1812         int res = SCST_EXEC_NOT_COMPLETED, rc;
1813         struct scst_device *dev = cmd->dev;
1814         struct scst_dev_type *handler = dev->handler;
1815
1816         TRACE_ENTRY();
1817
1818         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1819
1820         if (handler->exec) {
1821                 if (unlikely(!dev->handler->exec_atomic &&
1822                              scst_cmd_atomic(cmd))) {
1823                         /*
1824                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1825                          * optimization.
1826                          */
1827                         TRACE_DBG("Dev handler %s exec() needs thread "
1828                                 "context, rescheduling", dev->handler->name);
1829                         res = SCST_EXEC_NEED_THREAD;
1830                         goto out_restore;
1831                 }
1832
1833                 TRACE_DBG("Calling dev handler %s exec(%p)",
1834                       handler->name, cmd);
1835                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1836                         cmd->cdb_len);
1837                 res = handler->exec(cmd);
1838                 TRACE_DBG("Dev handler %s exec() returned %d",
1839                       handler->name, res);
1840
1841                 if (res == SCST_EXEC_COMPLETED)
1842                         goto out_complete;
1843                 else if (res == SCST_EXEC_NEED_THREAD)
1844                         goto out_restore;
1845
1846                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1847         }
1848
1849         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1850
1851         if (unlikely(dev->scsi_dev == NULL)) {
1852                 PRINT_ERROR("Command for virtual device must be "
1853                         "processed by device handler (lun %lld)!",
1854                         (long long unsigned int)cmd->lun);
1855                 goto out_error;
1856         }
1857
1858         res = scst_check_local_events(cmd);
1859         if (unlikely(res != 0))
1860                 goto out_done;
1861
1862 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1863         if (unlikely(scst_cmd_atomic(cmd))) {
1864                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1865                         "context, rescheduling to the thread (handler %s)",
1866                         handler->name);
1867                 res = SCST_EXEC_NEED_THREAD;
1868                 goto out_restore;
1869         }
1870 #endif
1871
1872 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1873         if (unlikely(scst_alloc_request(cmd) != 0)) {
1874                 if (scst_cmd_atomic(cmd)) {
1875                         res = SCST_EXEC_NEED_THREAD;
1876                         goto out_restore;
1877                 } else {
1878                         PRINT_INFO("%s", "Unable to allocate request, "
1879                                 "sending BUSY status");
1880                         goto out_busy;
1881                 }
1882         }
1883
1884         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1885                     (void *)cmd->scsi_req->sr_buffer,
1886                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1887                     cmd->retries);
1888 #else
1889         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1890                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1891                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1892                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1893         if (unlikely(rc != 0)) {
1894                 if (scst_cmd_atomic(cmd)) {
1895                         res = SCST_EXEC_NEED_THREAD;
1896                         goto out_restore;
1897                 } else {
1898                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1899                         goto out_error;
1900                 }
1901         }
1902 #endif
1903
1904 out_complete:
1905         res = SCST_EXEC_COMPLETED;
1906
1907 out:
1908         TRACE_EXIT();
1909         return res;
1910
1911 out_restore:
1912         /* Restore the state */
1913         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1914         goto out;
1915
1916 out_error:
1917         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1918         goto out_done;
1919
1920 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1921 out_busy:
1922         scst_set_busy(cmd);
1923         /* go through */
1924 #endif
1925
1926 out_done:
1927         res = SCST_EXEC_COMPLETED;
1928         /* Report the result */
1929         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1930         goto out_complete;
1931 }
1932
1933 static inline int scst_real_exec(struct scst_cmd *cmd)
1934 {
1935         int res;
1936
1937         TRACE_ENTRY();
1938
1939         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1940         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1941         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1942
1943         __scst_cmd_get(cmd);
1944
1945         res = scst_do_real_exec(cmd);
1946
1947         if (likely(res == SCST_EXEC_COMPLETED)) {
1948                 scst_post_exec_sn(cmd, true);
1949                 if (cmd->dev->scsi_dev != NULL)
1950                         generic_unplug_device(
1951                                 cmd->dev->scsi_dev->request_queue);
1952         } else
1953                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1954
1955         __scst_cmd_put(cmd);
1956
1957         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1958
1959         TRACE_EXIT_RES(res);
1960         return res;
1961 }
1962
1963 static int scst_do_local_exec(struct scst_cmd *cmd)
1964 {
1965         int res;
1966         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1967
1968         TRACE_ENTRY();
1969
1970         /* Check READ_ONLY device status */
1971         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1972             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1973              cmd->cdb[0] == WRITE_10 ||
1974              cmd->cdb[0] == WRITE_12 ||
1975              cmd->cdb[0] == WRITE_16 ||
1976              cmd->cdb[0] == WRITE_VERIFY ||
1977              cmd->cdb[0] == WRITE_VERIFY_12 ||
1978              cmd->cdb[0] == WRITE_VERIFY_16 ||
1979              (cmd->dev->handler->type == TYPE_TAPE &&
1980               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1981                 scst_set_cmd_error(cmd,
1982                            SCST_LOAD_SENSE(scst_sense_data_protect));
1983                 goto out_done;
1984         }
1985
1986         /*
1987          * Adding new commands here don't forget to update
1988          * scst_is_cmd_local() in scst.h, if necessary
1989          */
1990
1991         switch (cmd->cdb[0]) {
1992         case MODE_SELECT:
1993         case MODE_SELECT_10:
1994         case LOG_SELECT:
1995                 res = scst_pre_select(cmd);
1996                 break;
1997         case RESERVE:
1998         case RESERVE_10:
1999                 res = scst_reserve_local(cmd);
2000                 break;
2001         case RELEASE:
2002         case RELEASE_10:
2003                 res = scst_release_local(cmd);
2004                 break;
2005         case REPORT_LUNS:
2006                 res = scst_report_luns_local(cmd);
2007                 break;
2008         default:
2009                 res = SCST_EXEC_NOT_COMPLETED;
2010                 break;
2011         }
2012
2013 out:
2014         TRACE_EXIT_RES(res);
2015         return res;
2016
2017 out_done:
2018         /* Report the result */
2019         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2020         res = SCST_EXEC_COMPLETED;
2021         goto out;
2022 }
2023
2024 static int scst_local_exec(struct scst_cmd *cmd)
2025 {
2026         int res;
2027
2028         TRACE_ENTRY();
2029
2030         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2031         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2032         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2033
2034         __scst_cmd_get(cmd);
2035
2036         res = scst_do_local_exec(cmd);
2037         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2038                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2039         else if (res == SCST_EXEC_COMPLETED)
2040                 scst_post_exec_sn(cmd, true);
2041         else
2042                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2043
2044         __scst_cmd_put(cmd);
2045
2046         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2047         TRACE_EXIT_RES(res);
2048         return res;
2049 }
2050
2051 static int scst_exec(struct scst_cmd **active_cmd)
2052 {
2053         struct scst_cmd *cmd = *active_cmd;
2054         struct scst_cmd *ref_cmd;
2055         struct scst_device *dev = cmd->dev;
2056         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2057
2058         TRACE_ENTRY();
2059
2060         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2061                 goto out;
2062
2063         /* To protect tgt_dev */
2064         ref_cmd = cmd;
2065         __scst_cmd_get(ref_cmd);
2066
2067         count = 0;
2068         while (1) {
2069                 int rc;
2070
2071                 cmd->sent_for_exec = 1;
2072                 cmd->scst_cmd_done = scst_cmd_done_local;
2073                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2074
2075                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2076                     (cmd->data_direction == SCST_DATA_WRITE))
2077                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2078
2079                 rc = scst_do_local_exec(cmd);
2080                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2081                         /* Nothing to do */;
2082                 else if (rc == SCST_EXEC_NEED_THREAD) {
2083                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2084                                 "thread context, rescheduling");
2085                         scst_dec_on_dev_cmd(cmd);
2086                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2087                         break;
2088                 } else {
2089                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2090                         goto done;
2091                 }
2092
2093                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2094
2095                 rc = scst_do_real_exec(cmd);
2096                 if (likely(rc == SCST_EXEC_COMPLETED))
2097                         /* Nothing to do */;
2098                 else if (rc == SCST_EXEC_NEED_THREAD) {
2099                         TRACE_DBG("scst_real_exec() requested thread "
2100                                 "context, rescheduling (cmd %p)", cmd);
2101                         scst_dec_on_dev_cmd(cmd);
2102                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2103                         break;
2104                 } else
2105                         sBUG();
2106
2107 done:
2108                 count++;
2109
2110                 cmd = scst_post_exec_sn(cmd, false);
2111                 if (cmd == NULL)
2112                         break;
2113
2114                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2115                         break;
2116
2117                 __scst_cmd_put(ref_cmd);
2118                 ref_cmd = cmd;
2119                 __scst_cmd_get(ref_cmd);
2120         }
2121
2122         *active_cmd = cmd;
2123
2124         if (count == 0)
2125                 goto out_put;
2126
2127         if (dev->scsi_dev != NULL)
2128                 generic_unplug_device(dev->scsi_dev->request_queue);
2129
2130 out_put:
2131         __scst_cmd_put(ref_cmd);
2132         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2133
2134 out:
2135         TRACE_EXIT_RES(res);
2136         return res;
2137 }
2138
2139 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2140 {
2141         int res;
2142         struct scst_cmd *cmd = *active_cmd;
2143         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2144         typeof(tgt_dev->expected_sn) expected_sn;
2145
2146         TRACE_ENTRY();
2147
2148 #ifdef CONFIG_SCST_MEASURE_LATENCY
2149         if (cmd->pre_exec_finish == 0) {
2150                 struct timespec ts;
2151                 getnstimeofday(&ts);
2152                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2153                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2154                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2155                         ts.tv_sec, ts.tv_nsec);
2156         }
2157 #endif
2158
2159         if (unlikely(cmd->internal))
2160                 goto exec;
2161
2162         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2163                 goto exec;
2164
2165         sBUG_ON(!cmd->sn_set);
2166
2167         expected_sn = tgt_dev->expected_sn;
2168         /* Optimized for lockless fast path */
2169         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2170                 spin_lock_irq(&tgt_dev->sn_lock);
2171
2172                 tgt_dev->def_cmd_count++;
2173                 smp_mb();
2174
2175                 expected_sn = tgt_dev->expected_sn;
2176                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2177                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2178                                               &cmd->cmd_flags))) {
2179                                 /* Necessary to allow aborting out of sn cmds */
2180                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2181                                         "(tag %llu, sn %lu)", cmd,
2182                                         (long long unsigned)cmd->tag, cmd->sn);
2183                                 tgt_dev->def_cmd_count--;
2184                                 scst_set_cmd_abnormal_done_state(cmd);
2185                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2186                         } else {
2187                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2188                                         "expected_sn=%ld)", cmd, cmd->sn,
2189                                         cmd->sn_set, expected_sn);
2190                                 list_add_tail(&cmd->sn_cmd_list_entry,
2191                                               &tgt_dev->deferred_cmd_list);
2192                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2193                         }
2194                         spin_unlock_irq(&tgt_dev->sn_lock);
2195                         goto out;
2196                 } else {
2197                         TRACE_SN("Somebody incremented expected_sn %ld, "
2198                                 "continuing", expected_sn);
2199                         tgt_dev->def_cmd_count--;
2200                         spin_unlock_irq(&tgt_dev->sn_lock);
2201                 }
2202         }
2203
2204 exec:
2205         res = scst_exec(active_cmd);
2206
2207 out:
2208         TRACE_EXIT_HRES(res);
2209         return res;
2210 }
2211
2212 /* No locks supposed to be held */
2213 static int scst_check_sense(struct scst_cmd *cmd)
2214 {
2215         int res = 0;
2216         struct scst_device *dev = cmd->dev;
2217
2218         TRACE_ENTRY();
2219
2220         if (unlikely(cmd->ua_ignore))
2221                 goto out;
2222
2223         /* If we had internal bus reset behind us, set the command error UA */
2224         if ((dev->scsi_dev != NULL) &&
2225             unlikely(cmd->host_status == DID_RESET) &&
2226             scst_is_ua_command(cmd)) {
2227                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2228                       dev->scsi_dev->was_reset, cmd->host_status);
2229                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2230                 /* It looks like it is safe to clear was_reset here */
2231                 dev->scsi_dev->was_reset = 0;
2232         }
2233
2234         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2235             SCST_SENSE_VALID(cmd->sense)) {
2236                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2237                         SCST_SENSE_BUFFERSIZE);
2238
2239                 /* Check Unit Attention Sense Key */
2240                 if (scst_is_ua_sense(cmd->sense)) {
2241                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2242                                 if (cmd->double_ua_possible) {
2243                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2244                                                 "detected for device %p", dev);
2245                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2246                                                 " %p (tag %llu)", cmd,
2247                                                 (long long unsigned)cmd->tag);
2248
2249                                         cmd->status = 0;
2250                                         cmd->msg_status = 0;
2251                                         cmd->host_status = DID_OK;
2252                                         cmd->driver_status = 0;
2253
2254                                         mempool_free(cmd->sense,
2255                                                      scst_sense_mempool);
2256                                         cmd->sense = NULL;
2257
2258                                         scst_check_restore_sg_buff(cmd);
2259
2260                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2261                                         cmd->data_direction =
2262                                                 cmd->dbl_ua_orig_data_direction;
2263                                         cmd->resp_data_len =
2264                                                 cmd->dbl_ua_orig_resp_data_len;
2265
2266                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2267                                         cmd->retry = 1;
2268                                         res = 1;
2269                                         goto out;
2270                                 }
2271                         }
2272                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2273                                 SCST_SENSE_BUFFERSIZE);
2274                 }
2275         }
2276
2277         if (unlikely(cmd->double_ua_possible)) {
2278                 if (scst_is_ua_command(cmd)) {
2279                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2280                                 "cmd %p)", dev, cmd);
2281                         /*
2282                          * Lock used to protect other flags in the bitfield
2283                          * (just in case, actually). Those 2 flags can't be
2284                          * changed in parallel, because the device is
2285                          * serialized.
2286                          */
2287                         spin_lock_bh(&dev->dev_lock);
2288                         dev->dev_double_ua_possible = 0;
2289                         dev->dev_serialized = 0;
2290                         spin_unlock_bh(&dev->dev_lock);
2291                 }
2292         }
2293
2294 out:
2295         TRACE_EXIT_RES(res);
2296         return res;
2297 }
2298
2299 static int scst_check_auto_sense(struct scst_cmd *cmd)
2300 {
2301         int res = 0;
2302
2303         TRACE_ENTRY();
2304
2305         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2306             (!SCST_SENSE_VALID(cmd->sense) ||
2307              SCST_NO_SENSE(cmd->sense))) {
2308                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2309                       "cmd->status=%x, cmd->msg_status=%x, "
2310                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2311                       cmd->status, cmd->msg_status, cmd->host_status,
2312                       cmd->driver_status, cmd);
2313                 res = 1;
2314         } else if (unlikely(cmd->host_status)) {
2315                 if ((cmd->host_status == DID_REQUEUE) ||
2316                     (cmd->host_status == DID_IMM_RETRY) ||
2317                     (cmd->host_status == DID_SOFT_ERROR) ||
2318                     (cmd->host_status == DID_ABORT)) {
2319                         scst_set_busy(cmd);
2320                 } else {
2321                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2322                                 "received, returning HARDWARE ERROR instead "
2323                                 "(cmd %p)", cmd->host_status, cmd);
2324                         scst_set_cmd_error(cmd,
2325                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2326                 }
2327         }
2328
2329         TRACE_EXIT_RES(res);
2330         return res;
2331 }
2332
2333 static int scst_pre_dev_done(struct scst_cmd *cmd)
2334 {
2335         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2336
2337         TRACE_ENTRY();
2338
2339         if (unlikely(scst_check_auto_sense(cmd))) {
2340                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2341                             "without sense data (opcode 0x%x), issuing "
2342                             "REQUEST SENSE", cmd->cdb[0]);
2343                 rc = scst_prepare_request_sense(cmd);
2344                 if (rc == 0)
2345                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2346                 else {
2347                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2348                                     "returning HARDWARE ERROR");
2349                         scst_set_cmd_error(cmd,
2350                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2351                 }
2352                 goto out;
2353         } else if (unlikely(scst_check_sense(cmd)))
2354                 goto out;
2355
2356         if (likely(scsi_status_is_good(cmd->status))) {
2357                 unsigned char type = cmd->dev->handler->type;
2358                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2359                               cmd->cdb[0] == MODE_SENSE_10)) &&
2360                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2361                     (type == TYPE_DISK ||
2362                      type == TYPE_WORM ||
2363                      type == TYPE_MOD ||
2364                      type == TYPE_TAPE)) {
2365                         int32_t length;
2366                         uint8_t *address;
2367                         bool err = false;
2368
2369                         length = scst_get_buf_first(cmd, &address);
2370                         if (length < 0) {
2371                                 PRINT_ERROR("%s", "Unable to get "
2372                                         "MODE_SENSE buffer");
2373                                 scst_set_cmd_error(cmd,
2374                                         SCST_LOAD_SENSE(
2375                                                 scst_sense_hardw_error));
2376                                 err = true;
2377                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2378                                 address[2] |= 0x80;   /* Write Protect*/
2379                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2380                                 address[3] |= 0x80;   /* Write Protect*/
2381                         scst_put_buf(cmd, address);
2382
2383                         if (err)
2384                                 goto out;
2385                 }
2386
2387                 /*
2388                  * Check and clear NormACA option for the device, if necessary,
2389                  * since we don't support ACA
2390                  */
2391                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2392                     /* Std INQUIRY data (no EVPD) */
2393                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2394                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2395                         uint8_t *buffer;
2396                         int buflen;
2397                         bool err = false;
2398
2399                         /* ToDo: all pages ?? */
2400                         buflen = scst_get_buf_first(cmd, &buffer);
2401                         if (buflen > SCST_INQ_BYTE3) {
2402 #ifdef CONFIG_SCST_EXTRACHECKS
2403                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2404                                         PRINT_INFO("NormACA set for device: "
2405                                             "lun=%lld, type 0x%02x. Clear it, "
2406                                             "since it's unsupported.",
2407                                             (long long unsigned int)cmd->lun,
2408                                             buffer[0]);
2409                                 }
2410 #endif
2411                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2412                         } else if (buflen != 0) {
2413                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2414                                     "buffer");
2415                                 scst_set_cmd_error(cmd,
2416                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2417                                 err = true;
2418                         }
2419                         if (buflen > 0)
2420                                 scst_put_buf(cmd, buffer);
2421
2422                         if (err)
2423                                 goto out;
2424                 }
2425
2426                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2427                     (cmd->cdb[0] == MODE_SELECT_10) ||
2428                     (cmd->cdb[0] == LOG_SELECT))) {
2429                         TRACE(TRACE_SCSI,
2430                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2431                                 (long long unsigned int)cmd->lun);
2432                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2433                         goto out;
2434                 }
2435         } else {
2436                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2437                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2438                                         &cmd->tgt_dev->tgt_dev_flags)) {
2439                                 struct scst_tgt_dev *tgt_dev_tmp;
2440                                 struct scst_device *dev = cmd->dev;
2441
2442                                 TRACE(TRACE_SCSI,
2443                                         "Real RESERVE failed lun=%lld, "
2444                                         "status=%x",
2445                                         (long long unsigned int)cmd->lun,
2446                                         cmd->status);
2447                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2448                                         SCST_SENSE_BUFFERSIZE);
2449
2450                                 /* Clearing the reservation */
2451                                 spin_lock_bh(&dev->dev_lock);
2452                                 list_for_each_entry(tgt_dev_tmp,
2453                                                     &dev->dev_tgt_dev_list,
2454                                                     dev_tgt_dev_list_entry) {
2455                                         clear_bit(SCST_TGT_DEV_RESERVED,
2456                                                 &tgt_dev_tmp->tgt_dev_flags);
2457                                 }
2458                                 dev->dev_reserved = 0;
2459                                 spin_unlock_bh(&dev->dev_lock);
2460                         }
2461                 }
2462
2463                 /* Check for MODE PARAMETERS CHANGED UA */
2464                 if ((cmd->dev->scsi_dev != NULL) &&
2465                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2466                     SCST_SENSE_VALID(cmd->sense) &&
2467                     scst_is_ua_sense(cmd->sense) &&
2468                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2469                         TRACE(TRACE_SCSI,
2470                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2471                               (long long unsigned int)cmd->lun);
2472                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2473                         goto out;
2474                 }
2475         }
2476
2477         cmd->state = SCST_CMD_STATE_DEV_DONE;
2478
2479 out:
2480         TRACE_EXIT_RES(res);
2481         return res;
2482 }
2483
2484 static int scst_mode_select_checks(struct scst_cmd *cmd)
2485 {
2486         int res = SCST_CMD_STATE_RES_CONT_SAME;
2487         int atomic = scst_cmd_atomic(cmd);
2488
2489         TRACE_ENTRY();
2490
2491         if (likely(scsi_status_is_good(cmd->status))) {
2492                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2493                     (cmd->cdb[0] == MODE_SELECT_10) ||
2494                     (cmd->cdb[0] == LOG_SELECT))) {
2495                         struct scst_device *dev = cmd->dev;
2496                         if (atomic && (dev->scsi_dev != NULL)) {
2497                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2498                                         "context required");
2499                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2500                                 goto out;
2501                         }
2502
2503                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2504                                 "setting the SELECT UA (lun=%lld)",
2505                                 (long long unsigned int)cmd->lun);
2506
2507                         spin_lock_bh(&dev->dev_lock);
2508                         spin_lock(&scst_temp_UA_lock);
2509                         if (cmd->cdb[0] == LOG_SELECT) {
2510                                 scst_set_sense(scst_temp_UA,
2511                                         sizeof(scst_temp_UA),
2512                                         UNIT_ATTENTION, 0x2a, 0x02);
2513                         } else {
2514                                 scst_set_sense(scst_temp_UA,
2515                                         sizeof(scst_temp_UA),
2516                                         UNIT_ATTENTION, 0x2a, 0x01);
2517                         }
2518                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2519                                 sizeof(scst_temp_UA));
2520                         spin_unlock(&scst_temp_UA_lock);
2521                         spin_unlock_bh(&dev->dev_lock);
2522
2523                         if (dev->scsi_dev != NULL)
2524                                 scst_obtain_device_parameters(dev);
2525                 }
2526         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2527                     SCST_SENSE_VALID(cmd->sense) &&
2528                     scst_is_ua_sense(cmd->sense) &&
2529                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2530                      (cmd->sense[12] == 0x29) /* reset */ ||
2531                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2532                      /* cleared by another ini (just in case) */
2533                      (cmd->sense[12] == 0x2F))) {
2534                 if (atomic) {
2535                         TRACE_DBG("Possible parameters changed UA %x: "
2536                                 "thread context required", cmd->sense[12]);
2537                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2538                         goto out;
2539                 }
2540
2541                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2542                         "(lun %lld): getting new parameters", cmd->sense[12],
2543                         (long long unsigned int)cmd->lun);
2544
2545                 scst_obtain_device_parameters(cmd->dev);
2546         } else
2547                 sBUG();
2548
2549         cmd->state = SCST_CMD_STATE_DEV_DONE;
2550
2551 out:
2552         TRACE_EXIT_HRES(res);
2553         return res;
2554 }
2555
2556 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2557 {
2558         if (likely(cmd->sn_set))
2559                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2560
2561         scst_make_deferred_commands_active(cmd->tgt_dev);
2562 }
2563
2564 static int scst_dev_done(struct scst_cmd **pcmd)
2565 {
2566         int res = SCST_CMD_STATE_RES_CONT_SAME;
2567         struct scst_cmd *cmd = *pcmd;
2568         int state;
2569         struct scst_device *dev = cmd->dev;
2570
2571         TRACE_ENTRY();
2572
2573         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2574
2575         if (likely(!scst_is_cmd_local(cmd)) &&
2576             likely(dev->handler->dev_done != NULL)) {
2577                 int rc;
2578
2579                 if (unlikely(!dev->handler->dev_done_atomic &&
2580                              scst_cmd_atomic(cmd))) {
2581                         /*
2582                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2583                          * optimization.
2584                          */
2585                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2586                               "context, rescheduling", dev->handler->name);
2587                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2588                         goto out;
2589                 }
2590
2591                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2592                       dev->handler->name, cmd);
2593                 rc = dev->handler->dev_done(cmd);
2594                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2595                       dev->handler->name, rc);
2596                 if (rc != SCST_CMD_STATE_DEFAULT)
2597                         state = rc;
2598         }
2599
2600         switch (state) {
2601         case SCST_CMD_STATE_PRE_XMIT_RESP:
2602         case SCST_CMD_STATE_DEV_PARSE:
2603         case SCST_CMD_STATE_PRE_PARSE:
2604         case SCST_CMD_STATE_PREPARE_SPACE:
2605         case SCST_CMD_STATE_RDY_TO_XFER:
2606         case SCST_CMD_STATE_TGT_PRE_EXEC:
2607         case SCST_CMD_STATE_SEND_FOR_EXEC:
2608         case SCST_CMD_STATE_LOCAL_EXEC:
2609         case SCST_CMD_STATE_REAL_EXEC:
2610         case SCST_CMD_STATE_PRE_DEV_DONE:
2611         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2612         case SCST_CMD_STATE_DEV_DONE:
2613         case SCST_CMD_STATE_XMIT_RESP:
2614         case SCST_CMD_STATE_FINISHED:
2615                 cmd->state = state;
2616                 break;
2617
2618         case SCST_CMD_STATE_NEED_THREAD_CTX:
2619                 TRACE_DBG("Dev handler %s dev_done() requested "
2620                       "thread context, rescheduling",
2621                       dev->handler->name);
2622                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2623                 break;
2624
2625         default:
2626                 if (state >= 0) {
2627                         PRINT_ERROR("Dev handler %s dev_done() returned "
2628                                 "invalid cmd state %d",
2629                                 dev->handler->name, state);
2630                 } else {
2631                         PRINT_ERROR("Dev handler %s dev_done() returned "
2632                                 "error %d", dev->handler->name,
2633                                 state);
2634                 }
2635                 scst_set_cmd_error(cmd,
2636                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2637                 scst_set_cmd_abnormal_done_state(cmd);
2638                 break;
2639         }
2640
2641         if (cmd->needs_unblocking)
2642                 scst_unblock_dev_cmd(cmd);
2643
2644         if (likely(cmd->dec_on_dev_needed))
2645                 scst_dec_on_dev_cmd(cmd);
2646
2647         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2648                 scst_inc_check_expected_sn(cmd);
2649
2650         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2651                 *pcmd = scst_complete_request_sense(cmd);
2652
2653 out:
2654         TRACE_EXIT_HRES(res);
2655         return res;
2656 }
2657
2658 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2659 {
2660         int res;
2661
2662         TRACE_ENTRY();
2663
2664 #ifdef CONFIG_SCST_DEBUG_TM
2665         if (cmd->tm_dbg_delayed &&
2666                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2667                 if (scst_cmd_atomic(cmd)) {
2668                         TRACE_MGMT_DBG("%s",
2669                                 "DEBUG_TM delayed cmd needs a thread");
2670                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2671                         return res;
2672                 }
2673                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2674                         cmd, cmd->tag);
2675                 schedule_timeout_uninterruptible(HZ);
2676         }
2677 #endif
2678
2679         if (likely(cmd->tgt_dev != NULL)) {
2680                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2681                 atomic_dec(&cmd->dev->dev_cmd_count);
2682                 /* If expected values not set, expected direction is UNKNOWN */
2683                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2684                         atomic_dec(&cmd->dev->write_cmd_count);
2685
2686                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2687                         scst_on_hq_cmd_response(cmd);
2688
2689                 if (unlikely(!cmd->sent_for_exec)) {
2690                         TRACE_SN("cmd %p was not sent to mid-lev"
2691                                 " (sn %ld, set %d)",
2692                                 cmd, cmd->sn, cmd->sn_set);
2693                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2694                         cmd->sent_for_exec = 1;
2695                 }
2696         }
2697
2698         /*
2699          * If we don't remove cmd from the search list here, before
2700          * submitting it for transmittion, we will have a race, when for
2701          * some reason cmd's release is delayed after transmittion and
2702          * initiator sends cmd with the same tag => it is possible that
2703          * a wrong cmd will be found by find() functions.
2704          */
2705         spin_lock_irq(&cmd->sess->sess_list_lock);
2706         list_del(&cmd->search_cmd_list_entry);
2707         spin_unlock_irq(&cmd->sess->sess_list_lock);
2708
2709         cmd->done = 1;
2710         smp_mb(); /* to sync with scst_abort_cmd() */
2711
2712         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2713                 scst_xmit_process_aborted_cmd(cmd);
2714
2715         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2716                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2717                                 " skipping",
2718                                 cmd, (long long unsigned int)cmd->tag);
2719                 cmd->state = SCST_CMD_STATE_FINISHED;
2720                 res = SCST_CMD_STATE_RES_CONT_SAME;
2721                 goto out;
2722         }
2723
2724         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2725             (cmd->data_direction == SCST_DATA_READ))
2726                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2727
2728         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2729         res = SCST_CMD_STATE_RES_CONT_SAME;
2730
2731 out:
2732 #ifdef CONFIG_SCST_MEASURE_LATENCY
2733         {
2734                 struct timespec ts;
2735                 uint64_t finish, scst_time, proc_time;
2736                 struct scst_session *sess = cmd->sess;
2737
2738                 getnstimeofday(&ts);
2739                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2740
2741                 spin_lock_bh(&sess->meas_lock);
2742
2743                 scst_time = cmd->pre_exec_finish - cmd->start;
2744                 scst_time += finish - cmd->post_exec_start;
2745                 proc_time = finish - cmd->start;
2746
2747                 sess->scst_time += scst_time;
2748                 sess->processing_time += proc_time;
2749                 sess->processed_cmds++;
2750
2751                 spin_unlock_bh(&sess->meas_lock);
2752
2753                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2754                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2755                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2756                         proc_time);
2757         }
2758 #endif
2759         TRACE_EXIT_HRES(res);
2760         return res;
2761 }
2762
2763 static int scst_xmit_response(struct scst_cmd *cmd)
2764 {
2765         int res, rc;
2766
2767         TRACE_ENTRY();
2768
2769         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2770                      scst_cmd_atomic(cmd))) {
2771                 /*
2772                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2773                  * optimization.
2774                  */
2775                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2776                               "context, rescheduling", cmd->tgtt->name);
2777                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2778                 goto out;
2779         }
2780
2781         while (1) {
2782                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2783
2784                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2785                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2786
2787                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2788
2789 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2790                 if (cmd->tgt_sg) {
2791                         int i;
2792                         struct scatterlist *sg = cmd->tgt_sg;
2793                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2794                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2795                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2796                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2797                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2798                                         sg_virt(&sg[i]), sg[i].length);
2799                         }
2800                 }
2801 #endif
2802
2803 #ifdef CONFIG_SCST_DEBUG_RETRY
2804                 if (((scst_random() % 100) == 77))
2805                         rc = SCST_TGT_RES_QUEUE_FULL;
2806                 else
2807 #endif
2808                         rc = cmd->tgtt->xmit_response(cmd);
2809                 TRACE_DBG("xmit_response() returned %d", rc);
2810
2811                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2812                         goto out;
2813
2814                 /* Restore the previous state */
2815                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2816
2817                 switch (rc) {
2818                 case SCST_TGT_RES_QUEUE_FULL:
2819                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2820                                 break;
2821                         else
2822                                 continue;
2823
2824                 case SCST_TGT_RES_NEED_THREAD_CTX:
2825                         TRACE_DBG("Target driver %s xmit_response() "
2826                               "requested thread context, rescheduling",
2827                               cmd->tgtt->name);
2828                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2829                         break;
2830
2831                 default:
2832                         goto out_error;
2833                 }
2834                 break;
2835         }
2836
2837 out:
2838         /* Caution: cmd can be already dead here */
2839         TRACE_EXIT_HRES(res);
2840         return res;
2841
2842 out_error:
2843         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2844                 PRINT_ERROR("Target driver %s xmit_response() returned "
2845                         "fatal error", cmd->tgtt->name);
2846         } else {
2847                 PRINT_ERROR("Target driver %s xmit_response() returned "
2848                         "invalid value %d", cmd->tgtt->name, rc);
2849         }
2850         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2851         cmd->state = SCST_CMD_STATE_FINISHED;
2852         res = SCST_CMD_STATE_RES_CONT_SAME;
2853         goto out;
2854 }
2855
2856 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2857         enum scst_exec_context pref_context)
2858 {
2859         TRACE_ENTRY();
2860
2861         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2862
2863         cmd->state = SCST_CMD_STATE_FINISHED;
2864         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2865
2866         TRACE_EXIT();
2867         return;
2868 }
2869 EXPORT_SYMBOL(scst_tgt_cmd_done);
2870
2871 static int scst_finish_cmd(struct scst_cmd *cmd)
2872 {
2873         int res;
2874
2875         TRACE_ENTRY();
2876
2877         atomic_dec(&cmd->sess->sess_cmd_count);
2878
2879         cmd->finished = 1;
2880         smp_mb(); /* to sync with scst_abort_cmd() */
2881
2882         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2883                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2884                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2885                         atomic_read(&scst_cmd_count));
2886
2887                 scst_finish_cmd_mgmt(cmd);
2888         }
2889
2890         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2891                 if ((cmd->tgt_dev != NULL) &&
2892                     scst_is_ua_sense(cmd->sense)) {
2893                         /* This UA delivery failed, so requeue it */
2894                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2895                                 "%p", cmd);
2896                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2897                                         SCST_SENSE_BUFFERSIZE, 1);
2898                 }
2899         }
2900
2901         __scst_cmd_put(cmd);
2902
2903         res = SCST_CMD_STATE_RES_CONT_NEXT;
2904
2905         TRACE_EXIT_HRES(res);
2906         return res;
2907 }
2908
2909 /*
2910  * No locks, but it must be externally serialized (see comment for
2911  * scst_cmd_init_done() in scst.h)
2912  */
2913 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2914 {
2915         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2916         unsigned long flags;
2917
2918         TRACE_ENTRY();
2919
2920         if (scst_is_implicit_hq(cmd)) {
2921                 TRACE_SN("Implicit HQ cmd %p", cmd);
2922                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2923         }
2924
2925         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2926
2927         /* Optimized for lockless fast path */
2928
2929         scst_check_debug_sn(cmd);
2930
2931         if (cmd->dev->queue_alg ==
2932                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2933                 /*
2934                  * Not the best way, but well enough until there will be a
2935                  * possibility to specify queue type during pass-through
2936                  * commands submission.
2937                  */
2938                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2939         }
2940
2941         switch (cmd->queue_type) {
2942         case SCST_CMD_QUEUE_SIMPLE:
2943         case SCST_CMD_QUEUE_UNTAGGED:
2944 #if 1 /* temporary, ToDo */
2945                 if (scst_cmd_is_expected_set(cmd)) {
2946                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2947                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2948                                 goto ordered;
2949                 } else
2950                         goto ordered;
2951 #endif
2952                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2953                         /*
2954                          * atomic_inc_return() implies memory barrier to sync
2955                          * with scst_inc_expected_sn()
2956                          */
2957                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2958                                 tgt_dev->curr_sn++;
2959                                 TRACE_SN("Incremented curr_sn %ld",
2960                                         tgt_dev->curr_sn);
2961                         }
2962                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2963                         cmd->sn = tgt_dev->curr_sn;
2964
2965                         tgt_dev->prev_cmd_ordered = 0;
2966                 } else {
2967                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2968                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2969                         goto ordered;
2970                 }
2971                 break;
2972
2973         case SCST_CMD_QUEUE_ORDERED:
2974                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
2975 ordered:
2976                 if (!tgt_dev->prev_cmd_ordered) {
2977                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2978                         if (tgt_dev->num_free_sn_slots >= 0) {
2979                                 tgt_dev->num_free_sn_slots--;
2980                                 if (tgt_dev->num_free_sn_slots >= 0) {
2981                                         int i = 0;
2982                                         /* Commands can finish in any order, so
2983                                          * we don't know which slot is empty.
2984                                          */
2985                                         while (1) {
2986                                                 tgt_dev->cur_sn_slot++;
2987                                                 if (tgt_dev->cur_sn_slot ==
2988                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
2989                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2990
2991                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2992                                                         break;
2993
2994                                                 i++;
2995                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2996                                         }
2997                                         TRACE_SN("New cur SN slot %zd",
2998                                                 tgt_dev->cur_sn_slot -
2999                                                 tgt_dev->sn_slots);
3000                                 }
3001                         }
3002                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3003                 }
3004                 tgt_dev->prev_cmd_ordered = 1;
3005                 tgt_dev->curr_sn++;
3006                 cmd->sn = tgt_dev->curr_sn;
3007                 break;
3008
3009         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3010                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3011                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3012                 tgt_dev->hq_cmd_count++;
3013                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3014                 cmd->hq_cmd_inced = 1;
3015                 goto out;
3016
3017         default:
3018                 sBUG();
3019         }
3020
3021         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3022                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3023                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3024                 atomic_read(tgt_dev->cur_sn_slot),
3025                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3026                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3027
3028         cmd->sn_set = 1;
3029
3030 out:
3031         TRACE_EXIT();
3032         return;
3033 }
3034
3035 /*
3036  * Returns 0 on success, > 0 when we need to wait for unblock,
3037  * < 0 if there is no device (lun) or device type handler.
3038  *
3039  * No locks, but might be on IRQ, protection is done by the
3040  * suspended activity.
3041  */
3042 static int scst_translate_lun(struct scst_cmd *cmd)
3043 {
3044         struct scst_tgt_dev *tgt_dev = NULL;
3045         int res;
3046
3047         TRACE_ENTRY();
3048
3049         __scst_get(1);
3050
3051         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3052                 struct list_head *sess_tgt_dev_list_head =
3053                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3054                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3055                         (long long unsigned int)cmd->lun);
3056                 res = -1;
3057                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3058                                 sess_tgt_dev_list_entry) {
3059                         if (tgt_dev->lun == cmd->lun) {
3060                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3061
3062                                 if (unlikely(tgt_dev->dev->handler ==
3063                                                 &scst_null_devtype)) {
3064                                         PRINT_INFO("Dev handler for device "
3065                                           "%lld is NULL, the device will not "
3066                                           "be visible remotely",
3067                                            (long long unsigned int)cmd->lun);
3068                                         break;
3069                                 }
3070
3071                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3072                                 cmd->tgt_dev = tgt_dev;
3073                                 cmd->dev = tgt_dev->dev;
3074
3075                                 res = 0;
3076                                 break;
3077                         }
3078                 }
3079                 if (res != 0) {
3080                         TRACE(TRACE_MINOR,
3081                                 "tgt_dev for lun %lld not found, command to "
3082                                 "unexisting LU?",
3083                                 (long long unsigned int)cmd->lun);
3084                         __scst_put();
3085                 }
3086         } else {
3087                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3088                 __scst_put();
3089                 res = 1;
3090         }
3091
3092         TRACE_EXIT_RES(res);
3093         return res;
3094 }
3095
3096 /*
3097  * No locks, but might be on IRQ
3098  *
3099  * Returns 0 on success, > 0 when we need to wait for unblock,
3100  * < 0 if there is no device (lun) or device type handler.
3101  */
3102 static int __scst_init_cmd(struct scst_cmd *cmd)
3103 {
3104         int res = 0;
3105
3106         TRACE_ENTRY();
3107
3108         res = scst_translate_lun(cmd);
3109         if (likely(res == 0)) {
3110                 int cnt;
3111                 bool failure = false;
3112
3113                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3114
3115                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3116                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3117                         TRACE(TRACE_MGMT_MINOR,
3118                                 "Too many pending commands (%d) in "
3119                                 "session, returning BUSY to initiator \"%s\"",
3120                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3121                                   "Anonymous" : cmd->sess->initiator_name);
3122                         failure = true;
3123                 }
3124
3125                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3126                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3127                         if (!failure) {
3128                                 TRACE(TRACE_MGMT_MINOR,
3129                                         "Too many pending device "
3130                                         "commands (%d), returning BUSY to "
3131                                         "initiator \"%s\"", cnt,
3132                                         (cmd->sess->initiator_name[0] == '\0') ?
3133                                                 "Anonymous" :
3134                                                 cmd->sess->initiator_name);
3135                                 failure = true;
3136                         }
3137                 }
3138
3139                 /* If expected values not set, expected direction is UNKNOWN */
3140                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3141                         atomic_inc(&cmd->dev->write_cmd_count);
3142
3143                 if (unlikely(failure))
3144                         goto out_busy;
3145
3146                 if (!cmd->set_sn_on_restart_cmd)
3147                         scst_cmd_set_sn(cmd);
3148         } else if (res < 0) {
3149                 TRACE_DBG("Finishing cmd %p", cmd);
3150                 scst_set_cmd_error(cmd,
3151                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3152                 scst_set_cmd_abnormal_done_state(cmd);
3153         } else
3154                 goto out;
3155
3156 out:
3157         TRACE_EXIT_RES(res);
3158         return res;
3159
3160 out_busy:
3161         scst_set_busy(cmd);
3162         scst_set_cmd_abnormal_done_state(cmd);
3163         goto out;
3164 }
3165
3166 /* Called under scst_init_lock and IRQs disabled */
3167 static void scst_do_job_init(void)
3168 {
3169         struct scst_cmd *cmd;
3170         int susp;
3171
3172         TRACE_ENTRY();
3173
3174 restart:
3175         /*
3176          * There is no need for read barrier here, because we don't care where
3177          * this check will be done.
3178          */
3179         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3180         if (scst_init_poll_cnt > 0)
3181                 scst_init_poll_cnt--;
3182
3183         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3184                 int rc;
3185                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3186                         continue;
3187                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3188                         spin_unlock_irq(&scst_init_lock);
3189                         rc = __scst_init_cmd(cmd);
3190                         spin_lock_irq(&scst_init_lock);
3191                         if (rc > 0) {
3192                                 TRACE_MGMT_DBG("%s",
3193                                         "FLAG SUSPENDED set, restarting");
3194                                 goto restart;
3195                         }
3196                 } else {
3197                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3198                                        cmd, (long long unsigned int)cmd->tag);
3199                         scst_set_cmd_abnormal_done_state(cmd);
3200                 }
3201
3202                 /*
3203                  * Deleting cmd from init cmd list after __scst_init_cmd()
3204                  * is necessary to keep the check in scst_init_cmd() correct
3205                  * to preserve the commands order.
3206                  *
3207                  * We don't care about the race, when init cmd list is empty
3208                  * and one command detected that it just was not empty, so
3209                  * it's inserting to it, but another command at the same time
3210                  * seeing init cmd list empty and goes directly, because it
3211                  * could affect only commands from the same initiator to the
3212                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3213                  * in case of simultaneous such calls anyway.
3214                  */
3215                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3216                 smp_wmb();
3217                 list_del(&cmd->cmd_list_entry);
3218                 spin_unlock(&scst_init_lock);
3219
3220                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3221                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3222                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3223                         list_add(&cmd->cmd_list_entry,
3224                                 &cmd->cmd_lists->active_cmd_list);
3225                 else
3226                         list_add_tail(&cmd->cmd_list_entry,
3227                                 &cmd->cmd_lists->active_cmd_list);
3228                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3229                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3230
3231                 spin_lock(&scst_init_lock);
3232                 goto restart;
3233         }
3234
3235         /* It isn't really needed, but let's keep it */
3236         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3237                 goto restart;
3238
3239         TRACE_EXIT();
3240         return;
3241 }
3242
3243 static inline int test_init_cmd_list(void)
3244 {
3245         int res = (!list_empty(&scst_init_cmd_list) &&
3246                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3247                   unlikely(kthread_should_stop()) ||
3248                   (scst_init_poll_cnt > 0);
3249         return res;
3250 }
3251
3252 int scst_init_cmd_thread(void *arg)
3253 {
3254         TRACE_ENTRY();
3255
3256         PRINT_INFO("Init thread started, PID %d", current->pid);
3257
3258         current->flags |= PF_NOFREEZE;
3259
3260         set_user_nice(current, -10);
3261
3262         spin_lock_irq(&scst_init_lock);
3263         while (!kthread_should_stop()) {
3264                 wait_queue_t wait;
3265                 init_waitqueue_entry(&wait, current);
3266
3267                 if (!test_init_cmd_list()) {
3268                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3269                                                  &wait);
3270                         for (;;) {
3271                                 set_current_state(TASK_INTERRUPTIBLE);
3272                                 if (test_init_cmd_list())
3273                                         break;
3274                                 spin_unlock_irq(&scst_init_lock);
3275                                 schedule();
3276                                 spin_lock_irq(&scst_init_lock);
3277                         }
3278                         set_current_state(TASK_RUNNING);
3279                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3280                 }
3281                 scst_do_job_init();
3282         }
3283         spin_unlock_irq(&scst_init_lock);
3284
3285         /*
3286          * If kthread_should_stop() is true, we are guaranteed to be
3287          * on the module unload, so scst_init_cmd_list must be empty.
3288          */
3289         sBUG_ON(!list_empty(&scst_init_cmd_list));
3290
3291         PRINT_INFO("Init thread PID %d finished", current->pid);
3292
3293         TRACE_EXIT();
3294         return 0;
3295 }
3296
3297 /* Called with no locks held */
3298 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3299 {
3300         int res;
3301
3302         TRACE_ENTRY();
3303
3304         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3305
3306         cmd->atomic = atomic;
3307
3308         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3309
3310         do {
3311                 switch (cmd->state) {
3312                 case SCST_CMD_STATE_PRE_PARSE:
3313                         res = scst_pre_parse(cmd);
3314                         EXTRACHECKS_BUG_ON(res ==
3315                                 SCST_CMD_STATE_RES_NEED_THREAD);
3316                         break;
3317
3318                 case SCST_CMD_STATE_DEV_PARSE:
3319                         res = scst_parse_cmd(cmd);
3320                         break;
3321
3322                 case SCST_CMD_STATE_PREPARE_SPACE:
3323                         res = scst_prepare_space(cmd);
3324                         break;
3325
3326                 case SCST_CMD_STATE_RDY_TO_XFER:
3327                         res = scst_rdy_to_xfer(cmd);
3328                         break;
3329
3330                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3331                         res = scst_tgt_pre_exec(cmd);
3332                         break;
3333
3334                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3335                         if (tm_dbg_check_cmd(cmd) != 0) {
3336                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3337                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3338                                         "because of TM DBG delay", cmd,
3339                                         (long long unsigned int)cmd->tag);
3340                                 break;
3341                         }
3342                         res = scst_send_for_exec(&cmd);
3343                         /*
3344                          * !! At this point cmd, sess & tgt_dev can already be
3345                          * freed !!
3346                          */
3347                         break;
3348
3349                 case SCST_CMD_STATE_LOCAL_EXEC:
3350                         res = scst_local_exec(cmd);
3351                         /*
3352                          * !! At this point cmd, sess & tgt_dev can already be
3353                          * freed !!
3354                          */
3355                         break;
3356
3357                 case SCST_CMD_STATE_REAL_EXEC:
3358                         res = scst_real_exec(cmd);
3359                         /*
3360                          * !! At this point cmd, sess & tgt_dev can already be
3361                          * freed !!
3362                          */
3363                         break;
3364
3365                 case SCST_CMD_STATE_PRE_DEV_DONE:
3366                         res = scst_pre_dev_done(cmd);
3367                         EXTRACHECKS_BUG_ON(res ==
3368                                 SCST_CMD_STATE_RES_NEED_THREAD);
3369                         break;
3370
3371                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3372                         res = scst_mode_select_checks(cmd);
3373                         break;
3374
3375                 case SCST_CMD_STATE_DEV_DONE:
3376                         res = scst_dev_done(&cmd);
3377                         break;
3378
3379                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3380                         res = scst_pre_xmit_response(cmd);
3381                         EXTRACHECKS_BUG_ON(res ==
3382                                 SCST_CMD_STATE_RES_NEED_THREAD);
3383                         break;
3384
3385                 case SCST_CMD_STATE_XMIT_RESP:
3386                         res = scst_xmit_response(cmd);
3387                         break;
3388
3389                 case SCST_CMD_STATE_FINISHED:
3390                         res = scst_finish_cmd(cmd);
3391                         EXTRACHECKS_BUG_ON(res ==
3392                                 SCST_CMD_STATE_RES_NEED_THREAD);
3393                         break;
3394
3395                 default:
3396                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3397                                 "be", cmd, cmd->state);
3398                         sBUG();
3399                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3400                         break;
3401                 }
3402         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3403
3404         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3405                 /* None */
3406         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3407                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3408                 switch (cmd->state) {
3409                 case SCST_CMD_STATE_PRE_PARSE:
3410                 case SCST_CMD_STATE_DEV_PARSE:
3411                 case SCST_CMD_STATE_PREPARE_SPACE:
3412                 case SCST_CMD_STATE_RDY_TO_XFER:
3413                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3414                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3415                 case SCST_CMD_STATE_LOCAL_EXEC:
3416                 case SCST_CMD_STATE_REAL_EXEC:
3417                 case SCST_CMD_STATE_PRE_DEV_DONE:
3418                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3419                 case SCST_CMD_STATE_DEV_DONE:
3420                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3421                 case SCST_CMD_STATE_XMIT_RESP:
3422                 case SCST_CMD_STATE_FINISHED:
3423                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3424                                   cmd);
3425                         list_add(&cmd->cmd_list_entry,
3426                                 &cmd->cmd_lists->active_cmd_list);
3427                         break;
3428 #ifdef CONFIG_SCST_EXTRACHECKS
3429                 /* not very valid commands */
3430                 case SCST_CMD_STATE_DEFAULT:
3431                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3432                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3433                                 cmd->state);
3434                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3435                         sBUG();
3436                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3437                         break;
3438 #endif
3439                 default:
3440                         break;
3441                 }
3442                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3443                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3444         } else
3445                 sBUG();
3446
3447         TRACE_EXIT();
3448         return;
3449 }
3450 EXPORT_SYMBOL(scst_process_active_cmd);
3451
3452 /* Called under cmd_list_lock and IRQs disabled */
3453 static void scst_do_job_active(struct list_head *cmd_list,
3454         spinlock_t *cmd_list_lock, bool atomic)
3455 {
3456         TRACE_ENTRY();
3457
3458         while (!list_empty(cmd_list)) {
3459                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3460                                         cmd_list_entry);
3461                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3462                 list_del(&cmd->cmd_list_entry);
3463                 spin_unlock_irq(cmd_list_lock);
3464                 scst_process_active_cmd(cmd, atomic);
3465                 spin_lock_irq(cmd_list_lock);
3466         }
3467
3468         TRACE_EXIT();
3469         return;
3470 }
3471
3472 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3473 {
3474         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3475             unlikely(kthread_should_stop()) ||
3476             tm_dbg_is_release();
3477         return res;
3478 }
3479
3480 int scst_cmd_thread(void *arg)
3481 {
3482         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3483
3484         TRACE_ENTRY();
3485
3486         PRINT_INFO("Processing thread started, PID %d", current->pid);
3487
3488 #if 0
3489         set_user_nice(current, 10);
3490 #endif
3491         current->flags |= PF_NOFREEZE;
3492
3493         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3494         while (!kthread_should_stop()) {
3495                 wait_queue_t wait;
3496                 init_waitqueue_entry(&wait, current);
3497
3498                 if (!test_cmd_lists(p_cmd_lists)) {
3499                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3500                                 &wait);
3501                         for (;;) {
3502   &nbs