The patch below fixes the following:
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38
39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
40 {
41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
42         unsigned long flags;
43
44         spin_lock_irqsave(&t->tasklet_lock, flags);
45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
46                 smp_processor_id());
47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
49
50         tasklet_schedule(&t->tasklet);
51 }
52
53 /*
54  * Must not be called in parallel with scst_unregister_session_ex() for the
55  * same sess
56  */
57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
58                              const uint8_t *lun, int lun_len,
59                              const uint8_t *cdb, int cdb_len, int atomic)
60 {
61         struct scst_cmd *cmd;
62
63         TRACE_ENTRY();
64
65 #ifdef CONFIG_SCST_EXTRACHECKS
66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
67                 PRINT_CRIT_ERROR("%s",
68                         "New cmd while shutting down the session");
69                 sBUG();
70         }
71 #endif
72
73         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
74         if (cmd == NULL)
75                 goto out;
76
77         cmd->sess = sess;
78         cmd->tgt = sess->tgt;
79         cmd->tgtt = sess->tgt->tgtt;
80
81         /*
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100 EXPORT_SYMBOL(scst_rx_cmd);
101
102 static int scst_init_cmd(struct scst_cmd *cmd, int context)
103 {
104         int rc;
105
106         TRACE_ENTRY();
107
108         /* See the comment in scst_do_job_init() */
109         if (unlikely(!list_empty(&scst_init_cmd_list))) {
110                 TRACE_MGMT_DBG("%s", "init cmd list busy");
111                 goto out_redirect;
112         }
113         /*
114          * Memory barrier isn't necessary here, because CPU appears to
115          * be self-consistent
116          */
117
118         rc = __scst_init_cmd(cmd);
119         if (unlikely(rc > 0))
120                 goto out_redirect;
121         else if (unlikely(rc != 0))
122                 goto out;
123
124         /* Small context optimization */
125         if (((context == SCST_CONTEXT_TASKLET) ||
126              (context == SCST_CONTEXT_DIRECT_ATOMIC)) &&
127             scst_cmd_is_expected_set(cmd)) {
128                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
129                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
130                                         &cmd->tgt_dev->tgt_dev_flags))
131                                 context = SCST_CONTEXT_THREAD;
132                 } else {
133                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
134                                         &cmd->tgt_dev->tgt_dev_flags))
135                                 context = SCST_CONTEXT_THREAD;
136                 }
137         }
138
139 out:
140         TRACE_EXIT_RES(context);
141         return context;
142
143 out_redirect:
144         if (cmd->preprocessing_only) {
145                 /*
146                  * Poor man solution for single threaded targets, where
147                  * blocking receiver at least sometimes means blocking all.
148                  */
149                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
150                 scst_set_busy(cmd);
151                 scst_set_cmd_abnormal_done_state(cmd);
152                 /* Keep initiator away from too many BUSY commands */
153                 if (!in_interrupt() && !in_atomic())
154                         msleep(50);
155                 else
156                         WARN_ON_ONCE(1);
157         } else {
158                 unsigned long flags;
159                 spin_lock_irqsave(&scst_init_lock, flags);
160                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
161                         "%d)", cmd, atomic_read(&scst_cmd_count));
162                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
163                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
164                         scst_init_poll_cnt++;
165                 spin_unlock_irqrestore(&scst_init_lock, flags);
166                 wake_up(&scst_init_cmd_list_waitQ);
167                 context = -1;
168         }
169         goto out;
170 }
171
172 #ifdef CONFIG_SCST_MEASURE_LATENCY
173 static inline uint64_t scst_sec_to_nsec(time_t sec)
174 {
175         return (uint64_t)sec * 1000000000;
176 }
177 #endif
178
179 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
180 {
181         unsigned long flags;
182         struct scst_session *sess = cmd->sess;
183
184         TRACE_ENTRY();
185
186 #ifdef CONFIG_SCST_MEASURE_LATENCY
187         {
188                 struct timespec ts;
189                 getnstimeofday(&ts);
190                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
191                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
192                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
193                         ts.tv_nsec);
194         }
195 #endif
196
197         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
198         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d",
199               (long long unsigned int)cmd->tag,
200               (long long unsigned int)cmd->lun,
201               cmd->cdb_len);
202         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
203                 cmd->cdb, cmd->cdb_len);
204
205 #ifdef CONFIG_SCST_EXTRACHECKS
206         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
207                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
208                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
209                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
210                         cmd->tgtt->name);
211                 pref_context = SCST_CONTEXT_TASKLET;
212         }
213 #endif
214
215         atomic_inc(&sess->sess_cmd_count);
216
217         spin_lock_irqsave(&sess->sess_list_lock, flags);
218
219         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
220                 /*
221                  * We have to always keep command in the search list from the
222                  * very beginning, because otherwise it can be missed during
223                  * TM processing. This check is needed because there might be
224                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
225                  */
226                 if (cmd->search_cmd_list_entry.next == NULL)
227                         list_add_tail(&cmd->search_cmd_list_entry,
228                                 &sess->search_cmd_list);
229                 switch (sess->init_phase) {
230                 case SCST_SESS_IPH_SUCCESS:
231                         break;
232                 case SCST_SESS_IPH_INITING:
233                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
234                                   cmd);
235                         list_add_tail(&cmd->cmd_list_entry,
236                                 &sess->init_deferred_cmd_list);
237                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
238                         goto out;
239                 case SCST_SESS_IPH_FAILED:
240                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
241                         scst_set_busy(cmd);
242                         scst_set_cmd_abnormal_done_state(cmd);
243                         goto active;
244                 default:
245                         sBUG();
246                 }
247         } else
248                 list_add_tail(&cmd->search_cmd_list_entry,
249                               &sess->search_cmd_list);
250
251         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
252
253         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
254                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
255                 scst_set_cmd_error(cmd,
256                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
257                 scst_set_cmd_abnormal_done_state(cmd);
258                 goto active;
259         }
260
261         if (unlikely(cmd->cdb_len == 0)) {
262                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
263                 scst_set_cmd_error(cmd,
264                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
265                 scst_set_cmd_abnormal_done_state(cmd);
266                 goto active;
267         }
268
269         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
270                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
271                 scst_set_cmd_error(cmd,
272                         SCST_LOAD_SENSE(scst_sense_invalid_message));
273                 scst_set_cmd_abnormal_done_state(cmd);
274                 goto active;
275         }
276
277         cmd->state = SCST_CMD_STATE_INIT;
278         /* cmd must be inited here to preserve the order */
279         pref_context = scst_init_cmd(cmd, pref_context);
280         if (unlikely(pref_context < 0))
281                 goto out;
282
283 active:
284         /* Here cmd must not be in any cmd list, no locks */
285         switch (pref_context) {
286         case SCST_CONTEXT_TASKLET:
287                 scst_schedule_tasklet(cmd);
288                 break;
289
290         case SCST_CONTEXT_DIRECT:
291         case SCST_CONTEXT_DIRECT_ATOMIC:
292                 scst_process_active_cmd(cmd, pref_context);
293                 /* For *NEED_THREAD wake_up() is already done */
294                 break;
295
296         default:
297                 PRINT_ERROR("Context %x is undefined, using the thread one",
298                         pref_context);
299                 /* go through */
300         case SCST_CONTEXT_THREAD:
301                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
302                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
303                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
304                         list_add(&cmd->cmd_list_entry,
305                                 &cmd->cmd_lists->active_cmd_list);
306                 else
307                         list_add_tail(&cmd->cmd_list_entry,
308                                 &cmd->cmd_lists->active_cmd_list);
309                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
310                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
311                 break;
312         }
313
314 out:
315         TRACE_EXIT();
316         return;
317 }
318 EXPORT_SYMBOL(scst_cmd_init_done);
319
320 static int scst_pre_parse(struct scst_cmd *cmd)
321 {
322         int res = SCST_CMD_STATE_RES_CONT_SAME;
323         struct scst_device *dev = cmd->dev;
324         int rc;
325
326         TRACE_ENTRY();
327
328         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
329              (!dev->has_own_order_mgmt &&
330               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
331                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
332
333         /*
334          * Expected transfer data supplied by the SCSI transport via the
335          * target driver are untrusted, so we prefer to fetch them from CDB.
336          * Additionally, not all transports support supplying the expected
337          * transfer data.
338          */
339
340         rc = scst_get_cdb_info(cmd);
341         if (unlikely(rc != 0)) {
342                 if (rc > 0) {
343                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
344                         goto out_xmit;
345                 }
346                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
347                         "Should you update scst_scsi_op_table?",
348                         cmd->cdb[0], dev->handler->name);
349                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
350 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
351                 if (scst_cmd_is_expected_set(cmd)) {
352                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
353                                 "direction %d, transfer_len %d",
354                                 cmd->expected_data_direction,
355                                 cmd->expected_transfer_len);
356                         cmd->data_direction = cmd->expected_data_direction;
357
358                         cmd->bufflen = cmd->expected_transfer_len;
359                         /* Restore (likely) lost CDB length */
360                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
361                         if (cmd->cdb_len == -1) {
362                                 PRINT_ERROR("Unable to get CDB length for "
363                                         "opcode 0x%02x. Returning INVALID "
364                                         "OPCODE", cmd->cdb[0]);
365                                 scst_set_cmd_error(cmd,
366                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
367                                 goto out_xmit;
368                         }
369                 } else {
370                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
371                              "target %s not supplied expected values",
372                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
373                         scst_set_cmd_error(cmd,
374                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
375                         goto out_xmit;
376                 }
377 #else
378                 scst_set_cmd_error(cmd,
379                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
380                 goto out_xmit;
381 #endif
382         } else {
383                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
384                         "set %s), transfer_len=%d (expected len %d), flags=%d",
385                         cmd->op_name, cmd->data_direction,
386                         cmd->expected_data_direction,
387                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
388                         cmd->bufflen, cmd->expected_transfer_len,
389                         cmd->op_flags);
390
391                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
392                         if (scst_cmd_is_expected_set(cmd)) {
393                                 /*
394                                  * Command data length can't be easily
395                                  * determined from the CDB. ToDo, all such
396                                  * commands processing should be fixed. Until
397                                  * it's done, get the length from the supplied
398                                  * expected value, but limit it to some
399                                  * reasonable value (15MB).
400                                  */
401                                 cmd->bufflen = min(cmd->expected_transfer_len,
402                                                         15*1024*1024);
403                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
404                         } else
405                                 cmd->bufflen = 0;
406                 }
407         }
408
409         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
410                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
411                             "(opcode 0x%02x)", cmd->cdb[0]);
412                 scst_set_cmd_error(cmd,
413                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
414                 goto out_xmit;
415         }
416
417         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
418                 PRINT_ERROR("Linked commands are not supported "
419                             "(opcode 0x%02x)", cmd->cdb[0]);
420                 scst_set_cmd_error(cmd,
421                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
422                 goto out_xmit;
423         }
424
425         cmd->state = SCST_CMD_STATE_DEV_PARSE;
426
427 out:
428         TRACE_EXIT_RES(res);
429         return res;
430
431 out_xmit:
432         scst_set_cmd_abnormal_done_state(cmd);
433         res = SCST_CMD_STATE_RES_CONT_SAME;
434         goto out;
435 }
436
437 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
438 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
439 {
440         bool res = false;
441
442         switch (cmd->cdb[0]) {
443         case TEST_UNIT_READY:
444                 /* Crazy VMware people sometimes do TUR with READ direction */
445                 res = true;
446                 break;
447         case VERIFY:
448         case VERIFY_6:
449         case VERIFY_12:
450         case VERIFY_16:
451                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
452                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
453                     (cmd->cdb[1] & BYTCHK) == 0)
454                         res = true;
455                 break;
456         }
457
458         return res;
459 }
460 #endif
461
462 static int scst_parse_cmd(struct scst_cmd *cmd)
463 {
464         int res = SCST_CMD_STATE_RES_CONT_SAME;
465         int state;
466         struct scst_device *dev = cmd->dev;
467         int orig_bufflen = cmd->bufflen;
468
469         TRACE_ENTRY();
470
471         if (likely(!scst_is_cmd_local(cmd))) {
472                 if (unlikely(!dev->handler->parse_atomic &&
473                              scst_cmd_atomic(cmd))) {
474                         /*
475                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
476                          * optimization.
477                          */
478                         TRACE_DBG("Dev handler %s parse() needs thread "
479                                 "context, rescheduling", dev->handler->name);
480                         res = SCST_CMD_STATE_RES_NEED_THREAD;
481                         goto out;
482                 }
483
484                 TRACE_DBG("Calling dev handler %s parse(%p)",
485                       dev->handler->name, cmd);
486                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
487                                 cmd->cdb, cmd->cdb_len);
488                 state = dev->handler->parse(cmd);
489                 /* Caution: cmd can be already dead here */
490                 TRACE_DBG("Dev handler %s parse() returned %d",
491                         dev->handler->name, state);
492
493                 switch (state) {
494                 case SCST_CMD_STATE_NEED_THREAD_CTX:
495                         TRACE_DBG("Dev handler %s parse() requested thread "
496                               "context, rescheduling", dev->handler->name);
497                         res = SCST_CMD_STATE_RES_NEED_THREAD;
498                         goto out;
499
500                 case SCST_CMD_STATE_STOP:
501                         TRACE_DBG("Dev handler %s parse() requested stop "
502                                 "processing", dev->handler->name);
503                         res = SCST_CMD_STATE_RES_CONT_NEXT;
504                         goto out;
505                 }
506
507                 if (state == SCST_CMD_STATE_DEFAULT)
508                         state = SCST_CMD_STATE_PREPARE_SPACE;
509         } else
510                 state = SCST_CMD_STATE_PREPARE_SPACE;
511
512         if (cmd->data_len == -1)
513                 cmd->data_len = cmd->bufflen;
514
515         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
516                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
517                         "is less, than required (size %d)", cmd->bufflen,
518                         orig_bufflen);
519                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
520                 goto out_error;
521         }
522
523         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
524                 goto set_res;
525
526         if (unlikely((cmd->bufflen == 0) &&
527                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
528                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
529                         "(handler %s, target %s)", cmd->cdb[0],
530                         dev->handler->name, cmd->tgtt->name);
531                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
532                 goto out_error;
533         }
534
535 #ifdef CONFIG_SCST_EXTRACHECKS
536         if ((cmd->bufflen != 0) &&
537             ((cmd->data_direction == SCST_DATA_NONE) ||
538              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
539                 PRINT_ERROR("Dev handler %s parse() returned "
540                         "invalid cmd data_direction %d, bufflen %d, state %d "
541                         "or sg %p (opcode 0x%x)", dev->handler->name,
542                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
543                         cmd->cdb[0]);
544                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
545                 goto out_error;
546         }
547 #endif
548
549         if (scst_cmd_is_expected_set(cmd)) {
550 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
551 #       ifdef CONFIG_SCST_EXTRACHECKS
552                 if ((cmd->data_direction != cmd->expected_data_direction) ||
553                     (cmd->bufflen != cmd->expected_transfer_len)) {
554                         PRINT_WARNING("Expected values don't match decoded "
555                                 "ones: data_direction %d, "
556                                 "expected_data_direction %d, "
557                                 "bufflen %d, expected_transfer_len %d",
558                                 cmd->data_direction,
559                                 cmd->expected_data_direction,
560                                 cmd->bufflen, cmd->expected_transfer_len);
561                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
562                 }
563 #       endif
564                 cmd->data_direction = cmd->expected_data_direction;
565                 cmd->bufflen = cmd->expected_transfer_len;
566 #else
567                 if (unlikely(cmd->data_direction !=
568                                 cmd->expected_data_direction)) {
569                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
570                              (cmd->bufflen != 0)) &&
571                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
572                                 PRINT_ERROR("Expected data direction %d for "
573                                         "opcode 0x%02x (handler %s, target %s) "
574                                         "doesn't match "
575                                         "decoded value %d",
576                                         cmd->expected_data_direction,
577                                         cmd->cdb[0], dev->handler->name,
578                                         cmd->tgtt->name, cmd->data_direction);
579                                 PRINT_BUFFER("Failed CDB",
580                                         cmd->cdb, cmd->cdb_len);
581                                 scst_set_cmd_error(cmd,
582                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
583                                 goto out_dev_done;
584                         }
585                 }
586                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
587                         static int repd;
588                         if (repd < 100) {
589                                 /*
590                                  * Intentionally unlocked. Few messages more
591                                  * or less don't matter.
592                                  */
593                                 repd++;
594                                 TRACE(TRACE_MINOR, "Warning: expected transfer "
595                                         "length %d for opcode 0x%02x (handler "
596                                         "%s, target %s) doesn't match decoded "
597                                         "value %d. Faulty initiator (e.g. "
598                                         "VMware is known to be such) or "
599                                         "scst_scsi_op_table should be updated?",
600                                         cmd->expected_transfer_len, cmd->cdb[0],
601                                         dev->handler->name, cmd->tgtt->name,
602                                         cmd->bufflen);
603                                 PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
604                                         cmd->cdb, cmd->cdb_len);
605                         }
606                 }
607 #endif
608         }
609
610         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
611                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
612                         "target %s", cmd->cdb[0], dev->handler->name,
613                         cmd->tgtt->name);
614                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
615                 goto out_error;
616         }
617
618 set_res:
619         switch (state) {
620         case SCST_CMD_STATE_PREPARE_SPACE:
621         case SCST_CMD_STATE_PRE_PARSE:
622         case SCST_CMD_STATE_DEV_PARSE:
623         case SCST_CMD_STATE_RDY_TO_XFER:
624         case SCST_CMD_STATE_TGT_PRE_EXEC:
625         case SCST_CMD_STATE_SEND_FOR_EXEC:
626         case SCST_CMD_STATE_LOCAL_EXEC:
627         case SCST_CMD_STATE_REAL_EXEC:
628         case SCST_CMD_STATE_PRE_DEV_DONE:
629         case SCST_CMD_STATE_DEV_DONE:
630         case SCST_CMD_STATE_PRE_XMIT_RESP:
631         case SCST_CMD_STATE_XMIT_RESP:
632         case SCST_CMD_STATE_FINISHED:
633                 cmd->state = state;
634                 res = SCST_CMD_STATE_RES_CONT_SAME;
635                 break;
636
637         default:
638                 if (state >= 0) {
639                         PRINT_ERROR("Dev handler %s parse() returned "
640                              "invalid cmd state %d (opcode %d)",
641                              dev->handler->name, state, cmd->cdb[0]);
642                 } else {
643                         PRINT_ERROR("Dev handler %s parse() returned "
644                                 "error %d (opcode %d)", dev->handler->name,
645                                 state, cmd->cdb[0]);
646                 }
647                 goto out_error;
648         }
649
650         if (cmd->resp_data_len == -1) {
651                 if (cmd->data_direction == SCST_DATA_READ)
652                         cmd->resp_data_len = cmd->bufflen;
653                 else
654                          cmd->resp_data_len = 0;
655         }
656
657 out:
658         TRACE_EXIT_HRES(res);
659         return res;
660
661 out_error:
662         /* dev_done() will be called as part of the regular cmd's finish */
663         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
664
665 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
666 out_dev_done:
667 #endif
668         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
669         res = SCST_CMD_STATE_RES_CONT_SAME;
670         goto out;
671 }
672
673 static int scst_prepare_space(struct scst_cmd *cmd)
674 {
675         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
676
677         TRACE_ENTRY();
678
679         if (cmd->data_direction == SCST_DATA_NONE)
680                 goto prep_done;
681
682         if (cmd->data_buf_tgt_alloc) {
683                 int orig_bufflen = cmd->bufflen;
684
685                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
686
687                 r = cmd->tgtt->alloc_data_buf(cmd);
688                 if (r > 0)
689                         goto alloc;
690                 else if (r == 0) {
691                         if (unlikely(cmd->bufflen == 0)) {
692                                 /* See comment in scst_alloc_space() */
693                                 if (cmd->sg == NULL)
694                                         goto alloc;
695                         }
696                         cmd->data_buf_alloced = 1;
697                         if (unlikely(orig_bufflen < cmd->bufflen)) {
698                                 PRINT_ERROR("Target driver allocated data "
699                                         "buffer (size %d), is less, than "
700                                         "required (size %d)", orig_bufflen,
701                                         cmd->bufflen);
702                                 goto out_error;
703                         }
704                         TRACE_MEM("%s", "data_buf_alloced, returning");
705                 }
706                 goto check;
707         }
708
709 alloc:
710         if (!cmd->data_buf_alloced)
711                 r = scst_alloc_space(cmd);
712         else
713                 TRACE_MEM("%s", "data_buf_alloced set, returning");
714
715 check:
716         if (r != 0) {
717                 if (scst_cmd_atomic(cmd)) {
718                         TRACE_MEM("%s", "Atomic memory allocation failed, "
719                               "rescheduling to the thread");
720                         res = SCST_CMD_STATE_RES_NEED_THREAD;
721                         goto out;
722                 } else
723                         goto out_no_space;
724         }
725
726 prep_done:
727         if (cmd->preprocessing_only) {
728                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
729                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
730                                 "cmd %p", cmd);
731                         scst_set_cmd_abnormal_done_state(cmd);
732                         res = SCST_CMD_STATE_RES_CONT_SAME;
733                         goto out;
734                 }
735
736                 res = SCST_CMD_STATE_RES_CONT_NEXT;
737                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
738
739                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
740                 cmd->tgtt->preprocessing_done(cmd);
741                 TRACE_DBG("%s", "preprocessing_done() returned");
742                 goto out;
743
744         }
745
746         switch (cmd->data_direction) {
747         case SCST_DATA_WRITE:
748                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
749                 break;
750
751         default:
752                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
753                 break;
754         }
755
756 out:
757         TRACE_EXIT_HRES(res);
758         return res;
759
760 out_no_space:
761         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
762                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
763         scst_set_busy(cmd);
764         scst_set_cmd_abnormal_done_state(cmd);
765         res = SCST_CMD_STATE_RES_CONT_SAME;
766         goto out;
767
768 out_error:
769         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
770         scst_set_cmd_abnormal_done_state(cmd);
771         res = SCST_CMD_STATE_RES_CONT_SAME;
772         goto out;
773 }
774
775 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
776 {
777         TRACE_ENTRY();
778
779         TRACE_DBG("Preferred context: %d", pref_context);
780         TRACE_DBG("tag=%llu, status=%#x",
781                   (long long unsigned int)scst_cmd_get_tag(cmd),
782                   status);
783
784 #ifdef CONFIG_SCST_EXTRACHECKS
785         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
786                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
787                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
788                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
789                         cmd->tgtt->name);
790                 pref_context = SCST_CONTEXT_TASKLET;
791         }
792 #endif
793
794         switch (status) {
795         case SCST_PREPROCESS_STATUS_SUCCESS:
796                 switch (cmd->data_direction) {
797                 case SCST_DATA_WRITE:
798                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
799                         break;
800                 default:
801                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
802                         break;
803                 }
804                 if (cmd->set_sn_on_restart_cmd)
805                         scst_cmd_set_sn(cmd);
806                 /* Small context optimization */
807                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
808                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
809                         if (cmd->data_direction == SCST_DATA_WRITE) {
810                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
811                                                 &cmd->tgt_dev->tgt_dev_flags))
812                                         pref_context = SCST_CONTEXT_THREAD;
813                         } else {
814                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
815                                                 &cmd->tgt_dev->tgt_dev_flags))
816                                         pref_context = SCST_CONTEXT_THREAD;
817                         }
818                 }
819                 break;
820
821         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
822                 scst_set_cmd_abnormal_done_state(cmd);
823                 break;
824
825         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
826                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
827                 /* go through */
828         case SCST_PREPROCESS_STATUS_ERROR:
829                 scst_set_cmd_error(cmd,
830                            SCST_LOAD_SENSE(scst_sense_hardw_error));
831                 scst_set_cmd_abnormal_done_state(cmd);
832                 break;
833
834         default:
835                 PRINT_ERROR("%s() received unknown status %x", __func__,
836                         status);
837                 scst_set_cmd_abnormal_done_state(cmd);
838                 break;
839         }
840
841         scst_proccess_redirect_cmd(cmd, pref_context, 1);
842
843         TRACE_EXIT();
844         return;
845 }
846 EXPORT_SYMBOL(scst_restart_cmd);
847
848 /* No locks */
849 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
850 {
851         struct scst_tgt *tgt = cmd->sess->tgt;
852         int res = 0;
853         unsigned long flags;
854
855         TRACE_ENTRY();
856
857         spin_lock_irqsave(&tgt->tgt_lock, flags);
858         tgt->retry_cmds++;
859         smp_mb();
860         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
861               tgt->retry_cmds);
862         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
863                 /* At least one cmd finished, so try again */
864                 tgt->retry_cmds--;
865                 TRACE_RETRY("Some command(s) finished, direct retry "
866                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
867                       "retry_cmds=%d)", finished_cmds,
868                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
869                 res = -1;
870                 goto out_unlock_tgt;
871         }
872
873         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
874         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
875
876         if (!tgt->retry_timer_active) {
877                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
878                 add_timer(&tgt->retry_timer);
879                 tgt->retry_timer_active = 1;
880         }
881
882 out_unlock_tgt:
883         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
884
885         TRACE_EXIT_RES(res);
886         return res;
887 }
888
889 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
890 {
891         int res, rc;
892
893         TRACE_ENTRY();
894
895         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
896                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
897                 goto out_dev_done;
898         }
899
900         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
901                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
902                 res = SCST_CMD_STATE_RES_CONT_SAME;
903                 goto out;
904         }
905
906         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
907                 /*
908                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
909                  * optimization.
910                  */
911                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
912                               "context, rescheduling", cmd->tgtt->name);
913                 res = SCST_CMD_STATE_RES_NEED_THREAD;
914                 goto out;
915         }
916
917         while (1) {
918                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
919
920                 res = SCST_CMD_STATE_RES_CONT_NEXT;
921                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
922
923                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
924 #ifdef CONFIG_SCST_DEBUG_RETRY
925                 if (((scst_random() % 100) == 75))
926                         rc = SCST_TGT_RES_QUEUE_FULL;
927                 else
928 #endif
929                         rc = cmd->tgtt->rdy_to_xfer(cmd);
930                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
931
932                 if (likely(rc == SCST_TGT_RES_SUCCESS))
933                         goto out;
934
935                 /* Restore the previous state */
936                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
937
938                 switch (rc) {
939                 case SCST_TGT_RES_QUEUE_FULL:
940                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
941                                 break;
942                         else
943                                 continue;
944
945                 case SCST_TGT_RES_NEED_THREAD_CTX:
946                         TRACE_DBG("Target driver %s "
947                               "rdy_to_xfer() requested thread "
948                               "context, rescheduling", cmd->tgtt->name);
949                         res = SCST_CMD_STATE_RES_NEED_THREAD;
950                         break;
951
952                 default:
953                         goto out_error_rc;
954                 }
955                 break;
956         }
957
958 out:
959         TRACE_EXIT_HRES(res);
960         return res;
961
962 out_error_rc:
963         if (rc == SCST_TGT_RES_FATAL_ERROR) {
964                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
965                      "fatal error", cmd->tgtt->name);
966         } else {
967                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
968                             "value %d", cmd->tgtt->name, rc);
969         }
970         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
971
972 out_dev_done:
973         scst_set_cmd_abnormal_done_state(cmd);
974         res = SCST_CMD_STATE_RES_CONT_SAME;
975         goto out;
976 }
977
978 /* No locks, but might be in IRQ */
979 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
980         int check_retries)
981 {
982         unsigned long flags;
983
984         TRACE_ENTRY();
985
986         TRACE_DBG("Context: %x", context);
987
988         switch (context & ~SCST_CONTEXT_PROCESSABLE) {
989         case SCST_CONTEXT_DIRECT_ATOMIC:
990                 context &= ~SCST_CONTEXT_PROCESSABLE;
991                 /* go through */
992         case SCST_CONTEXT_DIRECT:
993                 if (check_retries)
994                         scst_check_retries(cmd->tgt);
995                 scst_process_active_cmd(cmd, context);
996                 break;
997
998         default:
999                 PRINT_ERROR("Context %x is unknown, using the thread one",
1000                             context);
1001                 /* go through */
1002         case SCST_CONTEXT_THREAD:
1003                 if (check_retries)
1004                         scst_check_retries(cmd->tgt);
1005                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1006                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1007                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1008                         list_add(&cmd->cmd_list_entry,
1009                                 &cmd->cmd_lists->active_cmd_list);
1010                 else
1011                         list_add_tail(&cmd->cmd_list_entry,
1012                                 &cmd->cmd_lists->active_cmd_list);
1013                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1014                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1015                 break;
1016
1017         case SCST_CONTEXT_TASKLET:
1018                 if (check_retries)
1019                         scst_check_retries(cmd->tgt);
1020                 scst_schedule_tasklet(cmd);
1021                 break;
1022         }
1023
1024         TRACE_EXIT();
1025         return;
1026 }
1027
1028 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
1029 {
1030         TRACE_ENTRY();
1031
1032         TRACE_DBG("Preferred context: %d", pref_context);
1033         TRACE(TRACE_SCSI, "tag=%llu status=%#x",
1034               (long long unsigned int)scst_cmd_get_tag(cmd),
1035               status);
1036
1037 #ifdef CONFIG_SCST_EXTRACHECKS
1038         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
1039                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1040                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1041                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
1042                         cmd->tgtt->name);
1043                 pref_context = SCST_CONTEXT_TASKLET;
1044         }
1045 #endif
1046
1047         switch (status) {
1048         case SCST_RX_STATUS_SUCCESS:
1049                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1050                 /* Small context optimization */
1051                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1052                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1053                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1054                                         &cmd->tgt_dev->tgt_dev_flags))
1055                                 pref_context = SCST_CONTEXT_THREAD;
1056                 }
1057                 break;
1058
1059         case SCST_RX_STATUS_ERROR_SENSE_SET:
1060                 scst_set_cmd_abnormal_done_state(cmd);
1061                 break;
1062
1063         case SCST_RX_STATUS_ERROR_FATAL:
1064                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1065                 /* go through */
1066         case SCST_RX_STATUS_ERROR:
1067                 scst_set_cmd_error(cmd,
1068                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1069                 scst_set_cmd_abnormal_done_state(cmd);
1070                 break;
1071
1072         default:
1073                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1074                         status);
1075                 scst_set_cmd_abnormal_done_state(cmd);
1076                 break;
1077         }
1078
1079         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1080
1081         TRACE_EXIT();
1082         return;
1083 }
1084 EXPORT_SYMBOL(scst_rx_data);
1085
1086 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1087 {
1088         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1089
1090         TRACE_ENTRY();
1091
1092         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1093
1094         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1095                 goto out;
1096
1097         TRACE_DBG("Calling pre_exec(%p)", cmd);
1098         rc = cmd->tgtt->pre_exec(cmd);
1099         TRACE_DBG("pre_exec() returned %d", rc);
1100
1101         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1102                 switch (rc) {
1103                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1104                         scst_set_cmd_abnormal_done_state(cmd);
1105                         break;
1106                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1107                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1108                         /* go through */
1109                 case SCST_PREPROCESS_STATUS_ERROR:
1110                         scst_set_cmd_error(cmd,
1111                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1112                         scst_set_cmd_abnormal_done_state(cmd);
1113                         break;
1114                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1115                         TRACE_DBG("Target driver's %s pre_exec() requested "
1116                                 "thread context, rescheduling",
1117                                 cmd->tgtt->name);
1118                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1119                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1120                         break;
1121                 default:
1122                         sBUG();
1123                         break;
1124                 }
1125         }
1126
1127 out:
1128         TRACE_EXIT_RES(res);
1129         return res;
1130 }
1131
1132 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1133         const uint8_t *rq_sense, int rq_sense_len, int resid)
1134 {
1135         TRACE_ENTRY();
1136
1137 #ifdef CONFIG_SCST_MEASURE_LATENCY
1138         {
1139                 struct timespec ts;
1140                 getnstimeofday(&ts);
1141                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1142                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1143                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1144                         ts.tv_sec, ts.tv_nsec);
1145         }
1146 #endif
1147
1148         cmd->status = result & 0xff;
1149         cmd->msg_status = msg_byte(result);
1150         cmd->host_status = host_byte(result);
1151         cmd->driver_status = driver_byte(result);
1152         if (unlikely(resid != 0)) {
1153 #ifdef CONFIG_SCST_EXTRACHECKS
1154                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1155                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1156                                 "op %x)", resid, cmd->resp_data_len,
1157                                 cmd->cdb[0]);
1158                 } else
1159 #endif
1160                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1161         }
1162
1163         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1164                 /* We might have double reset UA here */
1165                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1166                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1167
1168                 scst_alloc_set_sense(cmd, scst_is_context_gfp_atomic(),
1169                                         rq_sense, rq_sense_len);
1170         }
1171
1172         TRACE(TRACE_SCSI, "cmd%p, result=%x, cmd->status=%x, resid=%d, "
1173               "cmd->msg_status=%x, cmd->host_status=%x, "
1174               "cmd->driver_status=%x", cmd, result, cmd->status, resid,
1175               cmd->msg_status, cmd->host_status, cmd->driver_status);
1176
1177         cmd->completed = 1;
1178
1179         TRACE_EXIT();
1180         return;
1181 }
1182
1183 /* For small context optimization */
1184 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1185         int context)
1186 {
1187         if ((context == SCST_CONTEXT_TASKLET) ||
1188             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1189                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1190                                 &cmd->tgt_dev->tgt_dev_flags))
1191                         context = SCST_CONTEXT_THREAD;
1192         }
1193         return context;
1194 }
1195
1196 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1197 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1198                                             struct scsi_request **req)
1199 {
1200         struct scst_cmd *cmd = NULL;
1201
1202         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1203                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1204
1205         if (cmd == NULL) {
1206                 PRINT_ERROR("%s", "Request with NULL cmd");
1207                 if (*req)
1208                         scsi_release_request(*req);
1209         }
1210
1211         return cmd;
1212 }
1213
1214 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1215 {
1216         struct scsi_request *req = NULL;
1217         struct scst_cmd *cmd;
1218
1219         TRACE_ENTRY();
1220
1221         cmd = scst_get_cmd(scsi_cmd, &req);
1222         if (cmd == NULL)
1223                 goto out;
1224
1225         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1226                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1227
1228         /* Clear out request structure */
1229         req->sr_use_sg = 0;
1230         req->sr_sglist_len = 0;
1231         req->sr_bufflen = 0;
1232         req->sr_buffer = NULL;
1233         req->sr_underflow = 0;
1234         req->sr_request->rq_disk = NULL; /* disown request blk */
1235
1236         scst_release_request(cmd);
1237
1238         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1239
1240         scst_proccess_redirect_cmd(cmd,
1241                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1242
1243 out:
1244         TRACE_EXIT();
1245         return;
1246 }
1247 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1248 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1249 {
1250         struct scst_cmd *cmd;
1251
1252         TRACE_ENTRY();
1253
1254         cmd = (struct scst_cmd *)data;
1255         if (cmd == NULL)
1256                 goto out;
1257
1258         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1259
1260         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1261
1262         scst_proccess_redirect_cmd(cmd,
1263                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1264
1265 out:
1266         TRACE_EXIT();
1267         return;
1268 }
1269 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1270
1271 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1272 {
1273         int context;
1274
1275         TRACE_ENTRY();
1276
1277 #ifdef CONFIG_SCST_MEASURE_LATENCY
1278         {
1279                 struct timespec ts;
1280                 getnstimeofday(&ts);
1281                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1282                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1283                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1284                         ts.tv_sec, ts.tv_nsec);
1285         }
1286 #endif
1287
1288         if (next_state == SCST_CMD_STATE_DEFAULT)
1289                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1290
1291 #if defined(CONFIG_SCST_DEBUG)
1292         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1293                 if (cmd->sg) {
1294                         int i;
1295                         struct scatterlist *sg = cmd->sg;
1296                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1297                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1298                         for (i = 0; i < cmd->sg_cnt; ++i) {
1299                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1300                                         "Exec'd sg", sg_virt(&sg[i]),
1301                                         sg[i].length);
1302                         }
1303                 }
1304         }
1305 #endif
1306
1307         cmd->state = next_state;
1308
1309 #ifdef CONFIG_SCST_EXTRACHECKS
1310         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1311             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1312             (next_state != SCST_CMD_STATE_FINISHED)) {
1313                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1314                         __func__, next_state, cmd->cdb[0]);
1315                 scst_set_cmd_error(cmd,
1316                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1317                 scst_set_cmd_abnormal_done_state(cmd);
1318         }
1319 #endif
1320         context = scst_optimize_post_exec_context(cmd, scst_get_context());
1321         if (cmd->context_processable)
1322                 context |= SCST_CONTEXT_PROCESSABLE;
1323         scst_proccess_redirect_cmd(cmd, context, 0);
1324
1325         TRACE_EXIT();
1326         return;
1327 }
1328
1329 static int scst_report_luns_local(struct scst_cmd *cmd)
1330 {
1331         int rc;
1332         int dev_cnt = 0;
1333         int buffer_size;
1334         int i;
1335         struct scst_tgt_dev *tgt_dev = NULL;
1336         uint8_t *buffer;
1337         int offs, overflow = 0;
1338
1339         TRACE_ENTRY();
1340
1341         rc = scst_check_local_events(cmd);
1342         if (unlikely(rc != 0))
1343                 goto out_done;
1344
1345         cmd->status = 0;
1346         cmd->msg_status = 0;
1347         cmd->host_status = DID_OK;
1348         cmd->driver_status = 0;
1349
1350         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1351                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1352                         "LUNS command", cmd->cdb[2]);
1353                 goto out_err;
1354         }
1355
1356         buffer_size = scst_get_buf_first(cmd, &buffer);
1357         if (unlikely(buffer_size == 0))
1358                 goto out_compl;
1359         else if (unlikely(buffer_size < 0))
1360                 goto out_err;
1361
1362         if (buffer_size < 16)
1363                 goto out_put_err;
1364
1365         memset(buffer, 0, buffer_size);
1366         offs = 8;
1367
1368         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1369         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1370                 struct list_head *sess_tgt_dev_list_head =
1371                         &cmd->sess->sess_tgt_dev_list_hash[i];
1372                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1373                                 sess_tgt_dev_list_entry) {
1374                         if (!overflow) {
1375                                 if (offs >= buffer_size) {
1376                                         scst_put_buf(cmd, buffer);
1377                                         buffer_size = scst_get_buf_next(cmd,
1378                                                                        &buffer);
1379                                         if (buffer_size > 0) {
1380                                                 memset(buffer, 0, buffer_size);
1381                                                 offs = 0;
1382                                         } else {
1383                                                 overflow = 1;
1384                                                 goto inc_dev_cnt;
1385                                         }
1386                                 }
1387                                 if ((buffer_size - offs) < 8) {
1388                                         PRINT_ERROR("Buffer allocated for "
1389                                                 "REPORT LUNS command doesn't "
1390                                                 "allow to fit 8 byte entry "
1391                                                 "(buffer_size=%d)",
1392                                                 buffer_size);
1393                                         goto out_put_hw_err;
1394                                 }
1395                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1396                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1397                                 offs += 8;
1398                         }
1399 inc_dev_cnt:
1400                         dev_cnt++;
1401                 }
1402         }
1403         if (!overflow)
1404                 scst_put_buf(cmd, buffer);
1405
1406         /* Set the response header */
1407         buffer_size = scst_get_buf_first(cmd, &buffer);
1408         if (unlikely(buffer_size == 0))
1409                 goto out_compl;
1410         else if (unlikely(buffer_size < 0))
1411                 goto out_err;
1412
1413         dev_cnt *= 8;
1414         buffer[0] = (dev_cnt >> 24) & 0xff;
1415         buffer[1] = (dev_cnt >> 16) & 0xff;
1416         buffer[2] = (dev_cnt >> 8) & 0xff;
1417         buffer[3] = dev_cnt & 0xff;
1418
1419         scst_put_buf(cmd, buffer);
1420
1421         dev_cnt += 8;
1422         if (dev_cnt < cmd->resp_data_len)
1423                 scst_set_resp_data_len(cmd, dev_cnt);
1424
1425 out_compl:
1426         cmd->completed = 1;
1427
1428 out_done:
1429         /* Report the result */
1430         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1431
1432         TRACE_EXIT();
1433         return SCST_EXEC_COMPLETED;
1434
1435 out_put_err:
1436         scst_put_buf(cmd, buffer);
1437
1438 out_err:
1439         scst_set_cmd_error(cmd,
1440                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1441         goto out_compl;
1442
1443 out_put_hw_err:
1444         scst_put_buf(cmd, buffer);
1445         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1446         goto out_compl;
1447 }
1448
1449 static int scst_pre_select(struct scst_cmd *cmd)
1450 {
1451         int res = SCST_EXEC_NOT_COMPLETED;
1452
1453         TRACE_ENTRY();
1454
1455         if (scst_cmd_atomic(cmd)) {
1456                 res = SCST_EXEC_NEED_THREAD;
1457                 goto out;
1458         }
1459
1460         scst_block_dev_cmd(cmd, 1);
1461
1462         /* Check for local events will be done when cmd will be executed */
1463
1464 out:
1465         TRACE_EXIT_RES(res);
1466         return res;
1467 }
1468
1469 static int scst_reserve_local(struct scst_cmd *cmd)
1470 {
1471         int res = SCST_EXEC_NOT_COMPLETED, rc;
1472         struct scst_device *dev;
1473         struct scst_tgt_dev *tgt_dev_tmp;
1474
1475         TRACE_ENTRY();
1476
1477         if (scst_cmd_atomic(cmd)) {
1478                 res = SCST_EXEC_NEED_THREAD;
1479                 goto out;
1480         }
1481
1482         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1483                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1484                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1485                 scst_set_cmd_error(cmd,
1486                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1487                 goto out_done;
1488         }
1489
1490         dev = cmd->dev;
1491
1492         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1493                 scst_block_dev_cmd(cmd, 1);
1494
1495         rc = scst_check_local_events(cmd);
1496         if (unlikely(rc != 0))
1497                 goto out_done;
1498
1499         spin_lock_bh(&dev->dev_lock);
1500
1501         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1502                 spin_unlock_bh(&dev->dev_lock);
1503                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1504                 goto out_done;
1505         }
1506
1507         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1508                             dev_tgt_dev_list_entry) {
1509                 if (cmd->tgt_dev != tgt_dev_tmp)
1510                         set_bit(SCST_TGT_DEV_RESERVED,
1511                                 &tgt_dev_tmp->tgt_dev_flags);
1512         }
1513         dev->dev_reserved = 1;
1514
1515         spin_unlock_bh(&dev->dev_lock);
1516
1517 out:
1518         TRACE_EXIT_RES(res);
1519         return res;
1520
1521 out_done:
1522         /* Report the result */
1523         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1524         res = SCST_EXEC_COMPLETED;
1525         goto out;
1526 }
1527
1528 static int scst_release_local(struct scst_cmd *cmd)
1529 {
1530         int res = SCST_EXEC_NOT_COMPLETED, rc;
1531         struct scst_tgt_dev *tgt_dev_tmp;
1532         struct scst_device *dev;
1533
1534         TRACE_ENTRY();
1535
1536         if (scst_cmd_atomic(cmd)) {
1537                 res = SCST_EXEC_NEED_THREAD;
1538                 goto out;
1539         }
1540
1541         dev = cmd->dev;
1542
1543         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1544                 scst_block_dev_cmd(cmd, 1);
1545
1546         rc = scst_check_local_events(cmd);
1547         if (unlikely(rc != 0))
1548                 goto out_done;
1549
1550         spin_lock_bh(&dev->dev_lock);
1551
1552         /*
1553          * The device could be RELEASED behind us, if RESERVING session
1554          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1555          * matter, so use lock and no retest for DEV_RESERVED bits again
1556          */
1557         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1558                 res = SCST_EXEC_COMPLETED;
1559                 cmd->status = 0;
1560                 cmd->msg_status = 0;
1561                 cmd->host_status = DID_OK;
1562                 cmd->driver_status = 0;
1563                 cmd->completed = 1;
1564         } else {
1565                 list_for_each_entry(tgt_dev_tmp,
1566                                     &dev->dev_tgt_dev_list,
1567                                     dev_tgt_dev_list_entry) {
1568                         clear_bit(SCST_TGT_DEV_RESERVED,
1569                                 &tgt_dev_tmp->tgt_dev_flags);
1570                 }
1571                 dev->dev_reserved = 0;
1572         }
1573
1574         spin_unlock_bh(&dev->dev_lock);
1575
1576         if (res == SCST_EXEC_COMPLETED)
1577                 goto out_done;
1578
1579 out:
1580         TRACE_EXIT_RES(res);
1581         return res;
1582
1583 out_done:
1584         res = SCST_EXEC_COMPLETED;
1585         /* Report the result */
1586         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1587         goto out;
1588 }
1589
1590 /* No locks, no IRQ or IRQ-safe context allowed */
1591 int scst_check_local_events(struct scst_cmd *cmd)
1592 {
1593         int res, rc;
1594         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1595         struct scst_device *dev = cmd->dev;
1596
1597         TRACE_ENTRY();
1598
1599         /*
1600          * There's no race here, because we need to trace commands sent
1601          * *after* dev_double_ua_possible flag was set.
1602          */
1603         if (unlikely(dev->dev_double_ua_possible))
1604                 cmd->double_ua_possible = 1;
1605
1606         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1607                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1608                 goto out_uncomplete;
1609         }
1610
1611         /* Reserve check before Unit Attention */
1612         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1613                               &tgt_dev->tgt_dev_flags))) {
1614                 if (cmd->cdb[0] != INQUIRY &&
1615                     cmd->cdb[0] != REPORT_LUNS &&
1616                     cmd->cdb[0] != RELEASE &&
1617                     cmd->cdb[0] != RELEASE_10 &&
1618                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1619                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1620                      (cmd->cdb[4] & 3)) &&
1621                     cmd->cdb[0] != LOG_SENSE &&
1622                     cmd->cdb[0] != REQUEST_SENSE) {
1623                         scst_set_cmd_error_status(cmd,
1624                                 SAM_STAT_RESERVATION_CONFLICT);
1625                         goto out_complete;
1626                 }
1627         }
1628
1629         /* If we had internal bus reset, set the command error unit attention */
1630         if ((dev->scsi_dev != NULL) &&
1631             unlikely(dev->scsi_dev->was_reset)) {
1632                 if (scst_is_ua_command(cmd)) {
1633                         int done = 0;
1634                         /*
1635                          * Prevent more than 1 cmd to be triggered by
1636                          * was_reset.
1637                          */
1638                         spin_lock_bh(&dev->dev_lock);
1639                         barrier(); /* to reread was_reset */
1640                         if (dev->scsi_dev->was_reset) {
1641                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1642                                 scst_set_cmd_error(cmd,
1643                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1644                                 /*
1645                                  * It looks like it is safe to clear was_reset
1646                                  * here.
1647                                  */
1648                                 dev->scsi_dev->was_reset = 0;
1649                                 done = 1;
1650                         }
1651                         spin_unlock_bh(&dev->dev_lock);
1652
1653                         if (done)
1654                                 goto out_complete;
1655                 }
1656         }
1657
1658         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1659                         &cmd->tgt_dev->tgt_dev_flags))) {
1660                 if (scst_is_ua_command(cmd)) {
1661                         rc = scst_set_pending_UA(cmd);
1662                         if (rc == 0)
1663                                 goto out_complete;
1664                 }
1665         }
1666
1667         res = 0;
1668
1669 out:
1670         TRACE_EXIT_RES(res);
1671         return res;
1672
1673 out_complete:
1674         res = 1;
1675         sBUG_ON(!cmd->completed);
1676         goto out;
1677
1678 out_uncomplete:
1679         res = -1;
1680         goto out;
1681 }
1682 EXPORT_SYMBOL(scst_check_local_events);
1683
1684 /* No locks */
1685 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1686 {
1687         if (slot == NULL)
1688                 goto inc;
1689
1690         /* Optimized for lockless fast path */
1691
1692         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1693                 atomic_read(slot));
1694
1695         if (!atomic_dec_and_test(slot))
1696                 goto out;
1697
1698         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1699                 tgt_dev->num_free_sn_slots);
1700         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1701                 spin_lock_irq(&tgt_dev->sn_lock);
1702                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1703                         if (tgt_dev->num_free_sn_slots < 0)
1704                                 tgt_dev->cur_sn_slot = slot;
1705                         /*
1706                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1707                          */
1708                         smp_mb();
1709                         tgt_dev->num_free_sn_slots++;
1710                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1711                                 tgt_dev->num_free_sn_slots);
1712
1713                 }
1714                 spin_unlock_irq(&tgt_dev->sn_lock);
1715         }
1716
1717 inc:
1718         /*
1719          * No locks is needed, because only one thread at time can
1720          * be here (serialized by sn). Also it is supposed that there
1721          * could not be half-incremented halves.
1722          */
1723         tgt_dev->expected_sn++;
1724         smp_mb(); /* write must be before def_cmd_count read */
1725         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1726
1727 out:
1728         return;
1729 }
1730
1731 /* No locks */
1732 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1733         bool make_active)
1734 {
1735         /* For HQ commands SN is not set */
1736         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1737                                cmd->sn_set && !cmd->retry;
1738         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1739         struct scst_cmd *res;
1740
1741         TRACE_ENTRY();
1742
1743         if (inc_expected_sn)
1744                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1745
1746         if (make_active) {
1747                 scst_make_deferred_commands_active(tgt_dev);
1748                 res = NULL;
1749         } else
1750                 res = scst_check_deferred_commands(tgt_dev);
1751
1752         TRACE_EXIT_HRES(res);
1753         return res;
1754 }
1755
1756 /* cmd must be additionally referenced to not die inside */
1757 static int scst_do_real_exec(struct scst_cmd *cmd)
1758 {
1759         int res = SCST_EXEC_NOT_COMPLETED, rc;
1760         struct scst_device *dev = cmd->dev;
1761         struct scst_dev_type *handler = dev->handler;
1762
1763         TRACE_ENTRY();
1764
1765         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1766
1767         if (!handler->exec_sync)
1768                 cmd->context_processable = 0;
1769
1770         if (handler->exec) {
1771                 if (unlikely(!dev->handler->exec_atomic &&
1772                              scst_cmd_atomic(cmd))) {
1773                         /*
1774                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1775                          * optimization.
1776                          */
1777                         TRACE_DBG("Dev handler %s exec() needs thread "
1778                                 "context, rescheduling", dev->handler->name);
1779                         res = SCST_EXEC_NEED_THREAD;
1780                         goto out_restore;
1781                 }
1782
1783                 TRACE_DBG("Calling dev handler %s exec(%p)",
1784                       handler->name, cmd);
1785                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1786                         cmd->cdb_len);
1787                 res = handler->exec(cmd);
1788                 TRACE_DBG("Dev handler %s exec() returned %d",
1789                       handler->name, res);
1790
1791                 if (res == SCST_EXEC_COMPLETED)
1792                         goto out_complete;
1793                 else if (res == SCST_EXEC_NEED_THREAD)
1794                         goto out_restore;
1795
1796                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1797         }
1798
1799         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1800
1801         if (unlikely(dev->scsi_dev == NULL)) {
1802                 PRINT_ERROR("Command for virtual device must be "
1803                         "processed by device handler (lun %lld)!",
1804                         (long long unsigned int)cmd->lun);
1805                 goto out_error;
1806         }
1807
1808         res = scst_check_local_events(cmd);
1809         if (unlikely(res != 0))
1810                 goto out_done;
1811
1812 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1813         if (unlikely(scst_cmd_atomic(cmd))) {
1814                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1815                         "context, rescheduling to the thread (handler %s)",
1816                         handler->name);
1817                 res = SCST_EXEC_NEED_THREAD;
1818                 goto out_restore;
1819         }
1820 #endif
1821
1822 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1823         if (unlikely(scst_alloc_request(cmd) != 0)) {
1824                 if (scst_cmd_atomic(cmd)) {
1825                         res = SCST_EXEC_NEED_THREAD;
1826                         goto out_restore;
1827                 } else {
1828                         PRINT_INFO("%s", "Unable to allocate request, "
1829                                 "sending BUSY status");
1830                         goto out_busy;
1831                 }
1832         }
1833
1834         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1835                     (void *)cmd->scsi_req->sr_buffer,
1836                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1837                     cmd->retries);
1838 #else
1839         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1840                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1841                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1842                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1843         if (unlikely(rc != 0)) {
1844                 if (scst_cmd_atomic(cmd)) {
1845                         res = SCST_EXEC_NEED_THREAD;
1846                         goto out_restore;
1847                 } else {
1848                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1849                         goto out_error;
1850                 }
1851         }
1852 #endif
1853
1854 out_complete:
1855         res = SCST_EXEC_COMPLETED;
1856
1857 out:
1858         TRACE_EXIT();
1859         return res;
1860
1861 out_restore:
1862         /* Restore the state */
1863         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1864         goto out;
1865
1866 out_error:
1867         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1868         goto out_done;
1869
1870 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1871 out_busy:
1872         scst_set_busy(cmd);
1873         /* go through */
1874 #endif
1875
1876 out_done:
1877         res = SCST_EXEC_COMPLETED;
1878         /* Report the result */
1879         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1880         goto out_complete;
1881 }
1882
1883 static inline int scst_real_exec(struct scst_cmd *cmd)
1884 {
1885         int res;
1886
1887         TRACE_ENTRY();
1888
1889         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1890         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1891         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1892
1893         __scst_cmd_get(cmd);
1894
1895         res = scst_do_real_exec(cmd);
1896
1897         if (likely(res == SCST_EXEC_COMPLETED)) {
1898                 scst_post_exec_sn(cmd, true);
1899                 if (cmd->dev->scsi_dev != NULL)
1900                         generic_unplug_device(
1901                                 cmd->dev->scsi_dev->request_queue);
1902         } else
1903                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1904
1905         __scst_cmd_put(cmd);
1906
1907         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1908
1909         TRACE_EXIT_RES(res);
1910         return res;
1911 }
1912
1913 static int scst_do_local_exec(struct scst_cmd *cmd)
1914 {
1915         int res;
1916         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1917
1918         TRACE_ENTRY();
1919
1920         /* Check READ_ONLY device status */
1921         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1922             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1923              cmd->cdb[0] == WRITE_10 ||
1924              cmd->cdb[0] == WRITE_12 ||
1925              cmd->cdb[0] == WRITE_16 ||
1926              cmd->cdb[0] == WRITE_VERIFY ||
1927              cmd->cdb[0] == WRITE_VERIFY_12 ||
1928              cmd->cdb[0] == WRITE_VERIFY_16 ||
1929              (cmd->dev->handler->type == TYPE_TAPE &&
1930               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1931                 scst_set_cmd_error(cmd,
1932                            SCST_LOAD_SENSE(scst_sense_data_protect));
1933                 goto out_done;
1934         }
1935
1936         /*
1937          * Adding new commands here don't forget to update
1938          * scst_is_cmd_local() in scst.h, if necessary
1939          */
1940
1941         switch (cmd->cdb[0]) {
1942         case MODE_SELECT:
1943         case MODE_SELECT_10:
1944         case LOG_SELECT:
1945                 res = scst_pre_select(cmd);
1946                 break;
1947         case RESERVE:
1948         case RESERVE_10:
1949                 res = scst_reserve_local(cmd);
1950                 break;
1951         case RELEASE:
1952         case RELEASE_10:
1953                 res = scst_release_local(cmd);
1954                 break;
1955         case REPORT_LUNS:
1956                 res = scst_report_luns_local(cmd);
1957                 break;
1958         default:
1959                 res = SCST_EXEC_NOT_COMPLETED;
1960                 break;
1961         }
1962
1963 out:
1964         TRACE_EXIT_RES(res);
1965         return res;
1966
1967 out_done:
1968         /* Report the result */
1969         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1970         res = SCST_EXEC_COMPLETED;
1971         goto out;
1972 }
1973
1974 static int scst_local_exec(struct scst_cmd *cmd)
1975 {
1976         int res;
1977
1978         TRACE_ENTRY();
1979
1980         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1981         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1982         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1983
1984         __scst_cmd_get(cmd);
1985
1986         res = scst_do_local_exec(cmd);
1987         if (likely(res == SCST_EXEC_NOT_COMPLETED))
1988                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
1989         else if (res == SCST_EXEC_COMPLETED)
1990                 scst_post_exec_sn(cmd, true);
1991         else
1992                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1993
1994         __scst_cmd_put(cmd);
1995
1996         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1997         TRACE_EXIT_RES(res);
1998         return res;
1999 }
2000
2001 static int scst_exec(struct scst_cmd **active_cmd)
2002 {
2003         struct scst_cmd *cmd = *active_cmd;
2004         struct scst_cmd *ref_cmd;
2005         struct scst_device *dev = cmd->dev;
2006         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2007
2008         TRACE_ENTRY();
2009
2010         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2011                 goto out;
2012
2013         /* To protect tgt_dev */
2014         ref_cmd = cmd;
2015         __scst_cmd_get(ref_cmd);
2016
2017         count = 0;
2018         while (1) {
2019                 int rc;
2020
2021                 cmd->sent_for_exec = 1;
2022                 cmd->scst_cmd_done = scst_cmd_done_local;
2023                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2024
2025                 rc = scst_do_local_exec(cmd);
2026                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2027                         /* Nothing to do */;
2028                 else if (rc == SCST_EXEC_NEED_THREAD) {
2029                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2030                                 "thread context, rescheduling");
2031                         scst_dec_on_dev_cmd(cmd);
2032                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2033                         break;
2034                 } else {
2035                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2036                         goto done;
2037                 }
2038
2039                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2040
2041                 rc = scst_do_real_exec(cmd);
2042                 if (likely(rc == SCST_EXEC_COMPLETED))
2043                         /* Nothing to do */;
2044                 else if (rc == SCST_EXEC_NEED_THREAD) {
2045                         TRACE_DBG("scst_real_exec() requested thread "
2046                                 "context, rescheduling (cmd %p)", cmd);
2047                         scst_dec_on_dev_cmd(cmd);
2048                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2049                         break;
2050                 } else
2051                         sBUG();
2052
2053 done:
2054                 count++;
2055
2056                 cmd = scst_post_exec_sn(cmd, false);
2057                 if (cmd == NULL)
2058                         break;
2059
2060                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2061                         break;
2062
2063                 __scst_cmd_put(ref_cmd);
2064                 ref_cmd = cmd;
2065                 __scst_cmd_get(ref_cmd);
2066         }
2067
2068         *active_cmd = cmd;
2069
2070         if (count == 0)
2071                 goto out_put;
2072
2073         if (dev->scsi_dev != NULL)
2074                 generic_unplug_device(dev->scsi_dev->request_queue);
2075
2076 out_put:
2077         __scst_cmd_put(ref_cmd);
2078         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2079
2080 out:
2081         TRACE_EXIT_RES(res);
2082         return res;
2083 }
2084
2085 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2086 {
2087         int res;
2088         struct scst_cmd *cmd = *active_cmd;
2089         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2090         typeof(tgt_dev->expected_sn) expected_sn;
2091
2092         TRACE_ENTRY();
2093
2094 #ifdef CONFIG_SCST_MEASURE_LATENCY
2095         if (cmd->pre_exec_finish == 0) {
2096                 struct timespec ts;
2097                 getnstimeofday(&ts);
2098                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2099                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2100                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2101                         ts.tv_sec, ts.tv_nsec);
2102         }
2103 #endif
2104
2105         if (unlikely(cmd->internal))
2106                 goto exec;
2107
2108         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2109                 goto exec;
2110
2111         sBUG_ON(!cmd->sn_set);
2112
2113         expected_sn = tgt_dev->expected_sn;
2114         /* Optimized for lockless fast path */
2115         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2116                 spin_lock_irq(&tgt_dev->sn_lock);
2117
2118                 tgt_dev->def_cmd_count++;
2119                 smp_mb();
2120
2121                 expected_sn = tgt_dev->expected_sn;
2122                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2123                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2124                                               &cmd->cmd_flags))) {
2125                                 /* Necessary to allow aborting out of sn cmds */
2126                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2127                                         "(tag %llu, sn %lu)", cmd,
2128                                         (long long unsigned)cmd->tag, cmd->sn);
2129                                 tgt_dev->def_cmd_count--;
2130                                 scst_set_cmd_abnormal_done_state(cmd);
2131                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2132                         } else {
2133                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2134                                         "expected_sn=%ld)", cmd, cmd->sn,
2135                                         cmd->sn_set, expected_sn);
2136                                 list_add_tail(&cmd->sn_cmd_list_entry,
2137                                               &tgt_dev->deferred_cmd_list);
2138                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2139                         }
2140                         spin_unlock_irq(&tgt_dev->sn_lock);
2141                         goto out;
2142                 } else {
2143                         TRACE_SN("Somebody incremented expected_sn %ld, "
2144                                 "continuing", expected_sn);
2145                         tgt_dev->def_cmd_count--;
2146                         spin_unlock_irq(&tgt_dev->sn_lock);
2147                 }
2148         }
2149
2150 exec:
2151         res = scst_exec(active_cmd);
2152
2153 out:
2154         TRACE_EXIT_HRES(res);
2155         return res;
2156 }
2157
2158 /* No locks supposed to be held */
2159 static int scst_check_sense(struct scst_cmd *cmd)
2160 {
2161         int res = 0;
2162         struct scst_device *dev = cmd->dev;
2163
2164         TRACE_ENTRY();
2165
2166         if (unlikely(cmd->ua_ignore))
2167                 goto out;
2168
2169         /* If we had internal bus reset behind us, set the command error UA */
2170         if ((dev->scsi_dev != NULL) &&
2171             unlikely(cmd->host_status == DID_RESET) &&
2172             scst_is_ua_command(cmd)) {
2173                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2174                       dev->scsi_dev->was_reset, cmd->host_status);
2175                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2176                 /* It looks like it is safe to clear was_reset here */
2177                 dev->scsi_dev->was_reset = 0;
2178         }
2179
2180         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2181             SCST_SENSE_VALID(cmd->sense)) {
2182                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2183                         SCST_SENSE_BUFFERSIZE);
2184
2185                 /* Check Unit Attention Sense Key */
2186                 if (scst_is_ua_sense(cmd->sense)) {
2187                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2188                                 if (cmd->double_ua_possible) {
2189                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2190                                                 "detected for device %p", dev);
2191                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2192                                                 " %p (tag %llu)", cmd,
2193                                                 (long long unsigned)cmd->tag);
2194
2195                                         cmd->status = 0;
2196                                         cmd->msg_status = 0;
2197                                         cmd->host_status = DID_OK;
2198                                         cmd->driver_status = 0;
2199
2200                                         mempool_free(cmd->sense,
2201                                                      scst_sense_mempool);
2202                                         cmd->sense = NULL;
2203
2204                                         scst_check_restore_sg_buff(cmd);
2205
2206                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2207                                         cmd->data_direction =
2208                                                 cmd->dbl_ua_orig_data_direction;
2209                                         cmd->resp_data_len =
2210                                                 cmd->dbl_ua_orig_resp_data_len;
2211
2212                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2213                                         cmd->retry = 1;
2214                                         res = 1;
2215                                         goto out;
2216                                 }
2217                         }
2218                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2219                                 SCST_SENSE_BUFFERSIZE);
2220                 }
2221         }
2222
2223         if (unlikely(cmd->double_ua_possible)) {
2224                 if (scst_is_ua_command(cmd)) {
2225                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2226                                 "cmd %p)", dev, cmd);
2227                         /*
2228                          * Lock used to protect other flags in the bitfield
2229                          * (just in case, actually). Those 2 flags can't be
2230                          * changed in parallel, because the device is
2231                          * serialized.
2232                          */
2233                         spin_lock_bh(&dev->dev_lock);
2234                         dev->dev_double_ua_possible = 0;
2235                         dev->dev_serialized = 0;
2236                         spin_unlock_bh(&dev->dev_lock);
2237                 }
2238         }
2239
2240 out:
2241         TRACE_EXIT_RES(res);
2242         return res;
2243 }
2244
2245 static int scst_check_auto_sense(struct scst_cmd *cmd)
2246 {
2247         int res = 0;
2248
2249         TRACE_ENTRY();
2250
2251         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2252             (!SCST_SENSE_VALID(cmd->sense) ||
2253              SCST_NO_SENSE(cmd->sense))) {
2254                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2255                       "cmd->status=%x, cmd->msg_status=%x, "
2256                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2257                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2258                 res = 1;
2259         } else if (unlikely(cmd->host_status)) {
2260                 if ((cmd->host_status == DID_REQUEUE) ||
2261                     (cmd->host_status == DID_IMM_RETRY) ||
2262                     (cmd->host_status == DID_SOFT_ERROR) ||
2263                     (cmd->host_status == DID_ABORT)) {
2264                         scst_set_busy(cmd);
2265                 } else {
2266                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2267                                 "received, returning HARDWARE ERROR instead",
2268                                 cmd->host_status);
2269                         scst_set_cmd_error(cmd,
2270                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2271                 }
2272         }
2273
2274         TRACE_EXIT_RES(res);
2275         return res;
2276 }
2277
2278 static int scst_pre_dev_done(struct scst_cmd *cmd)
2279 {
2280         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2281
2282         TRACE_ENTRY();
2283
2284         if (unlikely(scst_check_auto_sense(cmd))) {
2285                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2286                             "without sense data (opcode 0x%x), issuing "
2287                             "REQUEST SENSE", cmd->cdb[0]);
2288                 rc = scst_prepare_request_sense(cmd);
2289                 if (rc == 0)
2290                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2291                 else {
2292                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2293                                     "returning HARDWARE ERROR");
2294                         scst_set_cmd_error(cmd,
2295                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2296                 }
2297                 goto out;
2298         } else if (unlikely(scst_check_sense(cmd)))
2299                 goto out;
2300
2301         if (likely(scsi_status_is_good(cmd->status))) {
2302                 unsigned char type = cmd->dev->handler->type;
2303                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2304                               cmd->cdb[0] == MODE_SENSE_10)) &&
2305                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2306                     (type == TYPE_DISK ||
2307                      type == TYPE_WORM ||
2308                      type == TYPE_MOD ||
2309                      type == TYPE_TAPE)) {
2310                         int32_t length;
2311                         uint8_t *address;
2312                         bool err = false;
2313
2314                         length = scst_get_buf_first(cmd, &address);
2315                         if (length < 0) {
2316                                 PRINT_ERROR("%s", "Unable to get "
2317                                         "MODE_SENSE buffer");
2318                                 scst_set_cmd_error(cmd,
2319                                         SCST_LOAD_SENSE(
2320                                                 scst_sense_hardw_error));
2321                                 err = true;
2322                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2323                                 address[2] |= 0x80;   /* Write Protect*/
2324                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2325                                 address[3] |= 0x80;   /* Write Protect*/
2326                         scst_put_buf(cmd, address);
2327
2328                         if (err)
2329                                 goto out;
2330                 }
2331
2332                 /*
2333                  * Check and clear NormACA option for the device, if necessary,
2334                  * since we don't support ACA
2335                  */
2336                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2337                     /* Std INQUIRY data (no EVPD) */
2338                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2339                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2340                         uint8_t *buffer;
2341                         int buflen;
2342                         bool err = false;
2343
2344                         /* ToDo: all pages ?? */
2345                         buflen = scst_get_buf_first(cmd, &buffer);
2346                         if (buflen > SCST_INQ_BYTE3) {
2347 #ifdef CONFIG_SCST_EXTRACHECKS
2348                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2349                                         PRINT_INFO("NormACA set for device: "
2350                                             "lun=%lld, type 0x%02x. Clear it, "
2351                                             "since it's unsupported.",
2352                                             (long long unsigned int)cmd->lun,
2353                                             buffer[0]);
2354                                 }
2355 #endif
2356                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2357                         } else if (buflen != 0) {
2358                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2359                                     "buffer");
2360                                 scst_set_cmd_error(cmd,
2361                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2362                                 err = true;
2363                         }
2364                         if (buflen > 0)
2365                                 scst_put_buf(cmd, buffer);
2366
2367                         if (err)
2368                                 goto out;
2369                 }
2370
2371                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2372                     (cmd->cdb[0] == MODE_SELECT_10) ||
2373                     (cmd->cdb[0] == LOG_SELECT))) {
2374                         TRACE(TRACE_SCSI,
2375                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2376                                 (long long unsigned int)cmd->lun);
2377                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2378                         goto out;
2379                 }
2380         } else {
2381                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2382                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2383                                         &cmd->tgt_dev->tgt_dev_flags)) {
2384                                 struct scst_tgt_dev *tgt_dev_tmp;
2385                                 struct scst_device *dev = cmd->dev;
2386
2387                                 TRACE(TRACE_SCSI,
2388                                         "Real RESERVE failed lun=%lld, "
2389                                         "status=%x",
2390                                         (long long unsigned int)cmd->lun,
2391                                         cmd->status);
2392                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2393                                         SCST_SENSE_BUFFERSIZE);
2394
2395                                 /* Clearing the reservation */
2396                                 spin_lock_bh(&dev->dev_lock);
2397                                 list_for_each_entry(tgt_dev_tmp,
2398                                                     &dev->dev_tgt_dev_list,
2399                                                     dev_tgt_dev_list_entry) {
2400                                         clear_bit(SCST_TGT_DEV_RESERVED,
2401                                                 &tgt_dev_tmp->tgt_dev_flags);
2402                                 }
2403                                 dev->dev_reserved = 0;
2404                                 spin_unlock_bh(&dev->dev_lock);
2405                         }
2406                 }
2407
2408                 /* Check for MODE PARAMETERS CHANGED UA */
2409                 if ((cmd->dev->scsi_dev != NULL) &&
2410                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2411                     SCST_SENSE_VALID(cmd->sense) &&
2412                     scst_is_ua_sense(cmd->sense) &&
2413                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2414                         TRACE(TRACE_SCSI,
2415                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2416                               (long long unsigned int)cmd->lun);
2417                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2418                         goto out;
2419                 }
2420         }
2421
2422         cmd->state = SCST_CMD_STATE_DEV_DONE;
2423
2424 out:
2425         TRACE_EXIT_RES(res);
2426         return res;
2427 }
2428
2429 static int scst_mode_select_checks(struct scst_cmd *cmd)
2430 {
2431         int res = SCST_CMD_STATE_RES_CONT_SAME;
2432         int atomic = scst_cmd_atomic(cmd);
2433
2434         TRACE_ENTRY();
2435
2436         if (likely(scsi_status_is_good(cmd->status))) {
2437                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2438                     (cmd->cdb[0] == MODE_SELECT_10) ||
2439                     (cmd->cdb[0] == LOG_SELECT))) {
2440                         struct scst_device *dev = cmd->dev;
2441                         if (atomic && (dev->scsi_dev != NULL)) {
2442                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2443                                         "context required");
2444                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2445                                 goto out;
2446                         }
2447
2448                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2449                                 "setting the SELECT UA (lun=%lld)",
2450                                 (long long unsigned int)cmd->lun);
2451
2452                         spin_lock_bh(&dev->dev_lock);
2453                         spin_lock(&scst_temp_UA_lock);
2454                         if (cmd->cdb[0] == LOG_SELECT) {
2455                                 scst_set_sense(scst_temp_UA,
2456                                         sizeof(scst_temp_UA),
2457                                         UNIT_ATTENTION, 0x2a, 0x02);
2458                         } else {
2459                                 scst_set_sense(scst_temp_UA,
2460                                         sizeof(scst_temp_UA),
2461                                         UNIT_ATTENTION, 0x2a, 0x01);
2462                         }
2463                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2464                                 sizeof(scst_temp_UA));
2465                         spin_unlock(&scst_temp_UA_lock);
2466                         spin_unlock_bh(&dev->dev_lock);
2467
2468                         if (dev->scsi_dev != NULL)
2469                                 scst_obtain_device_parameters(dev);
2470                 }
2471         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2472                     SCST_SENSE_VALID(cmd->sense) &&
2473                     scst_is_ua_sense(cmd->sense) &&
2474                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2475                      (cmd->sense[12] == 0x29) /* reset */ ||
2476                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2477                      /* cleared by another ini (just in case) */
2478                      (cmd->sense[12] == 0x2F))) {
2479                 if (atomic) {
2480                         TRACE_DBG("Possible parameters changed UA %x: "
2481                                 "thread context required", cmd->sense[12]);
2482                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2483                         goto out;
2484                 }
2485
2486                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2487                         "(lun %lld): getting new parameters", cmd->sense[12],
2488                         (long long unsigned int)cmd->lun);
2489
2490                 scst_obtain_device_parameters(cmd->dev);
2491         } else
2492                 sBUG();
2493
2494         cmd->state = SCST_CMD_STATE_DEV_DONE;
2495
2496 out:
2497         TRACE_EXIT_HRES(res);
2498         return res;
2499 }
2500
2501 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2502 {
2503         if (likely(cmd->sn_set))
2504                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2505
2506         scst_make_deferred_commands_active(cmd->tgt_dev);
2507 }
2508
2509 static int scst_dev_done(struct scst_cmd **pcmd)
2510 {
2511         int res = SCST_CMD_STATE_RES_CONT_SAME;
2512         struct scst_cmd *cmd = *pcmd;
2513         int state;
2514         struct scst_device *dev = cmd->dev;
2515
2516         TRACE_ENTRY();
2517
2518         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2519
2520         if (likely(!scst_is_cmd_local(cmd)) &&
2521             likely(dev->handler->dev_done != NULL)) {
2522                 int rc;
2523
2524                 if (unlikely(!dev->handler->dev_done_atomic &&
2525                              scst_cmd_atomic(cmd))) {
2526                         /*
2527                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2528                          * optimization.
2529                          */
2530                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2531                               "context, rescheduling", dev->handler->name);
2532                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2533                         goto out;
2534                 }
2535
2536                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2537                       dev->handler->name, cmd);
2538                 rc = dev->handler->dev_done(cmd);
2539                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2540                       dev->handler->name, rc);
2541                 if (rc != SCST_CMD_STATE_DEFAULT)
2542                         state = rc;
2543         }
2544
2545         switch (state) {
2546         case SCST_CMD_STATE_PRE_XMIT_RESP:
2547         case SCST_CMD_STATE_DEV_PARSE:
2548         case SCST_CMD_STATE_PRE_PARSE:
2549         case SCST_CMD_STATE_PREPARE_SPACE:
2550         case SCST_CMD_STATE_RDY_TO_XFER:
2551         case SCST_CMD_STATE_TGT_PRE_EXEC:
2552         case SCST_CMD_STATE_SEND_FOR_EXEC:
2553         case SCST_CMD_STATE_LOCAL_EXEC:
2554         case SCST_CMD_STATE_REAL_EXEC:
2555         case SCST_CMD_STATE_PRE_DEV_DONE:
2556         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2557         case SCST_CMD_STATE_DEV_DONE:
2558         case SCST_CMD_STATE_XMIT_RESP:
2559         case SCST_CMD_STATE_FINISHED:
2560                 cmd->state = state;
2561                 break;
2562
2563         case SCST_CMD_STATE_NEED_THREAD_CTX:
2564                 TRACE_DBG("Dev handler %s dev_done() requested "
2565                       "thread context, rescheduling",
2566                       dev->handler->name);
2567                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2568                 break;
2569
2570         default:
2571                 if (state >= 0) {
2572                         PRINT_ERROR("Dev handler %s dev_done() returned "
2573                                 "invalid cmd state %d",
2574                                 dev->handler->name, state);
2575                 } else {
2576                         PRINT_ERROR("Dev handler %s dev_done() returned "
2577                                 "error %d", dev->handler->name,
2578                                 state);
2579                 }
2580                 scst_set_cmd_error(cmd,
2581                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2582                 scst_set_cmd_abnormal_done_state(cmd);
2583                 break;
2584         }
2585
2586         if (cmd->needs_unblocking)
2587                 scst_unblock_dev_cmd(cmd);
2588
2589         if (likely(cmd->dec_on_dev_needed))
2590                 scst_dec_on_dev_cmd(cmd);
2591
2592         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2593                 scst_inc_check_expected_sn(cmd);
2594
2595         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2596                 *pcmd = scst_complete_request_sense(cmd);
2597
2598 out:
2599         TRACE_EXIT_HRES(res);
2600         return res;
2601 }
2602
2603 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2604 {
2605         int res;
2606
2607         TRACE_ENTRY();
2608
2609 #ifdef CONFIG_SCST_DEBUG_TM
2610         if (cmd->tm_dbg_delayed &&
2611                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2612                 if (scst_cmd_atomic(cmd)) {
2613                         TRACE_MGMT_DBG("%s",
2614                                 "DEBUG_TM delayed cmd needs a thread");
2615                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2616                         return res;
2617                 }
2618                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2619                         cmd, cmd->tag);
2620                 schedule_timeout_uninterruptible(HZ);
2621         }
2622 #endif
2623
2624         if (likely(cmd->tgt_dev != NULL)) {
2625                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2626                 atomic_dec(&cmd->dev->dev_cmd_count);
2627                 /* If expected values not set, expected direction is UNKNOWN */
2628                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2629                         atomic_dec(&cmd->dev->write_cmd_count);
2630
2631                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2632                         scst_on_hq_cmd_response(cmd);
2633
2634                 if (unlikely(!cmd->sent_for_exec)) {
2635                         TRACE_SN("cmd %p was not sent to mid-lev"
2636                                 " (sn %ld, set %d)",
2637                                 cmd, cmd->sn, cmd->sn_set);
2638                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2639                         cmd->sent_for_exec = 1;
2640                 }
2641         }
2642
2643         /*
2644          * If we don't remove cmd from the search list here, before
2645          * submitting it for transmittion, we will have a race, when for
2646          * some reason cmd's release is delayed after transmittion and
2647          * initiator sends cmd with the same tag => it is possible that
2648          * a wrong cmd will be found by find() functions.
2649          */
2650         spin_lock_irq(&cmd->sess->sess_list_lock);
2651         list_del(&cmd->search_cmd_list_entry);
2652         spin_unlock_irq(&cmd->sess->sess_list_lock);
2653
2654         cmd->done = 1;
2655         smp_mb(); /* to sync with scst_abort_cmd() */
2656
2657         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2658                 scst_xmit_process_aborted_cmd(cmd);
2659
2660         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2661                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2662                                 " skipping",
2663                                 cmd, (long long unsigned int)cmd->tag);
2664                 cmd->state = SCST_CMD_STATE_FINISHED;
2665                 res = SCST_CMD_STATE_RES_CONT_SAME;
2666                 goto out;
2667         }
2668
2669         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2670         res = SCST_CMD_STATE_RES_CONT_SAME;
2671
2672 out:
2673 #ifdef CONFIG_SCST_MEASURE_LATENCY
2674         {
2675                 struct timespec ts;
2676                 uint64_t finish, scst_time, proc_time;
2677                 struct scst_session *sess = cmd->sess;
2678
2679                 getnstimeofday(&ts);
2680                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2681
2682                 spin_lock_bh(&sess->meas_lock);
2683
2684                 scst_time = cmd->pre_exec_finish - cmd->start;
2685                 scst_time += finish - cmd->post_exec_start;
2686                 proc_time = finish - cmd->start;
2687
2688                 sess->scst_time += scst_time;
2689                 sess->processing_time += proc_time;
2690                 sess->processed_cmds++;
2691
2692                 spin_unlock_bh(&sess->meas_lock);
2693
2694                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2695                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2696                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2697                         proc_time);
2698         }
2699 #endif
2700         TRACE_EXIT_HRES(res);
2701         return res;
2702 }
2703
2704 static int scst_xmit_response(struct scst_cmd *cmd)
2705 {
2706         int res, rc;
2707
2708         TRACE_ENTRY();
2709
2710         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2711                      scst_cmd_atomic(cmd))) {
2712                 /*
2713                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2714                  * optimization.
2715                  */
2716                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2717                               "context, rescheduling", cmd->tgtt->name);
2718                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2719                 goto out;
2720         }
2721
2722         while (1) {
2723                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2724
2725                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2726                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2727
2728                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2729
2730 #if defined(CONFIG_SCST_DEBUG)
2731                 if (cmd->sg) {
2732                         int i;
2733                         struct scatterlist *sg = cmd->sg;
2734                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at "
2735                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
2736                         for (i = 0; i < cmd->sg_cnt; ++i) {
2737                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2738                                     "Xmitting sg", sg_virt(&sg[i]),
2739                                     sg[i].length);
2740                         }
2741                 }
2742 #endif
2743
2744 #ifdef CONFIG_SCST_DEBUG_RETRY
2745                 if (((scst_random() % 100) == 77))
2746                         rc = SCST_TGT_RES_QUEUE_FULL;
2747                 else
2748 #endif
2749                         rc = cmd->tgtt->xmit_response(cmd);
2750                 TRACE_DBG("xmit_response() returned %d", rc);
2751
2752                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2753                         goto out;
2754
2755                 /* Restore the previous state */
2756                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2757
2758                 switch (rc) {
2759                 case SCST_TGT_RES_QUEUE_FULL:
2760                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2761                                 break;
2762                         else
2763                                 continue;
2764
2765                 case SCST_TGT_RES_NEED_THREAD_CTX:
2766                         TRACE_DBG("Target driver %s xmit_response() "
2767                               "requested thread context, rescheduling",
2768                               cmd->tgtt->name);
2769                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2770                         break;
2771
2772                 default:
2773                         goto out_error;
2774                 }
2775                 break;
2776         }
2777
2778 out:
2779         /* Caution: cmd can be already dead here */
2780         TRACE_EXIT_HRES(res);
2781         return res;
2782
2783 out_error:
2784         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2785                 PRINT_ERROR("Target driver %s xmit_response() returned "
2786                         "fatal error", cmd->tgtt->name);
2787         } else {
2788                 PRINT_ERROR("Target driver %s xmit_response() returned "
2789                         "invalid value %d", cmd->tgtt->name, rc);
2790         }
2791         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2792         cmd->state = SCST_CMD_STATE_FINISHED;
2793         res = SCST_CMD_STATE_RES_CONT_SAME;
2794         goto out;
2795 }
2796
2797 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2798 {
2799         TRACE_ENTRY();
2800
2801         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2802
2803         cmd->state = SCST_CMD_STATE_FINISHED;
2804         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2805
2806         TRACE_EXIT();
2807         return;
2808 }
2809 EXPORT_SYMBOL(scst_tgt_cmd_done);
2810
2811 static int scst_finish_cmd(struct scst_cmd *cmd)
2812 {
2813         int res;
2814
2815         TRACE_ENTRY();
2816
2817         atomic_dec(&cmd->sess->sess_cmd_count);
2818
2819         cmd->finished = 1;
2820         smp_mb(); /* to sync with scst_abort_cmd() */
2821
2822         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2823                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2824                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2825                         atomic_read(&scst_cmd_count));
2826
2827                 scst_finish_cmd_mgmt(cmd);
2828         }
2829
2830         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2831                 if ((cmd->tgt_dev != NULL) &&
2832                     scst_is_ua_sense(cmd->sense)) {
2833                         /* This UA delivery failed, so requeue it */
2834                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2835                                 "%p", cmd);
2836                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2837                                         SCST_SENSE_BUFFERSIZE, 1);
2838                 }
2839         }
2840
2841         __scst_cmd_put(cmd);
2842
2843         res = SCST_CMD_STATE_RES_CONT_NEXT;
2844
2845         TRACE_EXIT_HRES(res);
2846         return res;
2847 }
2848
2849 /*
2850  * No locks, but it must be externally serialized (see comment for
2851  * scst_cmd_init_done() in scst.h)
2852  */
2853 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2854 {
2855         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2856         unsigned long flags;
2857
2858         TRACE_ENTRY();
2859
2860         if (scst_is_implicit_hq(cmd)) {
2861                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING,
2862                         "Implicit HQ cmd %p", cmd);
2863                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2864         }
2865
2866         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2867
2868         /* Optimized for lockless fast path */
2869
2870         scst_check_debug_sn(cmd);
2871
2872         if (cmd->dev->queue_alg ==
2873                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2874                 /*
2875                  * Not the best way, but well enough until there will be a
2876                  * possibility to specify queue type during pass-through
2877                  * commands submission.
2878                  */
2879                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2880         }
2881
2882         switch (cmd->queue_type) {
2883         case SCST_CMD_QUEUE_SIMPLE:
2884         case SCST_CMD_QUEUE_UNTAGGED:
2885 #if 1 /* temporary, ToDo */
2886                 if (scst_cmd_is_expected_set(cmd)) {
2887                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2888                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2889                                 goto ordered;
2890                 } else
2891                         goto ordered;
2892 #endif
2893                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2894                         /*
2895                          * atomic_inc_return() implies memory barrier to sync
2896                          * with scst_inc_expected_sn()
2897                          */
2898                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2899                                 tgt_dev->curr_sn++;
2900                                 TRACE_SN("Incremented curr_sn %ld",
2901                                         tgt_dev->curr_sn);
2902                         }
2903                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2904                         cmd->sn = tgt_dev->curr_sn;
2905
2906                         tgt_dev->prev_cmd_ordered = 0;
2907                 } else {
2908                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2909                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2910                         goto ordered;
2911                 }
2912                 break;
2913
2914         case SCST_CMD_QUEUE_ORDERED:
2915                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2916                         "(op %x)", cmd, cmd->cdb[0]);
2917 ordered:
2918                 if (!tgt_dev->prev_cmd_ordered) {
2919                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2920                         if (tgt_dev->num_free_sn_slots >= 0) {
2921                                 tgt_dev->num_free_sn_slots--;
2922                                 if (tgt_dev->num_free_sn_slots >= 0) {
2923                                         int i = 0;
2924                                         /* Commands can finish in any order, so
2925                                          * we don't know which slot is empty.
2926                                          */
2927                                         while (1) {
2928                                                 tgt_dev->cur_sn_slot++;
2929                                                 if (tgt_dev->cur_sn_slot ==
2930                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
2931                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2932
2933                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2934                                                         break;
2935
2936                                                 i++;
2937                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2938                                         }
2939                                         TRACE_SN("New cur SN slot %zd",
2940                                                 tgt_dev->cur_sn_slot -
2941                                                 tgt_dev->sn_slots);
2942                                 }
2943                         }
2944                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2945                 }
2946                 tgt_dev->prev_cmd_ordered = 1;
2947                 tgt_dev->curr_sn++;
2948                 cmd->sn = tgt_dev->curr_sn;
2949                 break;
2950
2951         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2952                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2953                         "(op %x)", cmd, cmd->cdb[0]);
2954                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2955                 tgt_dev->hq_cmd_count++;
2956                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2957                 cmd->hq_cmd_inced = 1;
2958                 goto out;
2959
2960         default:
2961                 sBUG();
2962         }
2963
2964         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2965                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2966                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2967                 atomic_read(tgt_dev->cur_sn_slot),
2968                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2969                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2970
2971         cmd->sn_set = 1;
2972
2973 out:
2974         TRACE_EXIT();
2975         return;
2976 }
2977
2978 /*
2979  * Returns 0 on success, > 0 when we need to wait for unblock,
2980  * < 0 if there is no device (lun) or device type handler.
2981  *
2982  * No locks, but might be on IRQ, protection is done by the
2983  * suspended activity.
2984  */
2985 static int scst_translate_lun(struct scst_cmd *cmd)
2986 {
2987         struct scst_tgt_dev *tgt_dev = NULL;
2988         int res;
2989
2990         TRACE_ENTRY();
2991
2992         __scst_get(1);
2993
2994         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2995                 struct list_head *sess_tgt_dev_list_head =
2996                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2997                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
2998                         (long long unsigned int)cmd->lun);
2999                 res = -1;
3000                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3001                                 sess_tgt_dev_list_entry) {
3002                         if (tgt_dev->lun == cmd->lun) {
3003                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3004
3005                                 if (unlikely(tgt_dev->dev->handler ==
3006                                                 &scst_null_devtype)) {
3007                                         PRINT_INFO("Dev handler for device "
3008                                           "%lld is NULL, the device will not "
3009                                           "be visible remotely",
3010                                            (long long unsigned int)cmd->lun);
3011                                         break;
3012                                 }
3013
3014                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3015                                 cmd->tgt_dev = tgt_dev;
3016                                 cmd->dev = tgt_dev->dev;
3017
3018                                 res = 0;
3019                                 break;
3020                         }
3021                 }
3022                 if (res != 0) {
3023                         TRACE(TRACE_MINOR,
3024                                 "tgt_dev for lun %lld not found, command to "
3025                                 "unexisting LU?",
3026                                 (long long unsigned int)cmd->lun);
3027                         __scst_put();
3028                 }
3029         } else {
3030                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3031                 __scst_put();
3032                 res = 1;
3033         }
3034
3035         TRACE_EXIT_RES(res);
3036         return res;
3037 }
3038
3039 /*
3040  * No locks, but might be on IRQ
3041  *
3042  * Returns 0 on success, > 0 when we need to wait for unblock,
3043  * < 0 if there is no device (lun) or device type handler.
3044  */
3045 static int __scst_init_cmd(struct scst_cmd *cmd)
3046 {
3047         int res = 0;
3048
3049         TRACE_ENTRY();
3050
3051         res = scst_translate_lun(cmd);
3052         if (likely(res == 0)) {
3053                 int cnt;
3054                 bool failure = false;
3055
3056                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3057
3058                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3059                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3060                         TRACE(TRACE_MGMT_MINOR,
3061                                 "Too many pending commands (%d) in "
3062                                 "session, returning BUSY to initiator \"%s\"",
3063                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3064                                   "Anonymous" : cmd->sess->initiator_name);
3065                         failure = true;
3066                 }
3067
3068                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3069                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3070                         if (!failure) {
3071                                 TRACE(TRACE_MGMT_MINOR,
3072                                         "Too many pending device "
3073                                         "commands (%d), returning BUSY to "
3074                                         "initiator \"%s\"", cnt,
3075                                         (cmd->sess->initiator_name[0] == '\0') ?
3076                                                 "Anonymous" :
3077                                                 cmd->sess->initiator_name);
3078                                 failure = true;
3079                         }
3080                 }
3081
3082                 /* If expected values not set, expected direction is UNKNOWN */
3083                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3084                         atomic_inc(&cmd->dev->write_cmd_count);
3085
3086                 if (unlikely(failure))
3087                         goto out_busy;
3088
3089                 if (!cmd->set_sn_on_restart_cmd)
3090                         scst_cmd_set_sn(cmd);
3091         } else if (res < 0) {
3092                 TRACE_DBG("Finishing cmd %p", cmd);
3093                 scst_set_cmd_error(cmd,
3094                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3095                 scst_set_cmd_abnormal_done_state(cmd);
3096         } else
3097                 goto out;
3098
3099 out:
3100         TRACE_EXIT_RES(res);
3101         return res;
3102
3103 out_busy:
3104         scst_set_busy(cmd);
3105         scst_set_cmd_abnormal_done_state(cmd);
3106         goto out;
3107 }
3108
3109 /* Called under scst_init_lock and IRQs disabled */
3110 static void scst_do_job_init(void)
3111 {
3112         struct scst_cmd *cmd;
3113         int susp;
3114
3115         TRACE_ENTRY();
3116
3117 restart:
3118         /*
3119          * There is no need for read barrier here, because we don't care where
3120          * this check will be done.
3121          */
3122         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3123         if (scst_init_poll_cnt > 0)
3124                 scst_init_poll_cnt--;
3125
3126         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3127                 int rc;
3128                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3129                         continue;
3130                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3131                         spin_unlock_irq(&scst_init_lock);
3132                         rc = __scst_init_cmd(cmd);
3133                         spin_lock_irq(&scst_init_lock);
3134                         if (rc > 0) {
3135                                 TRACE_MGMT_DBG("%s",
3136                                         "FLAG SUSPENDED set, restarting");
3137                                 goto restart;
3138                         }
3139                 } else {
3140                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3141                                        cmd, (long long unsigned int)cmd->tag);
3142                         scst_set_cmd_abnormal_done_state(cmd);
3143                 }
3144
3145                 /*
3146                  * Deleting cmd from init cmd list after __scst_init_cmd()
3147                  * is necessary to keep the check in scst_init_cmd() correct
3148                  * to preserve the commands order.
3149                  *
3150                  * We don't care about the race, when init cmd list is empty
3151                  * and one command detected that it just was not empty, so
3152                  * it's inserting to it, but another command at the same time
3153                  * seeing init cmd list empty and goes directly, because it
3154                  * could affect only commands from the same initiator to the
3155                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3156                  * in case of simultaneous such calls anyway.
3157                  */
3158                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3159                 smp_wmb();
3160                 list_del(&cmd->cmd_list_entry);
3161                 spin_unlock(&scst_init_lock);
3162
3163                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3164                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3165                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3166                         list_add(&cmd->cmd_list_entry,
3167                                 &cmd->cmd_lists->active_cmd_list);
3168                 else
3169                         list_add_tail(&cmd->cmd_list_entry,
3170                                 &cmd->cmd_lists->active_cmd_list);
3171                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3172                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3173
3174                 spin_lock(&scst_init_lock);
3175                 goto restart;
3176         }
3177
3178         /* It isn't really needed, but let's keep it */
3179         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3180                 goto restart;
3181
3182         TRACE_EXIT();
3183         return;
3184 }
3185
3186 static inline int test_init_cmd_list(void)
3187 {
3188         int res = (!list_empty(&scst_init_cmd_list) &&
3189                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3190                   unlikely(kthread_should_stop()) ||
3191                   (scst_init_poll_cnt > 0);
3192         return res;
3193 }
3194
3195 int scst_init_cmd_thread(void *arg)
3196 {
3197         TRACE_ENTRY();
3198
3199         PRINT_INFO("Init thread started, PID %d", current->pid);
3200
3201         current->flags |= PF_NOFREEZE;
3202
3203         set_user_nice(current, -10);
3204
3205         spin_lock_irq(&scst_init_lock);
3206         while (!kthread_should_stop()) {
3207                 wait_queue_t wait;
3208                 init_waitqueue_entry(&wait, current);
3209
3210                 if (!test_init_cmd_list()) {
3211                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3212                                                  &wait);
3213                         for (;;) {
3214                                 set_current_state(TASK_INTERRUPTIBLE);
3215                                 if (test_init_cmd_list())
3216                                         break;
3217                                 spin_unlock_irq(&scst_init_lock);
3218                                 schedule();
3219                                 spin_lock_irq(&scst_init_lock);
3220                         }
3221                         set_current_state(TASK_RUNNING);
3222                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3223                 }
3224                 scst_do_job_init();
3225         }
3226         spin_unlock_irq(&scst_init_lock);
3227
3228         /*
3229          * If kthread_should_stop() is true, we are guaranteed to be
3230          * on the module unload, so scst_init_cmd_list must be empty.
3231          */
3232         sBUG_ON(!list_empty(&scst_init_cmd_list));
3233
3234         PRINT_INFO("Init thread PID %d finished", current->pid);
3235
3236         TRACE_EXIT();
3237         return 0;
3238 }
3239
3240 /* Called with no locks held */
3241 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3242 {
3243         int res;
3244
3245         TRACE_ENTRY();
3246
3247         EXTRACHECKS_BUG_ON(in_irq());
3248
3249         cmd->context_processable = context | SCST_CONTEXT_PROCESSABLE;
3250         context &= ~SCST_CONTEXT_PROCESSABLE;
3251         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3252
3253         TRACE_DBG("cmd %p, context_processable %d, atomic %d", cmd,
3254                 cmd->context_processable, cmd->atomic);
3255
3256         do {
3257                 switch (cmd->state) {
3258                 case SCST_CMD_STATE_PRE_PARSE:
3259                         res = scst_pre_parse(cmd);
3260                         EXTRACHECKS_BUG_ON(res ==
3261                                 SCST_CMD_STATE_RES_NEED_THREAD);
3262                         break;
3263
3264                 case SCST_CMD_STATE_DEV_PARSE:
3265                         res = scst_parse_cmd(cmd);
3266                         break;
3267
3268                 case SCST_CMD_STATE_PREPARE_SPACE:
3269                         res = scst_prepare_space(cmd);
3270                         break;
3271
3272                 case SCST_CMD_STATE_RDY_TO_XFER:
3273                         res = scst_rdy_to_xfer(cmd);
3274                         break;
3275
3276                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3277                         res = scst_tgt_pre_exec(cmd);
3278                         break;
3279
3280                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3281                         if (tm_dbg_check_cmd(cmd) != 0) {
3282                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3283                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3284                                         "because of TM DBG delay", cmd,
3285                                         (long long unsigned int)cmd->tag);
3286                                 break;
3287                         }
3288                         res = scst_send_for_exec(&cmd);
3289                         /*
3290                          * !! At this point cmd, sess & tgt_dev can already be
3291                          * freed !!
3292                          */
3293                         break;
3294
3295                 case SCST_CMD_STATE_LOCAL_EXEC:
3296                         res = scst_local_exec(cmd);
3297                         /*
3298                          * !! At this point cmd, sess & tgt_dev can already be
3299                          * freed !!
3300                          */
3301                         break;
3302
3303                 case SCST_CMD_STATE_REAL_EXEC:
3304                         res = scst_real_exec(cmd);
3305                         /*
3306                          * !! At this point cmd, sess & tgt_dev can already be
3307                          * freed !!
3308                          */
3309                         break;
3310
3311                 case SCST_CMD_STATE_PRE_DEV_DONE:
3312                         res = scst_pre_dev_done(cmd);
3313                         EXTRACHECKS_BUG_ON(res ==
3314                                 SCST_CMD_STATE_RES_NEED_THREAD);
3315                         break;
3316
3317                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3318                         res = scst_mode_select_checks(cmd);
3319                         break;
3320
3321                 case SCST_CMD_STATE_DEV_DONE:
3322                         res = scst_dev_done(&cmd);
3323                         break;
3324
3325                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3326                         res = scst_pre_xmit_response(cmd);
3327                         EXTRACHECKS_BUG_ON(res ==
3328                                 SCST_CMD_STATE_RES_NEED_THREAD);
3329                         break;
3330
3331                 case SCST_CMD_STATE_XMIT_RESP:
3332                         res = scst_xmit_response(cmd);
3333                         break;
3334
3335                 case SCST_CMD_STATE_FINISHED:
3336                         res = scst_finish_cmd(cmd);
3337                         EXTRACHECKS_BUG_ON(res ==
3338                                 SCST_CMD_STATE_RES_NEED_THREAD);
3339                         break;
3340
3341                 default:
3342                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3343                                 "be", cmd, cmd->state);
3344                         sBUG();
3345                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3346                         break;
3347                 }
3348         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3349
3350         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3351                 /* None */
3352         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3353                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3354                 switch (cmd->state) {
3355                 case SCST_CMD_STATE_PRE_PARSE:
3356                 case SCST_CMD_STATE_DEV_PARSE:
3357                 case SCST_CMD_STATE_PREPARE_SPACE:
3358                 case SCST_CMD_STATE_RDY_TO_XFER:
3359                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3360                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3361                 case SCST_CMD_STATE_LOCAL_EXEC:
3362                 case SCST_CMD_STATE_REAL_EXEC:
3363                 case SCST_CMD_STATE_PRE_DEV_DONE:
3364                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3365                 case SCST_CMD_STATE_DEV_DONE:
3366                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3367                 case SCST_CMD_STATE_XMIT_RESP:
3368                 case SCST_CMD_STATE_FINISHED:
3369                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3370                                   cmd);
3371                         list_add(&cmd->cmd_list_entry,
3372                                 &cmd->cmd_lists->active_cmd_list);
3373                         break;
3374 #ifdef CONFIG_SCST_EXTRACHECKS
3375                 /* not very valid commands */
3376                 case SCST_CMD_STATE_DEFAULT:
3377                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3378                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3379                                 cmd->state);
3380                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3381                         sBUG();
3382                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3383                         break;
3384 #endif
3385                 default:
3386                         break;
3387                 }
3388                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3389                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3390         } else
3391                 sBUG();
3392
3393         TRACE_EXIT();
3394         return;
3395 }
3396 EXPORT_SYMBOL(scst_process_active_cmd);
3397
3398 /* Called under cmd_list_lock and IRQs disabled */
3399 static void scst_do_job_active(struct list_head *cmd_list,
3400        spinlock_t *cmd_list_lock, int context)
3401 {
3402         TRACE_ENTRY();
3403
3404 #ifdef CONFIG_SCST_EXTRACHECKS
3405         {
3406                 int c = context & ~SCST_CONTEXT_PROCESSABLE;
3407                 sBUG_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) &&
3408                         (c != SCST_CONTEXT_DIRECT));
3409         }
3410 #endif
3411
3412         while (!list_empty(cmd_list)) {
3413                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3414                                         cmd_list_entry);
3415                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3416                 list_del(&cmd->cmd_list_entry);
3417                 spin_unlock_irq(cmd_list_lock);
3418                 scst_process_active_cmd(cmd, context);
3419                 spin_lock_irq(cmd_list_lock);
3420         }
3421
3422         TRACE_EXIT();
3423         return;
3424 }
3425
3426 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3427 {
3428         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3429             unlikely(kthread_should_stop()) ||
3430             tm_dbg_is_release();
3431         return res;
3432 }
3433
3434 int scst_cmd_thread(void *arg)
3435 {
3436         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3437
3438         TRACE_ENTRY();
3439
3440         PRINT_INFO("Processing thread started, PID %d", current->pid);
3441
3442 #if 0
3443         set_user_nice(current, 10);
3444 #endif
3445         current->flags |= PF_NOFREEZE;
3446
3447         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3448         while (!kthread_should_stop()) {
3449                 wait_queue_t wait;
3450                 init_waitqueue_entry(&wait, current);
3451
3452                 if (!test_cmd_lists(p_cmd_lists)) {
3453                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3454                                 &wait);
3455                         for (;;) {
3456                                 set_current_state(TASK_INTERRUPTIBLE);
3457                                 if (test_cmd_lists(p_cmd_lists))
3458                                         break;
3459                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3460                                 schedule();
3461                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3462                         }
3463                         set_current_state(TASK_RUNNING);
3464                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3465                 }
3466
3467                 if (tm_dbg_is_release()) {
3468                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3469                         tm_dbg_check_released_cmds();
3470                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3471                 }
3472
3473                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3474                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT |
3475                                                 SCST_CONTEXT_PROCESSABLE);
3476         }
3477         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3478
3479 #ifdef CONFIG_SCST_EXTRACHECKS
3480         /*
3481          * If kthread_should_stop() is true, we are guaranteed to be either
3482          * on the module unload, or there must be at least one other thread to
3483          * process the commands lists.
3484          */
3485         if (p_cmd_lists == &scst_main_cmd_lists) {
3486                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3487                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3488         }
3489 #endif
3490
3491         PRINT_INFO("Processing thread PID %d finished", current->pid);
3492
3493         TRACE_EXIT();
3494         return 0;
3495 }
3496
3497 void scst_cmd_tasklet(long p)
3498 {
3499         struct scst_tasklet *t = (struct scst_tasklet *)p;
3500
3501         TRACE_ENTRY();
3502
3503         spin_lock_irq(&t->tasklet_lock);
3504         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3505                 SCST_CONTEXT_DIRECT_ATOMIC);
3506         spin_unlock_irq(&t->tasklet_lock);
3507
3508         TRACE_EXIT();
3509        &nbs