Let's don't check if data should be copied between dev handler's and target driver...
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31 #include <linux/ktime.h>
32
33 #include "scst.h"
34 #include "scst_priv.h"
35
36 static void scst_cmd_set_sn(struct scst_cmd *cmd);
37 static int __scst_init_cmd(struct scst_cmd *cmd);
38 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
39 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
40         uint64_t tag);
41 static void scst_process_redirect_cmd(struct scst_cmd *cmd,
42         enum scst_exec_context context, int check_retries);
43
44 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
45 {
46         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
47         unsigned long flags;
48
49         spin_lock_irqsave(&t->tasklet_lock, flags);
50         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
51                 smp_processor_id());
52         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
53         spin_unlock_irqrestore(&t->tasklet_lock, flags);
54
55         tasklet_schedule(&t->tasklet);
56 }
57
58 /*
59  * Must not be called in parallel with scst_unregister_session() for the
60  * same sess
61  */
62 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
63                              const uint8_t *lun, int lun_len,
64                              const uint8_t *cdb, int cdb_len, int atomic)
65 {
66         struct scst_cmd *cmd;
67
68         TRACE_ENTRY();
69
70 #ifdef CONFIG_SCST_EXTRACHECKS
71         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
72                 PRINT_CRIT_ERROR("%s",
73                         "New cmd while shutting down the session");
74                 sBUG();
75         }
76 #endif
77
78         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
79         if (cmd == NULL)
80                 goto out;
81
82         cmd->sess = sess;
83         cmd->tgt = sess->tgt;
84         cmd->tgtt = sess->tgt->tgtt;
85
86         /*
87          * For both wrong lun and CDB defer the error reporting for
88          * scst_cmd_init_done()
89          */
90
91         cmd->lun = scst_unpack_lun(lun, lun_len);
92
93         if (cdb_len <= SCST_MAX_CDB_SIZE) {
94                 memcpy(cmd->cdb, cdb, cdb_len);
95                 cmd->cdb_len = cdb_len;
96         }
97
98         TRACE_DBG("cmd %p, sess %p", cmd, sess);
99         scst_sess_get(sess);
100
101 out:
102         TRACE_EXIT();
103         return cmd;
104 }
105 EXPORT_SYMBOL(scst_rx_cmd);
106
107 /*
108  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
109  * this command should be stopped.
110  */
111 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
112 {
113         int rc, res = 0;
114
115         TRACE_ENTRY();
116
117         /* See the comment in scst_do_job_init() */
118         if (unlikely(!list_empty(&scst_init_cmd_list))) {
119                 TRACE_MGMT_DBG("%s", "init cmd list busy");
120                 goto out_redirect;
121         }
122         /*
123          * Memory barrier isn't necessary here, because CPU appears to
124          * be self-consistent and we don't care about the race, described
125          * in comment in scst_do_job_init().
126          */
127
128         rc = __scst_init_cmd(cmd);
129         if (unlikely(rc > 0))
130                 goto out_redirect;
131         else if (unlikely(rc != 0)) {
132                 res = 1;
133                 goto out;
134         }
135
136         /* Small context optimization */
137         if (((*context == SCST_CONTEXT_TASKLET) ||
138              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
139              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
140               scst_cmd_is_expected_set(cmd)) {
141                 if (cmd->expected_data_direction & SCST_DATA_WRITE) {
142                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
143                                         &cmd->tgt_dev->tgt_dev_flags))
144                                 *context = SCST_CONTEXT_THREAD;
145                 } else {
146                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
147                                         &cmd->tgt_dev->tgt_dev_flags))
148                                 *context = SCST_CONTEXT_THREAD;
149                 }
150         }
151
152 out:
153         TRACE_EXIT_RES(res);
154         return res;
155
156 out_redirect:
157         if (cmd->preprocessing_only) {
158                 /*
159                  * Poor man solution for single threaded targets, where
160                  * blocking receiver at least sometimes means blocking all.
161                  */
162                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
163                 scst_set_busy(cmd);
164                 scst_set_cmd_abnormal_done_state(cmd);
165                 res = 1;
166                 /* Keep initiator away from too many BUSY commands */
167                 msleep(50);
168         } else {
169                 unsigned long flags;
170                 spin_lock_irqsave(&scst_init_lock, flags);
171                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
172                         "%d)", cmd, atomic_read(&scst_cmd_count));
173                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
174                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
175                         scst_init_poll_cnt++;
176                 spin_unlock_irqrestore(&scst_init_lock, flags);
177                 wake_up(&scst_init_cmd_list_waitQ);
178                 res = -1;
179         }
180         goto out;
181 }
182
183 void scst_cmd_init_done(struct scst_cmd *cmd,
184         enum scst_exec_context pref_context)
185 {
186         unsigned long flags;
187         struct scst_session *sess = cmd->sess;
188         int rc;
189
190         TRACE_ENTRY();
191
192         scst_set_start_time(cmd);
193
194         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
195         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
196                 "(cmd %p)", (long long unsigned int)cmd->tag,
197                 (long long unsigned int)cmd->lun, cmd->cdb_len,
198                 cmd->queue_type, cmd);
199         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
200                 cmd->cdb, cmd->cdb_len);
201
202 #ifdef CONFIG_SCST_EXTRACHECKS
203         if (unlikely((in_irq() || irqs_disabled())) &&
204             ((pref_context == SCST_CONTEXT_DIRECT) ||
205              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
206                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
207                         "SCST_CONTEXT_THREAD instead", pref_context,
208                         cmd->tgtt->name);
209                 pref_context = SCST_CONTEXT_THREAD;
210         }
211 #endif
212
213         atomic_inc(&sess->sess_cmd_count);
214
215         spin_lock_irqsave(&sess->sess_list_lock, flags);
216
217         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
218                 /*
219                  * We have to always keep command in the search list from the
220                  * very beginning, because otherwise it can be missed during
221                  * TM processing. This check is needed because there might be
222                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
223                  */
224                 if (cmd->sess_cmd_list_entry.next == NULL)
225                         list_add_tail(&cmd->sess_cmd_list_entry,
226                                 &sess->search_cmd_list);
227                 switch (sess->init_phase) {
228                 case SCST_SESS_IPH_SUCCESS:
229                         break;
230                 case SCST_SESS_IPH_INITING:
231                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
232                                   cmd);
233                         list_add_tail(&cmd->cmd_list_entry,
234                                 &sess->init_deferred_cmd_list);
235                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
236                         goto out;
237                 case SCST_SESS_IPH_FAILED:
238                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
239                         scst_set_busy(cmd);
240                         scst_set_cmd_abnormal_done_state(cmd);
241                         goto active;
242                 default:
243                         sBUG();
244                 }
245         } else
246                 list_add_tail(&cmd->sess_cmd_list_entry,
247                               &sess->search_cmd_list);
248
249         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
250
251         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
252                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
253                 scst_set_cmd_error(cmd,
254                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
255                 scst_set_cmd_abnormal_done_state(cmd);
256                 goto active;
257         }
258
259         if (unlikely(cmd->cdb_len == 0)) {
260                 PRINT_ERROR("%s", "Wrong CDB len, finishing cmd");
261                 scst_set_cmd_error(cmd,
262                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
263                 scst_set_cmd_abnormal_done_state(cmd);
264                 goto active;
265         }
266
267         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
268                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
269                 scst_set_cmd_error(cmd,
270                         SCST_LOAD_SENSE(scst_sense_invalid_message));
271                 scst_set_cmd_abnormal_done_state(cmd);
272                 goto active;
273         }
274
275         /*
276          * Cmd must be inited here to preserve the order. In case if cmd
277          * already preliminary completed by target driver we need to init
278          * cmd anyway to find out in which format we should return sense.
279          */
280         cmd->state = SCST_CMD_STATE_INIT;
281         rc = scst_init_cmd(cmd, &pref_context);
282         if (unlikely(rc < 0))
283                 goto out;
284         else if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
285                 if (rc == 0) {
286                         /* Target driver preliminary completed cmd */
287                         scst_set_cmd_abnormal_done_state(cmd);
288                 }
289         }
290
291 active:
292         /* Here cmd must not be in any cmd list, no locks */
293         switch (pref_context) {
294         case SCST_CONTEXT_TASKLET:
295                 scst_schedule_tasklet(cmd);
296                 break;
297
298         case SCST_CONTEXT_DIRECT:
299                 scst_process_active_cmd(cmd, false);
300                 /* For *NEED_THREAD wake_up() is already done */
301                 break;
302
303         case SCST_CONTEXT_DIRECT_ATOMIC:
304                 scst_process_active_cmd(cmd, true);
305                 /* For *NEED_THREAD wake_up() is already done */
306                 break;
307
308         default:
309                 PRINT_ERROR("Context %x is undefined, using the thread one",
310                         pref_context);
311                 /* go through */
312         case SCST_CONTEXT_THREAD:
313                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
314                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
315                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
316                         list_add(&cmd->cmd_list_entry,
317                                 &cmd->cmd_lists->active_cmd_list);
318                 else
319                         list_add_tail(&cmd->cmd_list_entry,
320                                 &cmd->cmd_lists->active_cmd_list);
321                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
322                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
323                 break;
324         }
325
326 out:
327         TRACE_EXIT();
328         return;
329 }
330 EXPORT_SYMBOL(scst_cmd_init_done);
331
332 static int scst_pre_parse(struct scst_cmd *cmd)
333 {
334         int res = SCST_CMD_STATE_RES_CONT_SAME;
335         struct scst_device *dev = cmd->dev;
336         int rc;
337
338         TRACE_ENTRY();
339
340         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
341              (!dev->has_own_order_mgmt &&
342               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
343                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
344
345         /*
346          * Expected transfer data supplied by the SCSI transport via the
347          * target driver are untrusted, so we prefer to fetch them from CDB.
348          * Additionally, not all transports support supplying the expected
349          * transfer data.
350          */
351
352         rc = scst_get_cdb_info(cmd);
353         if (unlikely(rc != 0)) {
354                 if (rc > 0) {
355                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
356                         goto out_xmit;
357                 }
358                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
359                         "Should you update scst_scsi_op_table?",
360                         cmd->cdb[0], dev->handler->name);
361                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
362 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
363                 if (scst_cmd_is_expected_set(cmd)) {
364                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
365                                 "direction %d, transfer_len %d",
366                                 cmd->expected_data_direction,
367                                 cmd->expected_transfer_len);
368                         cmd->data_direction = cmd->expected_data_direction;
369
370                         cmd->bufflen = cmd->expected_transfer_len;
371                         /* Restore (possibly) lost CDB length */
372                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
373                         if (cmd->cdb_len == -1) {
374                                 PRINT_ERROR("Unable to get CDB length for "
375                                         "opcode 0x%02x. Returning INVALID "
376                                         "OPCODE", cmd->cdb[0]);
377                                 scst_set_cmd_error(cmd,
378                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
379                                 goto out_xmit;
380                         }
381                 } else {
382                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
383                              "target %s not supplied expected values",
384                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
385                         scst_set_cmd_error(cmd,
386                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
387                         goto out_xmit;
388                 }
389 #else
390                 scst_set_cmd_error(cmd,
391                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
392                 goto out_xmit;
393 #endif
394         } else {
395                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
396                         "(expected %d, set %s), transfer_len=%d (expected "
397                         "len %d), flags=%d", cmd->op_name, cmd,
398                         cmd->data_direction, cmd->expected_data_direction,
399                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
400                         cmd->bufflen, cmd->expected_transfer_len,
401                         cmd->op_flags);
402
403                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
404                         if (scst_cmd_is_expected_set(cmd)) {
405                                 /*
406                                  * Command data length can't be easily
407                                  * determined from the CDB. ToDo, all such
408                                  * commands processing should be fixed. Until
409                                  * it's done, get the length from the supplied
410                                  * expected value, but limit it to some
411                                  * reasonable value (15MB).
412                                  */
413                                 cmd->bufflen = min(cmd->expected_transfer_len,
414                                                         15*1024*1024);
415                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
416                         } else
417                                 cmd->bufflen = 0;
418                 }
419         }
420
421         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
422                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
423                             "(opcode 0x%02x)", cmd->cdb[0]);
424                 scst_set_cmd_error(cmd,
425                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
426                 goto out_xmit;
427         }
428
429         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
430                 PRINT_ERROR("Linked commands are not supported "
431                             "(opcode 0x%02x)", cmd->cdb[0]);
432                 scst_set_cmd_error(cmd,
433                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
434                 goto out_xmit;
435         }
436
437         cmd->state = SCST_CMD_STATE_DEV_PARSE;
438
439 out:
440         TRACE_EXIT_RES(res);
441         return res;
442
443 out_xmit:
444         scst_set_cmd_abnormal_done_state(cmd);
445         res = SCST_CMD_STATE_RES_CONT_SAME;
446         goto out;
447 }
448
449 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
450 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
451 {
452         bool res = false;
453
454         switch (cmd->cdb[0]) {
455         case TEST_UNIT_READY:
456                 /* Crazy VMware people sometimes do TUR with READ direction */
457                 res = true;
458                 break;
459         case VERIFY:
460         case VERIFY_6:
461         case VERIFY_12:
462         case VERIFY_16:
463                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
464                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
465                     (cmd->cdb[1] & BYTCHK) == 0)
466                         res = true;
467                 break;
468         }
469
470         return res;
471 }
472 #endif
473
474 static int scst_parse_cmd(struct scst_cmd *cmd)
475 {
476         int res = SCST_CMD_STATE_RES_CONT_SAME;
477         int state;
478         struct scst_device *dev = cmd->dev;
479         int orig_bufflen = cmd->bufflen;
480
481         TRACE_ENTRY();
482
483         if (likely(!scst_is_cmd_local(cmd))) {
484                 if (unlikely(!dev->handler->parse_atomic &&
485                              scst_cmd_atomic(cmd))) {
486                         /*
487                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
488                          * optimization.
489                          */
490                         TRACE_DBG("Dev handler %s parse() needs thread "
491                                 "context, rescheduling", dev->handler->name);
492                         res = SCST_CMD_STATE_RES_NEED_THREAD;
493                         goto out;
494                 }
495
496                 TRACE_DBG("Calling dev handler %s parse(%p)",
497                       dev->handler->name, cmd);
498                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
499                                 cmd->cdb, cmd->cdb_len);
500                 scst_set_cur_start(cmd);
501                 state = dev->handler->parse(cmd);
502                 /* Caution: cmd can be already dead here */
503                 TRACE_DBG("Dev handler %s parse() returned %d",
504                         dev->handler->name, state);
505
506                 switch (state) {
507                 case SCST_CMD_STATE_NEED_THREAD_CTX:
508                         scst_set_parse_time(cmd);
509                         TRACE_DBG("Dev handler %s parse() requested thread "
510                               "context, rescheduling", dev->handler->name);
511                         res = SCST_CMD_STATE_RES_NEED_THREAD;
512                         goto out;
513
514                 case SCST_CMD_STATE_STOP:
515                         TRACE_DBG("Dev handler %s parse() requested stop "
516                                 "processing", dev->handler->name);
517                         res = SCST_CMD_STATE_RES_CONT_NEXT;
518                         goto out;
519                 }
520
521                 scst_set_parse_time(cmd);
522
523                 if (state == SCST_CMD_STATE_DEFAULT)
524                         state = SCST_CMD_STATE_PREPARE_SPACE;
525         } else
526                 state = SCST_CMD_STATE_PREPARE_SPACE;
527
528         if (cmd->data_len == -1)
529                 cmd->data_len = cmd->bufflen;
530
531         if (cmd->bufflen == 0) {
532                 /*
533                  * According to SPC bufflen 0 for data transfer commands isn't
534                  * an error, so we need to fix the transfer direction.
535                  */
536                 cmd->data_direction = SCST_DATA_NONE;
537         }
538
539         if (cmd->dh_data_buf_alloced &&
540             unlikely((orig_bufflen > cmd->bufflen))) {
541                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
542                         "is less, than required (size %d)", cmd->bufflen,
543                         orig_bufflen);
544                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
545                 goto out_error;
546         }
547
548         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
549                 goto set_res;
550
551         if (unlikely((cmd->bufflen == 0) &&
552                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
553                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
554                         "(handler %s, target %s)", cmd->cdb[0],
555                         dev->handler->name, cmd->tgtt->name);
556                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
557                 goto out_error;
558         }
559
560 #ifdef CONFIG_SCST_EXTRACHECKS
561         if ((cmd->bufflen != 0) &&
562             ((cmd->data_direction == SCST_DATA_NONE) ||
563              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
564                 PRINT_ERROR("Dev handler %s parse() returned "
565                         "invalid cmd data_direction %d, bufflen %d, state %d "
566                         "or sg %p (opcode 0x%x)", dev->handler->name,
567                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
568                         cmd->cdb[0]);
569                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
570                 goto out_error;
571         }
572 #endif
573
574         if (scst_cmd_is_expected_set(cmd)) {
575 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
576 #       ifdef CONFIG_SCST_EXTRACHECKS
577                 if ((cmd->data_direction != cmd->expected_data_direction) ||
578                     (cmd->bufflen != cmd->expected_transfer_len)) {
579                         PRINT_WARNING("Expected values don't match decoded "
580                                 "ones: data_direction %d, "
581                                 "expected_data_direction %d, "
582                                 "bufflen %d, expected_transfer_len %d",
583                                 cmd->data_direction,
584                                 cmd->expected_data_direction,
585                                 cmd->bufflen, cmd->expected_transfer_len);
586                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
587                 }
588 #       endif
589                 cmd->data_direction = cmd->expected_data_direction;
590                 cmd->bufflen = cmd->expected_transfer_len;
591 #else
592                 if (unlikely(cmd->data_direction !=
593                                 cmd->expected_data_direction)) {
594                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
595                              (cmd->bufflen != 0)) &&
596                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
597                                 PRINT_ERROR("Expected data direction %d for "
598                                         "opcode 0x%02x (handler %s, target %s) "
599                                         "doesn't match "
600                                         "decoded value %d",
601                                         cmd->expected_data_direction,
602                                         cmd->cdb[0], dev->handler->name,
603                                         cmd->tgtt->name, cmd->data_direction);
604                                 PRINT_BUFFER("Failed CDB",
605                                         cmd->cdb, cmd->cdb_len);
606                                 scst_set_cmd_error(cmd,
607                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
608                                 goto out_dev_done;
609                         }
610                 }
611                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
612                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
613                                 "transfer length %d for opcode 0x%02x "
614                                 "(handler %s, target %s) doesn't match "
615                                 "decoded value %d. Faulty initiator "
616                                 "(e.g. VMware is known to be such) or "
617                                 "scst_scsi_op_table should be updated?",
618                                 cmd->expected_transfer_len, cmd->cdb[0],
619                                 dev->handler->name, cmd->tgtt->name,
620                                 cmd->bufflen);
621                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
622                                 cmd->cdb, cmd->cdb_len);
623                 }
624 #endif
625         }
626
627         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
628                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
629                         "target %s", cmd->cdb[0], dev->handler->name,
630                         cmd->tgtt->name);
631                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
632                 goto out_error;
633         }
634
635 set_res:
636         switch (state) {
637         case SCST_CMD_STATE_PREPARE_SPACE:
638         case SCST_CMD_STATE_PRE_PARSE:
639         case SCST_CMD_STATE_DEV_PARSE:
640         case SCST_CMD_STATE_RDY_TO_XFER:
641         case SCST_CMD_STATE_TGT_PRE_EXEC:
642         case SCST_CMD_STATE_SEND_FOR_EXEC:
643         case SCST_CMD_STATE_LOCAL_EXEC:
644         case SCST_CMD_STATE_REAL_EXEC:
645         case SCST_CMD_STATE_PRE_DEV_DONE:
646         case SCST_CMD_STATE_DEV_DONE:
647         case SCST_CMD_STATE_PRE_XMIT_RESP:
648         case SCST_CMD_STATE_XMIT_RESP:
649         case SCST_CMD_STATE_FINISHED:
650         case SCST_CMD_STATE_FINISHED_INTERNAL:
651                 cmd->state = state;
652                 res = SCST_CMD_STATE_RES_CONT_SAME;
653                 break;
654
655         default:
656                 if (state >= 0) {
657                         PRINT_ERROR("Dev handler %s parse() returned "
658                              "invalid cmd state %d (opcode %d)",
659                              dev->handler->name, state, cmd->cdb[0]);
660                 } else {
661                         PRINT_ERROR("Dev handler %s parse() returned "
662                                 "error %d (opcode %d)", dev->handler->name,
663                                 state, cmd->cdb[0]);
664                 }
665                 goto out_error;
666         }
667
668         if (cmd->resp_data_len == -1) {
669                 if (cmd->data_direction & SCST_DATA_READ)
670                         cmd->resp_data_len = cmd->bufflen;
671                 else
672                          cmd->resp_data_len = 0;
673         }
674
675 out:
676         TRACE_EXIT_HRES(res);
677         return res;
678
679 out_error:
680         /* dev_done() will be called as part of the regular cmd's finish */
681         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
682
683 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
684 out_dev_done:
685 #endif
686         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
687         res = SCST_CMD_STATE_RES_CONT_SAME;
688         goto out;
689 }
690
691 static int scst_prepare_space(struct scst_cmd *cmd)
692 {
693         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
694
695         TRACE_ENTRY();
696
697         if (cmd->data_direction == SCST_DATA_NONE)
698                 goto prep_done;
699
700         if (cmd->tgt_need_alloc_data_buf) {
701                 int orig_bufflen = cmd->bufflen;
702
703                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
704                         cmd);
705
706                 scst_set_cur_start(cmd);
707                 r = cmd->tgtt->alloc_data_buf(cmd);
708                 scst_set_alloc_buf_time(cmd);
709
710                 if (r > 0)
711                         goto alloc;
712                 else if (r == 0) {
713                         if (unlikely(cmd->bufflen == 0)) {
714                                 /* See comment in scst_alloc_space() */
715                                 if (cmd->sg == NULL)
716                                         goto alloc;
717                         }
718
719                         cmd->tgt_data_buf_alloced = 1;
720
721                         if (unlikely(orig_bufflen < cmd->bufflen)) {
722                                 PRINT_ERROR("Target driver allocated data "
723                                         "buffer (size %d), is less, than "
724                                         "required (size %d)", orig_bufflen,
725                                         cmd->bufflen);
726                                 goto out_error;
727                         }
728                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
729                 } else
730                         goto check;
731         }
732
733 alloc:
734         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
735                 r = scst_alloc_space(cmd);
736         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
737                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
738                 r = 0;
739         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
740                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
741                 cmd->sg = cmd->tgt_sg;
742                 cmd->sg_cnt = cmd->tgt_sg_cnt;
743                 cmd->in_sg = cmd->tgt_in_sg;
744                 cmd->in_sg_cnt = cmd->tgt_in_sg_cnt;
745                 r = 0;
746         } else {
747                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
748                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
749                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
750                 r = 0;
751         }
752
753 check:
754         if (r != 0) {
755                 if (scst_cmd_atomic(cmd)) {
756                         TRACE_MEM("%s", "Atomic memory allocation failed, "
757                               "rescheduling to the thread");
758                         res = SCST_CMD_STATE_RES_NEED_THREAD;
759                         goto out;
760                 } else
761                         goto out_no_space;
762         }
763
764 prep_done:
765         if (cmd->preprocessing_only) {
766                 cmd->preprocessing_only = 0;
767
768                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
769                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
770                                 "cmd %p", cmd);
771                         scst_set_cmd_abnormal_done_state(cmd);
772                         res = SCST_CMD_STATE_RES_CONT_SAME;
773                         goto out;
774                 }
775
776                 res = SCST_CMD_STATE_RES_CONT_NEXT;
777                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
778
779                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
780                 scst_set_cur_start(cmd);
781                 cmd->tgtt->preprocessing_done(cmd);
782                 TRACE_DBG("%s", "preprocessing_done() returned");
783                 goto out;
784
785         }
786
787         if (cmd->data_direction & SCST_DATA_WRITE)
788                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
789         else
790                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
791
792 out:
793         TRACE_EXIT_HRES(res);
794         return res;
795
796 out_no_space:
797         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
798                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
799         scst_set_busy(cmd);
800         scst_set_cmd_abnormal_done_state(cmd);
801         res = SCST_CMD_STATE_RES_CONT_SAME;
802         goto out;
803
804 out_error:
805         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
806         scst_set_cmd_abnormal_done_state(cmd);
807         res = SCST_CMD_STATE_RES_CONT_SAME;
808         goto out;
809 }
810
811 void scst_restart_cmd(struct scst_cmd *cmd, int status,
812         enum scst_exec_context pref_context)
813 {
814         TRACE_ENTRY();
815
816         scst_set_restart_waiting_time(cmd);
817
818         TRACE_DBG("Preferred context: %d", pref_context);
819         TRACE_DBG("tag=%llu, status=%#x",
820                   (long long unsigned int)scst_cmd_get_tag(cmd),
821                   status);
822
823 #ifdef CONFIG_SCST_EXTRACHECKS
824         if ((in_irq() || irqs_disabled()) &&
825             ((pref_context == SCST_CONTEXT_DIRECT) ||
826              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
827                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
828                         "SCST_CONTEXT_THREAD instead", pref_context,
829                         cmd->tgtt->name);
830                 pref_context = SCST_CONTEXT_THREAD;
831         }
832 #endif
833
834         switch (status) {
835         case SCST_PREPROCESS_STATUS_SUCCESS:
836                 if (cmd->data_direction & SCST_DATA_WRITE)
837                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
838                 else
839                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
840                 if (cmd->set_sn_on_restart_cmd)
841                         scst_cmd_set_sn(cmd);
842                 /* Small context optimization */
843                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
844                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
845                     ((pref_context == SCST_CONTEXT_SAME) &&
846                      scst_cmd_atomic(cmd))) {
847                         if (cmd->data_direction & SCST_DATA_WRITE) {
848                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
849                                                 &cmd->tgt_dev->tgt_dev_flags))
850                                         pref_context = SCST_CONTEXT_THREAD;
851                         } else {
852                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
853                                                 &cmd->tgt_dev->tgt_dev_flags))
854                                         pref_context = SCST_CONTEXT_THREAD;
855                         }
856                 }
857                 break;
858
859         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
860                 scst_set_cmd_abnormal_done_state(cmd);
861                 break;
862
863         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
864                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
865                 /* go through */
866         case SCST_PREPROCESS_STATUS_ERROR:
867                 if (cmd->sense != NULL)
868                         scst_set_cmd_error(cmd,
869                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
870                 scst_set_cmd_abnormal_done_state(cmd);
871                 break;
872
873         default:
874                 PRINT_ERROR("%s() received unknown status %x", __func__,
875                         status);
876                 scst_set_cmd_abnormal_done_state(cmd);
877                 break;
878         }
879
880         scst_process_redirect_cmd(cmd, pref_context, 1);
881
882         TRACE_EXIT();
883         return;
884 }
885 EXPORT_SYMBOL(scst_restart_cmd);
886
887 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
888 {
889         int res, rc;
890         struct scst_tgt_template *tgtt = cmd->tgtt;
891
892         TRACE_ENTRY();
893
894         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
895                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
896                 goto out_dev_done;
897         }
898
899         if ((tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
900                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
901                 res = SCST_CMD_STATE_RES_CONT_SAME;
902                 goto out;
903         }
904
905         if (unlikely(!tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
906                 /*
907                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
908                  * optimization.
909                  */
910                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
911                               "context, rescheduling", tgtt->name);
912                 res = SCST_CMD_STATE_RES_NEED_THREAD;
913                 goto out;
914         }
915
916         while (1) {
917                 int finished_cmds = atomic_read(&cmd->tgt->finished_cmds);
918
919                 res = SCST_CMD_STATE_RES_CONT_NEXT;
920                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
921
922                 if (tgtt->on_hw_pending_cmd_timeout != NULL) {
923                         struct scst_session *sess = cmd->sess;
924                         cmd->hw_pending_start = jiffies;
925                         cmd->cmd_hw_pending = 1;
926                         if (!test_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED, &sess->sess_aflags)) {
927                                 TRACE_DBG("Sched HW pending work for sess %p "
928                                         "(max time %d)", sess,
929                                         tgtt->max_hw_pending_time);
930                                 set_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED,
931                                         &sess->sess_aflags);
932                                 schedule_delayed_work(&sess->hw_pending_work,
933                                         tgtt->max_hw_pending_time * HZ);
934                         }
935                 }
936
937                 scst_set_cur_start(cmd);
938
939                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
940 #ifdef CONFIG_SCST_DEBUG_RETRY
941                 if (((scst_random() % 100) == 75))
942                         rc = SCST_TGT_RES_QUEUE_FULL;
943                 else
944 #endif
945                         rc = tgtt->rdy_to_xfer(cmd);
946                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
947
948                 if (likely(rc == SCST_TGT_RES_SUCCESS))
949                         goto out;
950
951                 scst_set_rdy_to_xfer_time(cmd);
952
953                 cmd->cmd_hw_pending = 0;
954
955                 /* Restore the previous state */
956                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
957
958                 switch (rc) {
959                 case SCST_TGT_RES_QUEUE_FULL:
960                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
961                                 break;
962                         else
963                                 continue;
964
965                 case SCST_TGT_RES_NEED_THREAD_CTX:
966                         TRACE_DBG("Target driver %s "
967                               "rdy_to_xfer() requested thread "
968                               "context, rescheduling", tgtt->name);
969                         res = SCST_CMD_STATE_RES_NEED_THREAD;
970                         break;
971
972                 default:
973                         goto out_error_rc;
974                 }
975                 break;
976         }
977
978 out:
979         TRACE_EXIT_HRES(res);
980         return res;
981
982 out_error_rc:
983         if (rc == SCST_TGT_RES_FATAL_ERROR) {
984                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
985                      "fatal error", tgtt->name);
986         } else {
987                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
988                             "value %d", tgtt->name, rc);
989         }
990         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
991
992 out_dev_done:
993         scst_set_cmd_abnormal_done_state(cmd);
994         res = SCST_CMD_STATE_RES_CONT_SAME;
995         goto out;
996 }
997
998 /* No locks, but might be in IRQ */
999 static void scst_process_redirect_cmd(struct scst_cmd *cmd,
1000         enum scst_exec_context context, int check_retries)
1001 {
1002         struct scst_tgt *tgt = cmd->tgt;
1003         unsigned long flags;
1004
1005         TRACE_ENTRY();
1006
1007         TRACE_DBG("Context: %x", context);
1008
1009         if (context == SCST_CONTEXT_SAME)
1010                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1011                                                  SCST_CONTEXT_DIRECT;
1012
1013         switch (context) {
1014         case SCST_CONTEXT_DIRECT_ATOMIC:
1015                 scst_process_active_cmd(cmd, true);
1016                 break;
1017
1018         case SCST_CONTEXT_DIRECT:
1019                 if (check_retries)
1020                         scst_check_retries(tgt);
1021                 scst_process_active_cmd(cmd, false);
1022                 break;
1023
1024         default:
1025                 PRINT_ERROR("Context %x is unknown, using the thread one",
1026                             context);
1027                 /* go through */
1028         case SCST_CONTEXT_THREAD:
1029                 if (check_retries)
1030                         scst_check_retries(tgt);
1031                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1032                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1033                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1034                         list_add(&cmd->cmd_list_entry,
1035                                 &cmd->cmd_lists->active_cmd_list);
1036                 else
1037                         list_add_tail(&cmd->cmd_list_entry,
1038                                 &cmd->cmd_lists->active_cmd_list);
1039                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1040                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1041                 break;
1042
1043         case SCST_CONTEXT_TASKLET:
1044                 if (check_retries)
1045                         scst_check_retries(tgt);
1046                 scst_schedule_tasklet(cmd);
1047                 break;
1048         }
1049
1050         TRACE_EXIT();
1051         return;
1052 }
1053
1054 void scst_rx_data(struct scst_cmd *cmd, int status,
1055         enum scst_exec_context pref_context)
1056 {
1057         TRACE_ENTRY();
1058
1059         scst_set_rdy_to_xfer_time(cmd);
1060
1061         TRACE_DBG("Preferred context: %d", pref_context);
1062         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1063
1064         cmd->cmd_hw_pending = 0;
1065
1066 #ifdef CONFIG_SCST_EXTRACHECKS
1067         if ((in_irq() || irqs_disabled()) &&
1068             ((pref_context == SCST_CONTEXT_DIRECT) ||
1069              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1070                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1071                         "SCST_CONTEXT_THREAD instead", pref_context,
1072                         cmd->tgtt->name);
1073                 pref_context = SCST_CONTEXT_THREAD;
1074         }
1075 #endif
1076
1077         switch (status) {
1078         case SCST_RX_STATUS_SUCCESS:
1079 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1080                 if (trace_flag & TRACE_RCV_BOT) {
1081                         int i;
1082                         struct scatterlist *sg;
1083                         if (cmd->in_sg != NULL)
1084                                 sg = cmd->in_sg;
1085                         else if (cmd->tgt_in_sg != NULL)
1086                                 sg = cmd->tgt_in_sg;
1087                         else if (cmd->tgt_sg != NULL)
1088                                 sg = cmd->tgt_sg;
1089                         else
1090                                 sg = cmd->sg;
1091                         if (sg != NULL) {
1092                                 TRACE_RECV_BOT("RX data for cmd %p "
1093                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
1094                                         cmd, cmd->tgt_sg_cnt, sg,
1095                                         (void *)sg_page(&sg[0]));
1096                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1097                                         PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1098                                                 sg_virt(&sg[i]), sg[i].length);
1099                                 }
1100                         }
1101                 }
1102 #endif
1103                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1104                 /* Small context optimization */
1105                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1106                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1107                     ((pref_context == SCST_CONTEXT_SAME) &&
1108                      scst_cmd_atomic(cmd))) {
1109                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1110                                         &cmd->tgt_dev->tgt_dev_flags))
1111                                 pref_context = SCST_CONTEXT_THREAD;
1112                 }
1113                 break;
1114
1115         case SCST_RX_STATUS_ERROR_SENSE_SET:
1116                 scst_set_cmd_abnormal_done_state(cmd);
1117                 break;
1118
1119         case SCST_RX_STATUS_ERROR_FATAL:
1120                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1121                 /* go through */
1122         case SCST_RX_STATUS_ERROR:
1123                 scst_set_cmd_error(cmd,
1124                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1125                 scst_set_cmd_abnormal_done_state(cmd);
1126                 break;
1127
1128         default:
1129                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1130                         status);
1131                 scst_set_cmd_abnormal_done_state(cmd);
1132                 break;
1133         }
1134
1135         scst_process_redirect_cmd(cmd, pref_context, 1);
1136
1137         TRACE_EXIT();
1138         return;
1139 }
1140 EXPORT_SYMBOL(scst_rx_data);
1141
1142 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1143 {
1144         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1145
1146         TRACE_ENTRY();
1147
1148         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1149
1150         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1151                 goto out;
1152
1153         TRACE_DBG("Calling pre_exec(%p)", cmd);
1154         scst_set_cur_start(cmd);
1155         rc = cmd->tgtt->pre_exec(cmd);
1156         scst_set_pre_exec_time(cmd);
1157         TRACE_DBG("pre_exec() returned %d", rc);
1158
1159         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1160                 switch (rc) {
1161                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1162                         scst_set_cmd_abnormal_done_state(cmd);
1163                         break;
1164                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1165                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1166                         /* go through */
1167                 case SCST_PREPROCESS_STATUS_ERROR:
1168                         scst_set_cmd_error(cmd,
1169                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1170                         scst_set_cmd_abnormal_done_state(cmd);
1171                         break;
1172                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1173                         TRACE_DBG("Target driver's %s pre_exec() requested "
1174                                 "thread context, rescheduling",
1175                                 cmd->tgtt->name);
1176                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1177                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1178                         break;
1179                 default:
1180                         sBUG();
1181                         break;
1182                 }
1183         }
1184
1185 out:
1186         TRACE_EXIT_RES(res);
1187         return res;
1188 }
1189
1190 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1191         const uint8_t *rq_sense, int rq_sense_len, int resid)
1192 {
1193         TRACE_ENTRY();
1194
1195         scst_set_exec_time(cmd);
1196
1197         cmd->status = result & 0xff;
1198         cmd->msg_status = msg_byte(result);
1199         cmd->host_status = host_byte(result);
1200         cmd->driver_status = driver_byte(result);
1201         if (unlikely(resid != 0)) {
1202 #ifdef CONFIG_SCST_EXTRACHECKS
1203                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1204                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1205                                 "op %x)", resid, cmd->resp_data_len,
1206                                 cmd->cdb[0]);
1207                 } else
1208 #endif
1209                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1210         }
1211
1212         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1213                 /* We might have double reset UA here */
1214                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1215                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1216
1217                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1218         }
1219
1220         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1221               "cmd->msg_status=%x, cmd->host_status=%x, "
1222               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1223               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1224
1225         cmd->completed = 1;
1226
1227         TRACE_EXIT();
1228         return;
1229 }
1230
1231 /* For small context optimization */
1232 static inline enum scst_exec_context scst_optimize_post_exec_context(
1233         struct scst_cmd *cmd, enum scst_exec_context context)
1234 {
1235         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1236             (context == SCST_CONTEXT_TASKLET) ||
1237             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1238                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1239                                 &cmd->tgt_dev->tgt_dev_flags))
1240                         context = SCST_CONTEXT_THREAD;
1241         }
1242         return context;
1243 }
1244
1245 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1246 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1247                                             struct scsi_request **req)
1248 {
1249         struct scst_cmd *cmd = NULL;
1250
1251         if (scsi_cmd) {
1252                 *req = scsi_cmd->sc_request;
1253                 if (*req)
1254                         cmd = (struct scst_cmd *)(*req)->upper_private_data;
1255         }
1256
1257         if (cmd == NULL) {
1258                 PRINT_ERROR("%s", "Request with NULL cmd");
1259                 if (*req)
1260                         scsi_release_request(*req);
1261         }
1262
1263         return cmd;
1264 }
1265
1266 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1267 {
1268         struct scsi_request *req = NULL;
1269         struct scst_cmd *cmd;
1270
1271         TRACE_ENTRY();
1272
1273         cmd = scst_get_cmd(scsi_cmd, &req);
1274         if (cmd == NULL)
1275                 goto out;
1276
1277         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1278                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1279
1280         /* Clear out request structure */
1281         req->sr_use_sg = 0;
1282         req->sr_sglist_len = 0;
1283         req->sr_bufflen = 0;
1284         req->sr_buffer = NULL;
1285         req->sr_underflow = 0;
1286         req->sr_request->rq_disk = NULL; /* disown request blk */
1287
1288         scst_release_request(cmd);
1289
1290         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1291
1292         scst_process_redirect_cmd(cmd,
1293                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1294                                                 0);
1295
1296 out:
1297         TRACE_EXIT();
1298         return;
1299 }
1300 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1301 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1302 {
1303         struct scst_cmd *cmd;
1304
1305         TRACE_ENTRY();
1306
1307         cmd = (struct scst_cmd *)data;
1308         if (cmd == NULL)
1309                 goto out;
1310
1311         scst_do_cmd_done(cmd, result, sense, SCSI_SENSE_BUFFERSIZE, resid);
1312
1313         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1314
1315         scst_process_redirect_cmd(cmd,
1316             scst_optimize_post_exec_context(cmd, scst_estimate_context()), 0);
1317
1318 out:
1319         TRACE_EXIT();
1320         return;
1321 }
1322 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1323
1324 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1325         enum scst_exec_context pref_context)
1326 {
1327         TRACE_ENTRY();
1328
1329         scst_set_exec_time(cmd);
1330
1331         if (next_state == SCST_CMD_STATE_DEFAULT)
1332                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1333
1334 #if defined(CONFIG_SCST_DEBUG)
1335         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1336                 if ((trace_flag & TRACE_RCV_TOP) && (cmd->sg != NULL)) {
1337                         int i;
1338                         struct scatterlist *sg = cmd->sg;
1339                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1340                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1341                         for (i = 0; i < cmd->sg_cnt; ++i) {
1342                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1343                                         "Exec'd sg", sg_virt(&sg[i]),
1344                                         sg[i].length);
1345                         }
1346                 }
1347         }
1348 #endif
1349
1350         cmd->state = next_state;
1351
1352 #ifdef CONFIG_SCST_EXTRACHECKS
1353         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1354             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1355             (next_state != SCST_CMD_STATE_FINISHED) &&
1356             (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
1357                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1358                         __func__, next_state, cmd->cdb[0]);
1359                 scst_set_cmd_error(cmd,
1360                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1361                 scst_set_cmd_abnormal_done_state(cmd);
1362         }
1363 #endif
1364         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1365         scst_process_redirect_cmd(cmd, pref_context, 0);
1366
1367         TRACE_EXIT();
1368         return;
1369 }
1370
1371 static int scst_report_luns_local(struct scst_cmd *cmd)
1372 {
1373         int res = SCST_EXEC_COMPLETED, rc;
1374         int dev_cnt = 0;
1375         int buffer_size;
1376         int i;
1377         struct scst_tgt_dev *tgt_dev = NULL;
1378         uint8_t *buffer;
1379         int offs, overflow = 0;
1380
1381         TRACE_ENTRY();
1382
1383         if (scst_cmd_atomic(cmd)) {
1384                 res = SCST_EXEC_NEED_THREAD;
1385                 goto out;
1386         }
1387
1388         rc = scst_check_local_events(cmd);
1389         if (unlikely(rc != 0))
1390                 goto out_done;
1391
1392         cmd->status = 0;
1393         cmd->msg_status = 0;
1394         cmd->host_status = DID_OK;
1395         cmd->driver_status = 0;
1396
1397         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1398                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1399                         "LUNS command", cmd->cdb[2]);
1400                 goto out_err;
1401         }
1402
1403         buffer_size = scst_get_buf_first(cmd, &buffer);
1404         if (unlikely(buffer_size == 0))
1405                 goto out_compl;
1406         else if (unlikely(buffer_size < 0))
1407                 goto out_hw_err;
1408
1409         if (buffer_size < 16)
1410                 goto out_put_err;
1411
1412         memset(buffer, 0, buffer_size);
1413         offs = 8;
1414
1415         /*
1416          * cmd won't allow to suspend activities, so we can access
1417          * sess->sess_tgt_dev_list_hash without any additional protection.
1418          */
1419         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1420                 struct list_head *sess_tgt_dev_list_head =
1421                         &cmd->sess->sess_tgt_dev_list_hash[i];
1422                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1423                                 sess_tgt_dev_list_entry) {
1424                         if (!overflow) {
1425                                 if (offs >= buffer_size) {
1426                                         scst_put_buf(cmd, buffer);
1427                                         buffer_size = scst_get_buf_next(cmd,
1428                                                                        &buffer);
1429                                         if (buffer_size > 0) {
1430                                                 memset(buffer, 0, buffer_size);
1431                                                 offs = 0;
1432                                         } else {
1433                                                 overflow = 1;
1434                                                 goto inc_dev_cnt;
1435                                         }
1436                                 }
1437                                 if ((buffer_size - offs) < 8) {
1438                                         PRINT_ERROR("Buffer allocated for "
1439                                                 "REPORT LUNS command doesn't "
1440                                                 "allow to fit 8 byte entry "
1441                                                 "(buffer_size=%d)",
1442                                                 buffer_size);
1443                                         goto out_put_hw_err;
1444                                 }
1445                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1446                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1447                                 offs += 8;
1448                         }
1449 inc_dev_cnt:
1450                         dev_cnt++;
1451                 }
1452         }
1453         if (!overflow)
1454                 scst_put_buf(cmd, buffer);
1455
1456         /* Set the response header */
1457         buffer_size = scst_get_buf_first(cmd, &buffer);
1458         if (unlikely(buffer_size == 0))
1459                 goto out_compl;
1460         else if (unlikely(buffer_size < 0))
1461                 goto out_hw_err;
1462
1463         dev_cnt *= 8;
1464         buffer[0] = (dev_cnt >> 24) & 0xff;
1465         buffer[1] = (dev_cnt >> 16) & 0xff;
1466         buffer[2] = (dev_cnt >> 8) & 0xff;
1467         buffer[3] = dev_cnt & 0xff;
1468
1469         scst_put_buf(cmd, buffer);
1470
1471         dev_cnt += 8;
1472         if (dev_cnt < cmd->resp_data_len)
1473                 scst_set_resp_data_len(cmd, dev_cnt);
1474
1475 out_compl:
1476         cmd->completed = 1;
1477
1478         /* Clear left sense_reported_luns_data_changed UA, if any. */
1479
1480         /*
1481          * cmd won't allow to suspend activities, so we can access
1482          * sess->sess_tgt_dev_list_hash without any additional protection.
1483          */
1484         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1485                 struct list_head *sess_tgt_dev_list_head =
1486                         &cmd->sess->sess_tgt_dev_list_hash[i];
1487
1488                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1489                                 sess_tgt_dev_list_entry) {
1490                         struct scst_tgt_dev_UA *ua;
1491
1492                         spin_lock_bh(&tgt_dev->tgt_dev_lock);
1493                         list_for_each_entry(ua, &tgt_dev->UA_list,
1494                                                 UA_list_entry) {
1495                                 if (scst_analyze_sense(ua->UA_sense_buffer,
1496                                                 ua->UA_valid_sense_len,
1497                                                 SCST_SENSE_ALL_VALID,
1498                                                 SCST_LOAD_SENSE(scst_sense_reported_luns_data_changed))) {
1499                                         TRACE_MGMT_DBG("Freeing not needed "
1500                                                 "REPORTED LUNS DATA CHANGED UA "
1501                                                 "%p", ua);
1502                                         list_del(&ua->UA_list_entry);
1503                                         mempool_free(ua, scst_ua_mempool);
1504                                         break;
1505                                 }
1506                         }
1507                         spin_unlock_bh(&tgt_dev->tgt_dev_lock);
1508                 }
1509         }
1510
1511 out_done:
1512         /* Report the result */
1513         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1514
1515 out:
1516         TRACE_EXIT_RES(res);
1517         return res;
1518
1519 out_put_err:
1520         scst_put_buf(cmd, buffer);
1521
1522 out_err:
1523         scst_set_cmd_error(cmd,
1524                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1525         goto out_compl;
1526
1527 out_put_hw_err:
1528         scst_put_buf(cmd, buffer);
1529
1530 out_hw_err:
1531         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1532         goto out_compl;
1533 }
1534
1535 static int scst_request_sense_local(struct scst_cmd *cmd)
1536 {
1537         int res = SCST_EXEC_COMPLETED, rc;
1538         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1539         uint8_t *buffer;
1540         int buffer_size = 0, sl = 0;
1541
1542         TRACE_ENTRY();
1543
1544         rc = scst_check_local_events(cmd);
1545         if (unlikely(rc != 0))
1546                 goto out_done;
1547
1548         cmd->status = 0;
1549         cmd->msg_status = 0;
1550         cmd->host_status = DID_OK;
1551         cmd->driver_status = 0;
1552
1553         spin_lock_bh(&tgt_dev->tgt_dev_lock);
1554
1555         if (tgt_dev->tgt_dev_valid_sense_len == 0)
1556                 goto out_not_completed;
1557
1558         TRACE(TRACE_SCSI, "%s: Returning stored sense", cmd->op_name);
1559
1560         buffer_size = scst_get_buf_first(cmd, &buffer);
1561         if (unlikely(buffer_size == 0))
1562                 goto out_compl;
1563         else if (unlikely(buffer_size < 0))
1564                 goto out_hw_err;
1565
1566         memset(buffer, 0, buffer_size);
1567
1568         if (((tgt_dev->tgt_dev_sense[0] == 0x70) ||
1569              (tgt_dev->tgt_dev_sense[0] == 0x71)) && (cmd->cdb[1] & 1)) {
1570                 PRINT_WARNING("%s: Fixed format of the saved sense, but "
1571                         "descriptor format requested. Convertion will "
1572                         "truncated data", cmd->op_name);
1573                 PRINT_BUFFER("Original sense", tgt_dev->tgt_dev_sense,
1574                         tgt_dev->tgt_dev_valid_sense_len);
1575
1576                 buffer_size = min(SCST_STANDARD_SENSE_LEN, buffer_size);
1577                 sl = scst_set_sense(buffer, buffer_size, true,
1578                         tgt_dev->tgt_dev_sense[2], tgt_dev->tgt_dev_sense[12],
1579                         tgt_dev->tgt_dev_sense[13]);
1580         } else if (((tgt_dev->tgt_dev_sense[0] == 0x72) ||
1581                     (tgt_dev->tgt_dev_sense[0] == 0x73)) && !(cmd->cdb[1] & 1)) {
1582                 PRINT_WARNING("%s: Descriptor format of the "
1583                         "saved sense, but fixed format requested. Convertion "
1584                         "will truncated data", cmd->op_name);
1585                 PRINT_BUFFER("Original sense", tgt_dev->tgt_dev_sense,
1586                         tgt_dev->tgt_dev_valid_sense_len);
1587
1588                 buffer_size = min(SCST_STANDARD_SENSE_LEN, buffer_size);
1589                 sl = scst_set_sense(buffer, buffer_size, false,
1590                         tgt_dev->tgt_dev_sense[1], tgt_dev->tgt_dev_sense[2],
1591                         tgt_dev->tgt_dev_sense[3]);
1592         } else {
1593                 if (buffer_size >= tgt_dev->tgt_dev_valid_sense_len)
1594                         sl = tgt_dev->tgt_dev_valid_sense_len;
1595                 else {
1596                         sl = buffer_size;
1597                         PRINT_WARNING("%s: Being returned sense truncated to "
1598                                 "size %d (needed %d)", cmd->op_name,
1599                                 buffer_size, tgt_dev->tgt_dev_valid_sense_len);
1600                 }
1601                 memcpy(buffer, tgt_dev->tgt_dev_sense, sl);
1602         }
1603
1604         scst_put_buf(cmd, buffer);
1605
1606 out_compl:
1607         tgt_dev->tgt_dev_valid_sense_len = 0;
1608         scst_set_resp_data_len(cmd, sl);
1609
1610         spin_unlock_bh(&tgt_dev->tgt_dev_lock);
1611
1612         cmd->completed = 1;
1613
1614 out_done:
1615         /* Report the result */
1616         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1617
1618 out:
1619         TRACE_EXIT_RES(res);
1620         return res;
1621
1622 out_hw_err:
1623         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1624         goto out_compl;
1625
1626 out_not_completed:
1627         spin_unlock_bh(&tgt_dev->tgt_dev_lock);
1628         res = SCST_EXEC_NOT_COMPLETED;
1629         goto out;
1630 }
1631
1632 static int scst_pre_select(struct scst_cmd *cmd)
1633 {
1634         int res = SCST_EXEC_NOT_COMPLETED;
1635
1636         TRACE_ENTRY();
1637
1638         if (scst_cmd_atomic(cmd)) {
1639                 res = SCST_EXEC_NEED_THREAD;
1640                 goto out;
1641         }
1642
1643         scst_block_dev_cmd(cmd, 1);
1644
1645         /* Check for local events will be done when cmd will be executed */
1646
1647 out:
1648         TRACE_EXIT_RES(res);
1649         return res;
1650 }
1651
1652 static int scst_reserve_local(struct scst_cmd *cmd)
1653 {
1654         int res = SCST_EXEC_NOT_COMPLETED, rc;
1655         struct scst_device *dev;
1656         struct scst_tgt_dev *tgt_dev_tmp;
1657
1658         TRACE_ENTRY();
1659
1660         if (scst_cmd_atomic(cmd)) {
1661                 res = SCST_EXEC_NEED_THREAD;
1662                 goto out;
1663         }
1664
1665         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1666                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1667                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1668                 scst_set_cmd_error(cmd,
1669                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1670                 goto out_done;
1671         }
1672
1673         dev = cmd->dev;
1674
1675         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1676                 scst_block_dev_cmd(cmd, 1);
1677
1678         rc = scst_check_local_events(cmd);
1679         if (unlikely(rc != 0))
1680                 goto out_done;
1681
1682         spin_lock_bh(&dev->dev_lock);
1683
1684         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1685                 spin_unlock_bh(&dev->dev_lock);
1686                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1687                 goto out_done;
1688         }
1689
1690         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1691                             dev_tgt_dev_list_entry) {
1692                 if (cmd->tgt_dev != tgt_dev_tmp)
1693                         set_bit(SCST_TGT_DEV_RESERVED,
1694                                 &tgt_dev_tmp->tgt_dev_flags);
1695         }
1696         dev->dev_reserved = 1;
1697
1698         spin_unlock_bh(&dev->dev_lock);
1699
1700 out:
1701         TRACE_EXIT_RES(res);
1702         return res;
1703
1704 out_done:
1705         /* Report the result */
1706         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1707         res = SCST_EXEC_COMPLETED;
1708         goto out;
1709 }
1710
1711 static int scst_release_local(struct scst_cmd *cmd)
1712 {
1713         int res = SCST_EXEC_NOT_COMPLETED, rc;
1714         struct scst_tgt_dev *tgt_dev_tmp;
1715         struct scst_device *dev;
1716
1717         TRACE_ENTRY();
1718
1719         if (scst_cmd_atomic(cmd)) {
1720                 res = SCST_EXEC_NEED_THREAD;
1721                 goto out;
1722         }
1723
1724         dev = cmd->dev;
1725
1726         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1727                 scst_block_dev_cmd(cmd, 1);
1728
1729         rc = scst_check_local_events(cmd);
1730         if (unlikely(rc != 0))
1731                 goto out_done;
1732
1733         spin_lock_bh(&dev->dev_lock);
1734
1735         /*
1736          * The device could be RELEASED behind us, if RESERVING session
1737          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1738          * matter, so use lock and no retest for DEV_RESERVED bits again
1739          */
1740         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1741                 res = SCST_EXEC_COMPLETED;
1742                 cmd->status = 0;
1743                 cmd->msg_status = 0;
1744                 cmd->host_status = DID_OK;
1745                 cmd->driver_status = 0;
1746                 cmd->completed = 1;
1747         } else {
1748                 list_for_each_entry(tgt_dev_tmp,
1749                                     &dev->dev_tgt_dev_list,
1750                                     dev_tgt_dev_list_entry) {
1751                         clear_bit(SCST_TGT_DEV_RESERVED,
1752                                 &tgt_dev_tmp->tgt_dev_flags);
1753                 }
1754                 dev->dev_reserved = 0;
1755         }
1756
1757         spin_unlock_bh(&dev->dev_lock);
1758
1759         if (res == SCST_EXEC_COMPLETED)
1760                 goto out_done;
1761
1762 out:
1763         TRACE_EXIT_RES(res);
1764         return res;
1765
1766 out_done:
1767         res = SCST_EXEC_COMPLETED;
1768         /* Report the result */
1769         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1770         goto out;
1771 }
1772
1773 /* No locks, no IRQ or IRQ-disabled context allowed */
1774 int scst_check_local_events(struct scst_cmd *cmd)
1775 {
1776         int res, rc;
1777         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1778         struct scst_device *dev = cmd->dev;
1779
1780         TRACE_ENTRY();
1781
1782         /*
1783          * There's no race here, because we need to trace commands sent
1784          * *after* dev_double_ua_possible flag was set.
1785          */
1786         if (unlikely(dev->dev_double_ua_possible))
1787                 cmd->double_ua_possible = 1;
1788
1789         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1790                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1791                 goto out_uncomplete;
1792         }
1793
1794         /* Reserve check before Unit Attention */
1795         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1796                               &tgt_dev->tgt_dev_flags))) {
1797                 if (cmd->cdb[0] != INQUIRY &&
1798                     cmd->cdb[0] != REPORT_LUNS &&
1799                     cmd->cdb[0] != RELEASE &&
1800                     cmd->cdb[0] != RELEASE_10 &&
1801                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1802                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1803                      (cmd->cdb[4] & 3)) &&
1804                     cmd->cdb[0] != LOG_SENSE &&
1805                     cmd->cdb[0] != REQUEST_SENSE) {
1806                         scst_set_cmd_error_status(cmd,
1807                                 SAM_STAT_RESERVATION_CONFLICT);
1808                         goto out_complete;
1809                 }
1810         }
1811
1812         /* If we had internal bus reset, set the command error unit attention */
1813         if ((dev->scsi_dev != NULL) &&
1814             unlikely(dev->scsi_dev->was_reset)) {
1815                 if (scst_is_ua_command(cmd)) {
1816                         int done = 0;
1817                         /*
1818                          * Prevent more than 1 cmd to be triggered by
1819                          * was_reset.
1820                          */
1821                         spin_lock_bh(&dev->dev_lock);
1822                         if (dev->scsi_dev->was_reset) {
1823                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1824                                 scst_set_cmd_error(cmd,
1825                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1826                                 /*
1827                                  * It looks like it is safe to clear was_reset
1828                                  * here.
1829                                  */
1830                                 dev->scsi_dev->was_reset = 0;
1831                                 done = 1;
1832                         }
1833                         spin_unlock_bh(&dev->dev_lock);
1834
1835                         if (done)
1836                                 goto out_complete;
1837                 }
1838         }
1839
1840         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1841                         &cmd->tgt_dev->tgt_dev_flags))) {
1842                 if (scst_is_ua_command(cmd)) {
1843                         rc = scst_set_pending_UA(cmd);
1844                         if (rc == 0)
1845                                 goto out_complete;
1846                 }
1847         }
1848
1849         res = 0;
1850
1851 out:
1852         TRACE_EXIT_RES(res);
1853         return res;
1854
1855 out_complete:
1856         res = 1;
1857         sBUG_ON(!cmd->completed);
1858         goto out;
1859
1860 out_uncomplete:
1861         res = -1;
1862         goto out;
1863 }
1864 EXPORT_SYMBOL(scst_check_local_events);
1865
1866 /* No locks */
1867 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1868 {
1869         if (slot == NULL)
1870                 goto inc;
1871
1872         /* Optimized for lockless fast path */
1873
1874         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1875                 atomic_read(slot));
1876
1877         if (!atomic_dec_and_test(slot))
1878                 goto out;
1879
1880         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1881                 tgt_dev->num_free_sn_slots);
1882         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1883                 spin_lock_irq(&tgt_dev->sn_lock);
1884                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1885                         if (tgt_dev->num_free_sn_slots < 0)
1886                                 tgt_dev->cur_sn_slot = slot;
1887                         /*
1888                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1889                          */
1890                         smp_mb();
1891                         tgt_dev->num_free_sn_slots++;
1892                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1893                                 tgt_dev->num_free_sn_slots);
1894
1895                 }
1896                 spin_unlock_irq(&tgt_dev->sn_lock);
1897         }
1898
1899 inc:
1900         /*
1901          * No protection of expected_sn is needed, because only one thread
1902          * at time can be here (serialized by sn). Also it is supposed that
1903          * there could not be half-incremented halves.
1904          */
1905         tgt_dev->expected_sn++;
1906         /*
1907          * Write must be before def_cmd_count read to be in sync. with
1908          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1909          */
1910         smp_mb();
1911         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1912
1913 out:
1914         return;
1915 }
1916
1917 /* No locks */
1918 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1919         bool make_active)
1920 {
1921         /* For HQ commands SN is not set */
1922         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1923                                cmd->sn_set && !cmd->retry;
1924         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1925         struct scst_cmd *res;
1926
1927         TRACE_ENTRY();
1928
1929         if (inc_expected_sn)
1930                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1931
1932         if (make_active) {
1933                 scst_make_deferred_commands_active(tgt_dev);
1934                 res = NULL;
1935         } else
1936                 res = scst_check_deferred_commands(tgt_dev);
1937
1938         TRACE_EXIT_HRES(res);
1939         return res;
1940 }
1941
1942 /* cmd must be additionally referenced to not die inside */
1943 static int scst_do_real_exec(struct scst_cmd *cmd)
1944 {
1945         int res = SCST_EXEC_NOT_COMPLETED;
1946 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1947         int rc;
1948 #endif
1949         bool atomic = scst_cmd_atomic(cmd);
1950         struct scst_device *dev = cmd->dev;
1951         struct scst_dev_type *handler = dev->handler;
1952         struct io_context *old_ctx = NULL;
1953         bool ctx_changed = false;
1954
1955         TRACE_ENTRY();
1956
1957         if (!atomic)
1958                 ctx_changed = scst_set_io_context(cmd, &old_ctx);
1959
1960         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1961
1962         if (handler->exec) {
1963                 if (unlikely(!dev->handler->exec_atomic && atomic)) {
1964                         /*
1965                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1966                          * optimization.
1967                          */
1968                         TRACE_DBG("Dev handler %s exec() needs thread "
1969                                 "context, rescheduling", dev->handler->name);
1970                         res = SCST_EXEC_NEED_THREAD;
1971                         goto out_restore;
1972                 }
1973
1974                 TRACE_DBG("Calling dev handler %s exec(%p)",
1975                       handler->name, cmd);
1976                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1977                         cmd->cdb_len);
1978                 scst_set_cur_start(cmd);
1979                 res = handler->exec(cmd);
1980                 TRACE_DBG("Dev handler %s exec() returned %d",
1981                       handler->name, res);
1982
1983                 if (res == SCST_EXEC_COMPLETED)
1984                         goto out_complete;
1985                 else if (res == SCST_EXEC_NEED_THREAD)
1986                         goto out_restore;
1987
1988                 scst_set_exec_time(cmd);
1989
1990                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1991         }
1992
1993         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1994
1995         if (unlikely(dev->scsi_dev == NULL)) {
1996                 PRINT_ERROR("Command for virtual device must be "
1997                         "processed by device handler (LUN %lld)!",
1998                         (long long unsigned int)cmd->lun);
1999                 goto out_error;
2000         }
2001
2002         res = scst_check_local_events(cmd);
2003         if (unlikely(res != 0))
2004                 goto out_done;
2005
2006 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
2007         if (unlikely(atomic)) {
2008                 TRACE_DBG("Pass-through exec() can not be called in atomic "
2009                         "context, rescheduling to the thread (handler %s)",
2010                         handler->name);
2011                 res = SCST_EXEC_NEED_THREAD;
2012                 goto out_restore;
2013         }
2014 #endif
2015
2016         scst_set_cur_start(cmd);
2017
2018 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
2019         if (unlikely(scst_alloc_request(cmd) != 0)) {
2020                 if (atomic) {
2021                         res = SCST_EXEC_NEED_THREAD;
2022                         goto out_restore;
2023                 } else {
2024                         PRINT_INFO("%s", "Unable to allocate request, "
2025                                 "sending BUSY status");
2026                         goto out_busy;
2027                 }
2028         }
2029
2030         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
2031                     (void *)cmd->scsi_req->sr_buffer,
2032                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
2033                     cmd->retries);
2034 #else
2035 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2036         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
2037                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
2038                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
2039                         atomic ? GFP_ATOMIC : GFP_KERNEL);
2040 #else
2041         rc = scst_scsi_exec_async(cmd, scst_cmd_done);
2042 #endif
2043         if (unlikely(rc != 0)) {
2044                 if (atomic) {
2045                         res = SCST_EXEC_NEED_THREAD;
2046                         goto out_restore;
2047                 } else {
2048                         PRINT_ERROR("scst pass-through exec failed: %x", rc);
2049                         goto out_error;
2050                 }
2051         }
2052 #endif
2053
2054 out_complete:
2055         res = SCST_EXEC_COMPLETED;
2056
2057 out_reset_ctx:
2058         if (ctx_changed)
2059                 scst_reset_io_context(cmd->tgt_dev, old_ctx);
2060
2061         TRACE_EXIT();
2062         return res;
2063
2064 out_restore:
2065         scst_set_exec_time(cmd);
2066         /* Restore the state */
2067         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2068         goto out_reset_ctx;
2069
2070 out_error:
2071         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2072         goto out_done;
2073
2074 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
2075 out_busy:
2076         scst_set_busy(cmd);
2077         /* go through */
2078 #endif
2079
2080 out_done:
2081         res = SCST_EXEC_COMPLETED;
2082         /* Report the result */
2083         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2084         goto out_complete;
2085 }
2086
2087 static inline int scst_real_exec(struct scst_cmd *cmd)
2088 {
2089         int res;
2090
2091         TRACE_ENTRY();
2092
2093         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2094         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2095         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2096
2097         __scst_cmd_get(cmd);
2098
2099         res = scst_do_real_exec(cmd);
2100
2101         if (likely(res == SCST_EXEC_COMPLETED)) {
2102                 scst_post_exec_sn(cmd, true);
2103                 if (cmd->dev->scsi_dev != NULL)
2104                         generic_unplug_device(
2105                                 cmd->dev->scsi_dev->request_queue);
2106         } else
2107                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2108
2109         __scst_cmd_put(cmd);
2110
2111         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2112
2113         TRACE_EXIT_RES(res);
2114         return res;
2115 }
2116
2117 static int scst_do_local_exec(struct scst_cmd *cmd)
2118 {
2119         int res;
2120         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2121
2122         TRACE_ENTRY();
2123
2124         /* Check READ_ONLY device status */
2125         if ((cmd->op_flags & SCST_WRITE_MEDIUM) &&
2126             (tgt_dev->acg_dev->rd_only || cmd->dev->swp ||
2127              cmd->dev->rd_only)) {
2128                 PRINT_WARNING("Attempt of write access to read-only device: "
2129                         "initiator %s, LUN %lld, op %x",
2130                         cmd->sess->initiator_name, cmd->lun, cmd->cdb[0]);
2131                 scst_set_cmd_error(cmd,
2132                            SCST_LOAD_SENSE(scst_sense_data_protect));
2133                 goto out_done;
2134         }
2135
2136         /*
2137          * Adding new commands here don't forget to update
2138          * scst_is_cmd_local() in scst.h, if necessary
2139          */
2140
2141         if (!(cmd->op_flags & SCST_LOCAL_EXEC_NEEDED)) {
2142                 res = SCST_EXEC_NOT_COMPLETED;
2143                 goto out;
2144         }
2145
2146         switch (cmd->cdb[0]) {
2147         case MODE_SELECT:
2148         case MODE_SELECT_10:
2149         case LOG_SELECT:
2150                 res = scst_pre_select(cmd);
2151                 break;
2152         case RESERVE:
2153         case RESERVE_10:
2154                 res = scst_reserve_local(cmd);
2155                 break;
2156         case RELEASE:
2157         case RELEASE_10:
2158                 res = scst_release_local(cmd);
2159                 break;
2160         case REPORT_LUNS:
2161                 res = scst_report_luns_local(cmd);
2162                 break;
2163         case REQUEST_SENSE:
2164                 res = scst_request_sense_local(cmd);
2165                 break;
2166         default:
2167                 res = SCST_EXEC_NOT_COMPLETED;
2168                 break;
2169         }
2170
2171 out:
2172         TRACE_EXIT_RES(res);
2173         return res;
2174
2175 out_done:
2176         /* Report the result */
2177         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2178         res = SCST_EXEC_COMPLETED;
2179         goto out;
2180 }
2181
2182 static int scst_local_exec(struct scst_cmd *cmd)
2183 {
2184         int res;
2185
2186         TRACE_ENTRY();
2187
2188         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2189         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2190         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2191
2192         __scst_cmd_get(cmd);
2193
2194         res = scst_do_local_exec(cmd);
2195         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2196                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2197         else if (res == SCST_EXEC_COMPLETED)
2198                 scst_post_exec_sn(cmd, true);
2199         else
2200                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2201
2202         __scst_cmd_put(cmd);
2203
2204         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2205         TRACE_EXIT_RES(res);
2206         return res;
2207 }
2208
2209 static int scst_exec(struct scst_cmd **active_cmd)
2210 {
2211         struct scst_cmd *cmd = *active_cmd;
2212         struct scst_cmd *ref_cmd;
2213         struct scst_device *dev = cmd->dev;
2214         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2215
2216         TRACE_ENTRY();
2217
2218         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2219                 goto out;
2220
2221         /* To protect tgt_dev */
2222         ref_cmd = cmd;
2223         __scst_cmd_get(ref_cmd);
2224
2225         count = 0;
2226         while (1) {
2227                 int rc;
2228
2229                 cmd->sent_for_exec = 1;
2230                 /*
2231                  * To sync with scst_abort_cmd(). The above assignment must
2232                  * be before SCST_CMD_ABORTED test, done later in
2233                  * scst_check_local_events(). It's far from here, so the order
2234                  * is virtually guaranteed, but let's have it just in case.
2235                  */
2236                 smp_mb();
2237
2238                 cmd->scst_cmd_done = scst_cmd_done_local;
2239                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2240
2241                 rc = scst_do_local_exec(cmd);
2242                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2243                         /* Nothing to do */;
2244                 else if (rc == SCST_EXEC_NEED_THREAD) {
2245                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2246                                 "thread context, rescheduling");
2247                         scst_dec_on_dev_cmd(cmd);
2248                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2249                         break;
2250                 } else {
2251                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2252                         goto done;
2253                 }
2254
2255                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2256
2257                 rc = scst_do_real_exec(cmd);
2258                 if (likely(rc == SCST_EXEC_COMPLETED))
2259                         /* Nothing to do */;
2260                 else if (rc == SCST_EXEC_NEED_THREAD) {
2261                         TRACE_DBG("scst_real_exec() requested thread "
2262                                 "context, rescheduling (cmd %p)", cmd);
2263                         scst_dec_on_dev_cmd(cmd);
2264                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2265                         break;
2266                 } else
2267                         sBUG();
2268
2269 done:
2270                 count++;
2271
2272                 cmd = scst_post_exec_sn(cmd, false);
2273                 if (cmd == NULL)
2274                         break;
2275
2276                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2277                         break;
2278
2279                 __scst_cmd_put(ref_cmd);
2280                 ref_cmd = cmd;
2281                 __scst_cmd_get(ref_cmd);
2282         }
2283
2284         *active_cmd = cmd;
2285
2286         if (count == 0)
2287                 goto out_put;
2288
2289         if (dev->scsi_dev != NULL)
2290                 generic_unplug_device(dev->scsi_dev->request_queue);
2291
2292 out_put:
2293         __scst_cmd_put(ref_cmd);
2294         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2295
2296 out:
2297         TRACE_EXIT_RES(res);
2298         return res;
2299 }
2300
2301 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2302 {
2303         int res;
2304         struct scst_cmd *cmd = *active_cmd;
2305         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2306         typeof(tgt_dev->expected_sn) expected_sn;
2307
2308         TRACE_ENTRY();
2309
2310         if (unlikely(cmd->internal))
2311                 goto exec;
2312
2313         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2314                 goto exec;
2315
2316         sBUG_ON(!cmd->sn_set);
2317
2318         expected_sn = tgt_dev->expected_sn;
2319         /* Optimized for lockless fast path */
2320         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2321                 spin_lock_irq(&tgt_dev->sn_lock);
2322
2323                 tgt_dev->def_cmd_count++;
2324                 /*
2325                  * Memory barrier is needed here to implement lockless fast
2326                  * path. We need the exact order of read and write between
2327                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2328                  * when expected_sn was changed to be equal to cmd->sn while
2329                  * we are queuing cmd the deferred list after the expected_sn
2330                  * below. It will lead to a forever stuck command. But with
2331                  * the barrier in such case __scst_check_deferred_commands()
2332                  * will be called and it will take sn_lock, so we will be
2333                  * synchronized.
2334                  */
2335                 smp_mb();
2336
2337                 expected_sn = tgt_dev->expected_sn;
2338                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2339                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2340                                               &cmd->cmd_flags))) {
2341                                 /* Necessary to allow aborting out of sn cmds */
2342                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2343                                         "(tag %llu, sn %lu)", cmd,
2344                                         (long long unsigned)cmd->tag, cmd->sn);
2345                                 tgt_dev->def_cmd_count--;
2346                                 scst_set_cmd_abnormal_done_state(cmd);
2347                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2348                         } else {
2349                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2350                                         "expected_sn=%ld)", cmd, cmd->sn,
2351                                         cmd->sn_set, expected_sn);
2352                                 list_add_tail(&cmd->sn_cmd_list_entry,
2353                                               &tgt_dev->deferred_cmd_list);
2354                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2355                         }
2356                         spin_unlock_irq(&tgt_dev->sn_lock);
2357                         goto out;
2358                 } else {
2359                         TRACE_SN("Somebody incremented expected_sn %ld, "
2360                                 "continuing", expected_sn);
2361                         tgt_dev->def_cmd_count--;
2362                         spin_unlock_irq(&tgt_dev->sn_lock);
2363                 }
2364         }
2365
2366 exec:
2367         res = scst_exec(active_cmd);
2368
2369 out:
2370         TRACE_EXIT_HRES(res);
2371         return res;
2372 }
2373
2374 /* No locks supposed to be held */
2375 static int scst_check_sense(struct scst_cmd *cmd)
2376 {
2377         int res = 0;
2378         struct scst_device *dev = cmd->dev;
2379
2380         TRACE_ENTRY();
2381
2382         if (unlikely(cmd->ua_ignore))
2383                 goto out;
2384
2385         /* If we had internal bus reset behind us, set the command error UA */
2386         if ((dev->scsi_dev != NULL) &&
2387             unlikely(cmd->host_status == DID_RESET) &&
2388             scst_is_ua_command(cmd)) {
2389                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2390                       dev->scsi_dev->was_reset, cmd->host_status);
2391                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2392                 /* It looks like it is safe to clear was_reset here */
2393                 dev->scsi_dev->was_reset = 0;
2394         }
2395
2396         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2397             SCST_SENSE_VALID(cmd->sense)) {
2398                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2399                         cmd->sense_valid_len);
2400
2401                 /* Check Unit Attention Sense Key */
2402                 if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) {
2403                         if (scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2404                                         SCST_SENSE_ASC_VALID,
2405                                         0, SCST_SENSE_ASC_UA_RESET, 0)) {
2406                                 if (cmd->double_ua_possible) {
2407                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2408                                                 "detected for device %p", dev);
2409                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2410                                                 " %p (tag %llu)", cmd,
2411                                                 (long long unsigned)cmd->tag);
2412
2413                                         cmd->status = 0;
2414                                         cmd->msg_status = 0;
2415                                         cmd->host_status = DID_OK;
2416                                         cmd->driver_status = 0;
2417
2418                                         mempool_free(cmd->sense,
2419                                                      scst_sense_mempool);
2420                                         cmd->sense = NULL;
2421
2422                                         scst_check_restore_sg_buff(cmd);
2423
2424                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2425                                         cmd->data_direction =
2426                                                 cmd->dbl_ua_orig_data_direction;
2427                                         cmd->resp_data_len =
2428                                                 cmd->dbl_ua_orig_resp_data_len;
2429
2430                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2431                                         cmd->retry = 1;
2432                                         res = 1;
2433                                         goto out;
2434                                 }
2435                         }
2436                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2437                                 cmd->sense_valid_len);
2438                 }
2439         }
2440
2441         if (unlikely(cmd->double_ua_possible)) {
2442                 if (scst_is_ua_command(cmd)) {
2443                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2444                                 "cmd %p)", dev, cmd);
2445                         /*
2446                          * Lock used to protect other flags in the bitfield
2447                          * (just in case, actually). Those flags can't be
2448                          * changed in parallel, because the device is
2449                          * serialized.
2450                          */
2451                         spin_lock_bh(&dev->dev_lock);
2452                         dev->dev_double_ua_possible = 0;
2453                         spin_unlock_bh(&dev->dev_lock);
2454                 }
2455         }
2456
2457 out:
2458         TRACE_EXIT_RES(res);
2459         return res;
2460 }
2461
2462 static int scst_check_auto_sense(struct scst_cmd *cmd)
2463 {
2464         int res = 0;
2465
2466         TRACE_ENTRY();
2467
2468         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2469             (!SCST_SENSE_VALID(cmd->sense) ||
2470              SCST_NO_SENSE(cmd->sense))) {
2471                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2472                       "cmd->status=%x, cmd->msg_status=%x, "
2473                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2474                       cmd->status, cmd->msg_status, cmd->host_status,
2475                       cmd->driver_status, cmd);
2476                 res = 1;
2477         } else if (unlikely(cmd->host_status)) {
2478                 if ((cmd->host_status == DID_REQUEUE) ||
2479                     (cmd->host_status == DID_IMM_RETRY) ||
2480                     (cmd->host_status == DID_SOFT_ERROR) ||
2481                     (cmd->host_status == DID_ABORT)) {
2482                         scst_set_busy(cmd);
2483                 } else {
2484                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2485                                 "received, returning HARDWARE ERROR instead "
2486                                 "(cmd %p)", cmd->host_status, cmd);
2487                         scst_set_cmd_error(cmd,
2488                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2489                 }
2490         }
2491
2492         TRACE_EXIT_RES(res);
2493         return res;
2494 }
2495
2496 static int scst_pre_dev_done(struct scst_cmd *cmd)
2497 {
2498         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2499
2500         TRACE_ENTRY();
2501
2502         if (unlikely(scst_check_auto_sense(cmd))) {
2503                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2504                             "without sense data (opcode 0x%x), issuing "
2505                             "REQUEST SENSE", cmd->cdb[0]);
2506                 rc = scst_prepare_request_sense(cmd);
2507                 if (rc == 0)
2508                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2509                 else {
2510                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2511                                     "returning HARDWARE ERROR");
2512                         scst_set_cmd_error(cmd,
2513                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2514                 }
2515                 goto out;
2516         } else if (unlikely(scst_check_sense(cmd)))
2517                 goto out;
2518
2519         if (likely(scsi_status_is_good(cmd->status))) {
2520                 unsigned char type = cmd->dev->type;
2521                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2522                               cmd->cdb[0] == MODE_SENSE_10)) &&
2523                     (cmd->tgt_dev->acg_dev->rd_only || cmd->dev->swp ||
2524                      cmd->dev->rd_only) &&
2525                     (type == TYPE_DISK ||
2526                      type == TYPE_WORM ||
2527                      type == TYPE_MOD ||
2528                      type == TYPE_TAPE)) {
2529                         int32_t length;
2530                         uint8_t *address;
2531                         bool err = false;
2532
2533                         length = scst_get_buf_first(cmd, &address);
2534                         if (length < 0) {
2535                                 PRINT_ERROR("%s", "Unable to get "
2536                                         "MODE_SENSE buffer");
2537                                 scst_set_cmd_error(cmd,
2538                                         SCST_LOAD_SENSE(
2539                                                 scst_sense_hardw_error));
2540                                 err = true;
2541                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2542                                 address[2] |= 0x80;   /* Write Protect*/
2543                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2544                                 address[3] |= 0x80;   /* Write Protect*/
2545                         scst_put_buf(cmd, address);
2546
2547                         if (err)
2548                                 goto out;
2549                 }
2550
2551                 /*
2552                  * Check and clear NormACA option for the device, if necessary,
2553                  * since we don't support ACA
2554                  */
2555                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2556                     /* Std INQUIRY data (no EVPD) */
2557                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2558                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2559                         uint8_t *buffer;
2560                         int buflen;
2561                         bool err = false;
2562
2563                         /* ToDo: all pages ?? */
2564                         buflen = scst_get_buf_first(cmd, &buffer);
2565                         if (buflen > SCST_INQ_BYTE3) {
2566 #ifdef CONFIG_SCST_EXTRACHECKS
2567                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2568                                         PRINT_INFO("NormACA set for device: "
2569                                             "lun=%lld, type 0x%02x. Clear it, "
2570                                             "since it's unsupported.",
2571                                             (long long unsigned int)cmd->lun,
2572                                             buffer[0]);
2573                                 }
2574 #endif
2575                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2576                         } else if (buflen != 0) {
2577                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2578                                     "buffer");
2579                                 scst_set_cmd_error(cmd,
2580                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2581                                 err = true;
2582                         }
2583                         if (buflen > 0)
2584                                 scst_put_buf(cmd, buffer);
2585
2586                         if (err)
2587                                 goto out;
2588                 }
2589
2590                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2591                     (cmd->cdb[0] == MODE_SELECT_10) ||
2592                     (cmd->cdb[0] == LOG_SELECT))) {
2593                         TRACE(TRACE_SCSI,
2594                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2595                                 (long long unsigned int)cmd->lun);
2596                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2597                         goto out;
2598                 }
2599         } else {
2600                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2601                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2602                                         &cmd->tgt_dev->tgt_dev_flags)) {
2603                                 struct scst_tgt_dev *tgt_dev_tmp;
2604                                 struct scst_device *dev = cmd->dev;
2605
2606                                 TRACE(TRACE_SCSI,
2607                                         "Real RESERVE failed lun=%lld, "
2608                                         "status=%x",
2609                                         (long long unsigned int)cmd->lun,
2610                                         cmd->status);
2611                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2612                                         cmd->sense_valid_len);
2613
2614                                 /* Clearing the reservation */
2615                                 spin_lock_bh(&dev->dev_lock);
2616                                 list_for_each_entry(tgt_dev_tmp,
2617                                                     &dev->dev_tgt_dev_list,
2618                                                     dev_tgt_dev_list_entry) {
2619                                         clear_bit(SCST_TGT_DEV_RESERVED,
2620                                                 &tgt_dev_tmp->tgt_dev_flags);
2621                                 }
2622                                 dev->dev_reserved = 0;
2623                                 spin_unlock_bh(&dev->dev_lock);
2624                         }
2625                 }
2626
2627                 /* Check for MODE PARAMETERS CHANGED UA */
2628                 if ((cmd->dev->scsi_dev != NULL) &&
2629                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2630                     scst_is_ua_sense(cmd->sense, cmd->sense_valid_len) &&
2631                     scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2632                                         SCST_SENSE_ASCx_VALID,
2633                                         0, 0x2a, 0x01)) {
2634                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun "
2635                                 "%lld)", (long long unsigned int)cmd->lun);
2636                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2637                         goto out;
2638                 }
2639         }
2640
2641         cmd->state = SCST_CMD_STATE_DEV_DONE;
2642
2643 out:
2644         TRACE_EXIT_RES(res);
2645         return res;
2646 }
2647
2648 static int scst_mode_select_checks(struct scst_cmd *cmd)
2649 {
2650         int res = SCST_CMD_STATE_RES_CONT_SAME;
2651         int atomic = scst_cmd_atomic(cmd);
2652
2653         TRACE_ENTRY();
2654
2655         if (likely(scsi_status_is_good(cmd->status))) {
2656                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2657                     (cmd->cdb[0] == MODE_SELECT_10) ||
2658                     (cmd->cdb[0] == LOG_SELECT))) {
2659                         struct scst_device *dev = cmd->dev;
2660                         int sl;
2661                         uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN];
2662
2663                         if (atomic && (dev->scsi_dev != NULL)) {
2664                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2665                                         "context required");
2666                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2667                                 goto out;
2668                         }
2669
2670                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2671                                 "setting the SELECT UA (lun=%lld)",
2672                                 (long long unsigned int)cmd->lun);
2673
2674                         spin_lock_bh(&dev->dev_lock);
2675                         if (cmd->cdb[0] == LOG_SELECT) {
2676                                 sl = scst_set_sense(sense_buffer,
2677                                         sizeof(sense_buffer),
2678                                         dev->d_sense,
2679                                         UNIT_ATTENTION, 0x2a, 0x02);
2680                         } else {
2681                                 sl = scst_set_sense(sense_buffer,
2682                                         sizeof(sense_buffer),
2683                                         dev->d_sense,
2684                                         UNIT_ATTENTION, 0x2a, 0x01);
2685                         }
2686                         scst_dev_check_set_local_UA(dev, cmd, sense_buffer, sl);
2687                         spin_unlock_bh(&dev->dev_lock);
2688
2689                         if (dev->scsi_dev != NULL)
2690                                 scst_obtain_device_parameters(dev);
2691                 }
2692         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2693                     scst_is_ua_sense(cmd->sense, cmd->sense_valid_len) &&
2694                      /* mode parameters changed */
2695                     (scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2696                                         SCST_SENSE_ASCx_VALID,
2697                                         0, 0x2a, 0x01) ||
2698                      scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2699                                         SCST_SENSE_ASC_VALID,
2700                                         0, 0x29, 0) /* reset */ ||
2701                      scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2702                                         SCST_SENSE_ASC_VALID,
2703                                         0, 0x28, 0) /* medium changed */ ||
2704                      /* cleared by another ini (just in case) */
2705                      scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
2706                                         SCST_SENSE_ASC_VALID,
2707                                         0, 0x2F, 0))) {
2708                 if (atomic) {
2709                         TRACE_DBG("Possible parameters changed UA %x: "
2710                                 "thread context required", cmd->sense[12]);
2711                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2712                         goto out;
2713                 }
2714
2715                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2716                         "(LUN %lld): getting new parameters", cmd->sense[12],
2717                         (long long unsigned int)cmd->lun);
2718
2719                 scst_obtain_device_parameters(cmd->dev);
2720         } else
2721                 sBUG();
2722
2723         cmd->state = SCST_CMD_STATE_DEV_DONE;
2724
2725 out:
2726         TRACE_EXIT_HRES(res);
2727         return res;
2728 }
2729
2730 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2731 {
2732         if (likely(cmd->sn_set))
2733                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2734
2735         scst_make_deferred_commands_active(cmd->tgt_dev);
2736 }
2737
2738 static int scst_dev_done(struct scst_cmd *cmd)
2739 {
2740         int res = SCST_CMD_STATE_RES_CONT_SAME;
2741         int state;
2742         struct scst_device *dev = cmd->dev;
2743
2744         TRACE_ENTRY();
2745
2746         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2747
2748         if (likely(!scst_is_cmd_local(cmd)) &&
2749             likely(dev->handler->dev_done != NULL)) {
2750                 int rc;
2751
2752                 if (unlikely(!dev->handler->dev_done_atomic &&
2753                              scst_cmd_atomic(cmd))) {
2754                         /*
2755                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2756                          * optimization.
2757                          */
2758                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2759                               "context, rescheduling", dev->handler->name);
2760                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2761                         goto out;
2762                 }
2763
2764                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2765                         dev->handler->name, cmd);
2766                 scst_set_cur_start(cmd);
2767                 rc = dev->handler->dev_done(cmd);
2768                 scst_set_dev_done_time(cmd);
2769                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2770                       dev->handler->name, rc);
2771                 if (rc != SCST_CMD_STATE_DEFAULT)
2772                         state = rc;
2773         }
2774
2775         switch (state) {
2776         case SCST_CMD_STATE_PRE_XMIT_RESP:
2777         case SCST_CMD_STATE_DEV_PARSE:
2778         case SCST_CMD_STATE_PRE_PARSE:
2779         case SCST_CMD_STATE_PREPARE_SPACE:
2780         case SCST_CMD_STATE_RDY_TO_XFER:
2781         case SCST_CMD_STATE_TGT_PRE_EXEC:
2782         case SCST_CMD_STATE_SEND_FOR_EXEC:
2783         case SCST_CMD_STATE_LOCAL_EXEC:
2784         case SCST_CMD_STATE_REAL_EXEC:
2785         case SCST_CMD_STATE_PRE_DEV_DONE:
2786         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2787         case SCST_CMD_STATE_DEV_DONE:
2788         case SCST_CMD_STATE_XMIT_RESP:
2789         case SCST_CMD_STATE_FINISHED:
2790         case SCST_CMD_STATE_FINISHED_INTERNAL:
2791                 cmd->state = state;
2792                 break;
2793
2794         case SCST_CMD_STATE_NEED_THREAD_CTX:
2795                 TRACE_DBG("Dev handler %s dev_done() requested "
2796                       "thread context, rescheduling",
2797                       dev->handler->name);
2798                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2799                 break;
2800
2801         default:
2802                 if (state >= 0) {
2803                         PRINT_ERROR("Dev handler %s dev_done() returned "
2804                                 "invalid cmd state %d",
2805                                 dev->handler->name, state);
2806                 } else {
2807                         PRINT_ERROR("Dev handler %s dev_done() returned "
2808                                 "error %d", dev->handler->name,
2809                                 state);
2810                 }
2811                 scst_set_cmd_error(cmd,
2812                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2813                 scst_set_cmd_abnormal_done_state(cmd);
2814                 break;
2815         }
2816
2817         if (cmd->needs_unblocking)
2818                 scst_unblock_dev_cmd(cmd);
2819
2820         if (likely(cmd->dec_on_dev_needed))
2821                 scst_dec_on_dev_cmd(cmd);
2822
2823         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2824                 scst_inc_check_expected_sn(cmd);
2825
2826         if (unlikely(cmd->internal))
2827                 cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
2828
2829 out:
2830         TRACE_EXIT_HRES(res);
2831         return res;
2832 }
2833
2834 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2835 {
2836         int res;
2837         struct scst_session *sess = cmd->sess;
2838
2839         TRACE_ENTRY();
2840
2841         EXTRACHECKS_BUG_ON(cmd->internal);
2842
2843 #ifdef CONFIG_SCST_DEBUG_TM
2844         if (cmd->tm_dbg_delayed &&
2845                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2846                 if (scst_cmd_atomic(cmd)) {
2847                         TRACE_MGMT_DBG("%s",
2848                                 "DEBUG_TM delayed cmd needs a thread");
2849                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2850                         return res;
2851                 }
2852                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2853                         cmd, cmd->tag);
2854                 schedule_timeout_uninterruptible(HZ);
2855         }
2856 #endif
2857
2858         if (likely(cmd->tgt_dev != NULL)) {
2859                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2860                 atomic_dec(&cmd->dev->dev_cmd_count);
2861                 /* If expected values not set, expected direction is UNKNOWN */
2862                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
2863                         atomic_dec(&cmd->dev->write_cmd_count);
2864
2865                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2866                         scst_on_hq_cmd_response(cmd);
2867
2868                 if (unlikely(!cmd->sent_for_exec)) {
2869                         TRACE_SN("cmd %p was not sent to mid-lev"
2870                                 " (sn %ld, set %d)",
2871                                 cmd, cmd->sn, cmd->sn_set);
2872                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2873                         cmd->sent_for_exec = 1;
2874                 }
2875         }
2876
2877         /*
2878          * If we don't remove cmd from the search list here, before
2879          * submitting it for transmittion, we will have a race, when for
2880          * some reason cmd's release is delayed after transmittion and
2881          * initiator sends cmd with the same tag => it is possible that
2882          * a wrong cmd will be found by find() functions.
2883          */
2884         spin_lock_irq(&sess->sess_list_lock);
2885         list_move_tail(&cmd->sess_cmd_list_entry,
2886                 &sess->after_pre_xmit_cmd_list);
2887         spin_unlock_irq(&sess->sess_list_lock);
2888
2889         cmd->done = 1;
2890         smp_mb(); /* to sync with scst_abort_cmd() */
2891
2892         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2893                 scst_xmit_process_aborted_cmd(cmd);
2894         else if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION))
2895                 scst_store_sense(cmd);
2896
2897         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2898                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2899                                 " skipping",
2900                                 cmd, (long long unsigned int)cmd->tag);
2901                 cmd->state = SCST_CMD_STATE_FINISHED;
2902                 res = SCST_CMD_STATE_RES_CONT_SAME;
2903                 goto out;
2904         }
2905
2906         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2907         res = SCST_CMD_STATE_RES_CONT_SAME;
2908
2909 out:
2910         TRACE_EXIT_HRES(res);
2911         return res;
2912 }
2913
2914 static int scst_xmit_response(struct scst_cmd *cmd)
2915 {
2916         struct scst_tgt_template *tgtt = cmd->tgtt;
2917         int res, rc;
2918
2919         TRACE_ENTRY();
2920
2921         EXTRACHECKS_BUG_ON(cmd->internal);
2922
2923         if (unlikely(!tgtt->xmit_response_atomic &&
2924                      scst_cmd_atomic(cmd))) {
2925                 /*
2926                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2927                  * optimization.
2928                  */
2929                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2930                               "context, rescheduling", tgtt->name);
2931                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2932                 goto out;
2933         }
2934
2935         while (1) {
2936                 int finished_cmds = atomic_read(&cmd->tgt->finished_cmds);
2937
2938                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2939                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2940
2941                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2942
2943 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2944                 if (trace_flag & TRACE_SND_BOT) {
2945                         int i;
2946                         struct scatterlist *sg;
2947                         if (cmd->tgt_sg != NULL)
2948                                 sg = cmd->tgt_sg;
2949                         else
2950                                 sg = cmd->sg;
2951                         if (sg != NULL) {
2952                                 TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2953                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
2954                                         cmd, cmd->tgt_sg_cnt, sg,
2955                                         (void *)sg_page(&sg[0]));
2956                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2957                                         PRINT_BUFF_FLAG(TRACE_SND_BOT,
2958                                                 "Xmitting sg", sg_virt(&sg[i]),
2959                                                 sg[i].length);
2960                                 }
2961                         }
2962                 }
2963 #endif
2964
2965                 if (tgtt->on_hw_pending_cmd_timeout != NULL) {
2966                         struct scst_session *sess = cmd->sess;
2967                         cmd->hw_pending_start = jiffies;
2968                         cmd->cmd_hw_pending = 1;
2969                         if (!test_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED, &sess->sess_aflags)) {
2970                                 TRACE_DBG("Sched HW pending work for sess %p "
2971                                         "(max time %d)", sess,
2972                                         tgtt->max_hw_pending_time);
2973                                 set_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED,
2974                                         &sess->sess_aflags);
2975                                 schedule_delayed_work(&sess->hw_pending_work,
2976                                         tgtt->max_hw_pending_time * HZ);
2977                         }
2978                 }
2979
2980                 scst_set_cur_start(cmd);
2981
2982 #ifdef CONFIG_SCST_DEBUG_RETRY
2983                 if (((scst_random() % 100) == 77))
2984                         rc = SCST_TGT_RES_QUEUE_FULL;
2985                 else
2986 #endif
2987                         rc = tgtt->xmit_response(cmd);
2988                 TRACE_DBG("xmit_response() returned %d", rc);
2989
2990                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2991                         goto out;
2992
2993                 scst_set_xmit_time(cmd);
2994
2995                 cmd->cmd_hw_pending = 0;
2996
2997                 /* Restore the previous state */
2998                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2999
3000                 switch (rc) {
3001                 case SCST_TGT_RES_QUEUE_FULL:
3002                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
3003                                 break;
3004                         else
3005                                 continue;
3006
3007                 case SCST_TGT_RES_NEED_THREAD_CTX:
3008                         TRACE_DBG("Target driver %s xmit_response() "
3009                               "requested thread context, rescheduling",
3010                               tgtt->name);
3011                         res = SCST_CMD_STATE_RES_NEED_THREAD;
3012                         break;
3013
3014                 default:
3015                         goto out_error;
3016                 }
3017                 break;
3018         }
3019
3020 out:
3021         /* Caution: cmd can be already dead here */
3022         TRACE_EXIT_HRES(res);
3023         return res;
3024
3025 out_error:
3026         if (rc == SCST_TGT_RES_FATAL_ERROR) {
3027                 PRINT_ERROR("Target driver %s xmit_response() returned "
3028                         "fatal error", tgtt->name);
3029         } else {
3030                 PRINT_ERROR("Target driver %s xmit_response() returned "
3031                         "invalid value %d", tgtt->name, rc);
3032         }
3033         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
3034         cmd->state = SCST_CMD_STATE_FINISHED;
3035         res = SCST_CMD_STATE_RES_CONT_SAME;
3036         goto out;
3037 }
3038
3039 void scst_tgt_cmd_done(struct scst_cmd *cmd,
3040         enum scst_exec_context pref_context)
3041 {
3042         TRACE_ENTRY();
3043
3044         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
3045
3046         scst_set_xmit_time(cmd);
3047
3048         cmd->cmd_hw_pending = 0;
3049
3050         cmd->state = SCST_CMD_STATE_FINISHED;
3051         scst_process_redirect_cmd(cmd, pref_context, 1);
3052
3053         TRACE_EXIT();
3054         return;
3055 }
3056 EXPORT_SYMBOL(scst_tgt_cmd_done);
3057
3058 static int scst_finish_cmd(struct scst_cmd *cmd)
3059 {
3060         int res;
3061         struct scst_session *sess = cmd->sess;
3062
3063         TRACE_ENTRY();
3064
3065         scst_update_lat_stats(cmd);
3066
3067         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
3068                 if ((cmd->tgt_dev != NULL) &&
3069                     scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) {
3070                         /* This UA delivery failed, so we need to requeue it */
3071                         if (scst_cmd_atomic(cmd) &&
3072                             scst_is_ua_global(cmd->sense, cmd->sense_valid_len)) {
3073                                 TRACE_MGMT_DBG("Requeuing of global UA for "
3074                                         "failed cmd %p needs a thread", cmd);
3075                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
3076                                 goto out;
3077                         }
3078                         scst_requeue_ua(cmd);
3079                 }
3080         }
3081
3082         atomic_dec(&sess->sess_cmd_count);
3083
3084         spin_lock_irq(&sess->sess_list_lock);
3085         list_del(&cmd->sess_cmd_list_entry);
3086         spin_unlock_irq(&sess->sess_list_lock);
3087
3088         cmd->finished = 1;
3089         smp_mb(); /* to sync with scst_abort_cmd() */
3090
3091         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
3092                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
3093                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
3094                         atomic_read(&scst_cmd_count));
3095
3096                 scst_finish_cmd_mgmt(cmd);
3097         }
3098
3099         __scst_cmd_put(cmd);
3100
3101         res = SCST_CMD_STATE_RES_CONT_NEXT;
3102
3103 out:
3104         TRACE_EXIT_HRES(res);
3105         return res;
3106 }
3107
3108 /*
3109  * No locks, but it must be externally serialized (see comment for
3110  * scst_cmd_init_done() in scst.h)
3111  */
3112 static void scst_cmd_set_sn(struct scst_cmd *cmd)
3113 {
3114         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
3115         unsigned long flags;
3116
3117         TRACE_ENTRY();
3118
3119         if (scst_is_implicit_hq(cmd)) {
3120                 TRACE_SN("Implicit HQ cmd %p", cmd);
3121                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
3122         }
3123
3124         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
3125
3126         /* Optimized for lockless fast path */
3127
3128         scst_check_debug_sn(cmd);
3129
3130         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
3131                 /*
3132                  * Not the best way, but good enough until there is a
3133                  * possibility to specify queue type during pass-through
3134                  * commands submission.
3135                  */
3136                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
3137         }
3138
3139         switch (cmd->queue_type) {
3140         case SCST_CMD_QUEUE_SIMPLE:
3141         case SCST_CMD_QUEUE_UNTAGGED:
3142 #if 0 /* left for future performance investigations */
3143                 if (scst_cmd_is_expected_set(cmd)) {
3144                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
3145                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
3146                                 goto ordered;
3147                 } else
3148                         goto ordered;
3149 #endif
3150                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
3151                         /*
3152                          * atomic_inc_return() implies memory barrier to sync
3153                          * with scst_inc_expected_sn()
3154                          */
3155                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
3156                                 tgt_dev->curr_sn++;
3157                                 TRACE_SN("Incremented curr_sn %ld",
3158                                         tgt_dev->curr_sn);
3159                         }
3160                         cmd->sn_slot = tgt_dev->cur_sn_slot;
3161                         cmd->sn = tgt_dev->curr_sn;
3162
3163                         tgt_dev->prev_cmd_ordered = 0;
3164                 } else {
3165                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
3166                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
3167                         goto ordered;
3168                 }
3169                 break;
3170
3171         case SCST_CMD_QUEUE_ORDERED:
3172                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3173 ordered:
3174                 if (!tgt_dev->prev_cmd_ordered) {
3175                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3176                         if (tgt_dev->num_free_sn_slots >= 0) {
3177                                 tgt_dev->num_free_sn_slots--;
3178                                 if (tgt_dev->num_free_sn_slots >= 0) {
3179                                         int i = 0;
3180                                         /* Commands can finish in any order, so
3181                                          * we don't know which slot is empty.
3182                                          */
3183                                         while (1) {
3184                                                 tgt_dev->cur_sn_slot++;
3185                                                 if (tgt_dev->cur_sn_slot ==
3186                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3187                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3188
3189                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3190                                                         break;
3191
3192                                                 i++;
3193                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3194                                         }
3195                                         TRACE_SN("New cur SN slot %zd",
3196                                                 tgt_dev->cur_sn_slot -
3197                                                 tgt_dev->sn_slots);
3198                                 }
3199                         }
3200                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3201                 }
3202                 tgt_dev->prev_cmd_ordered = 1;
3203                 tgt_dev->curr_sn++;
3204                 cmd->sn = tgt_dev->curr_sn;
3205                 break;
3206
3207         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3208                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3209                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3210                 tgt_dev->hq_cmd_count++;
3211                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3212                 cmd->hq_cmd_inced = 1;
3213                 goto out;
3214
3215         default:
3216                 sBUG();
3217         }
3218
3219         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3220                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3221                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3222                 atomic_read(tgt_dev->cur_sn_slot),
3223                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3224                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3225
3226         cmd->sn_set = 1;
3227
3228 out:
3229         TRACE_EXIT();
3230         return;
3231 }
3232
3233 /*
3234  * Returns 0 on success, > 0 when we need to wait for unblock,
3235  * < 0 if there is no device (lun) or device type handler.
3236  *
3237  * No locks, but might be on IRQ, protection is done by the
3238  * suspended activity.
3239  */
3240 static int scst_translate_lun(struct scst_cmd *cmd)
3241 {
3242         struct scst_tgt_dev *tgt_dev = NULL;
3243         int res;
3244
3245         TRACE_ENTRY();
3246
3247         /* See comment about smp_mb() in scst_suspend_activity() */
3248         __scst_get(1);
3249
3250         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3251                 struct list_head *sess_tgt_dev_list_head =
3252                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3253                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3254                         (long long unsigned int)cmd->lun);
3255                 res = -1;
3256                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3257                                 sess_tgt_dev_list_entry) {
3258                         if (tgt_dev->lun == cmd->lun) {
3259                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3260
3261                                 if (unlikely(tgt_dev->dev->handler ==
3262                                                 &scst_null_devtype)) {
3263                                         PRINT_INFO("Dev handler for device "
3264                                           "%lld is NULL, the device will not "
3265                                           "be visible remotely",
3266                                            (long long unsigned int)cmd->lun);
3267                                         break;
3268                                 }
3269
3270                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3271                                 cmd->tgt_dev = tgt_dev;
3272                                 cmd->dev = tgt_dev->dev;
3273
3274                                 res = 0;
3275                                 break;
3276                         }
3277                 }
3278                 if (res != 0) {
3279                         TRACE(TRACE_MINOR,
3280                                 "tgt_dev for LUN %lld not found, command to "
3281                                 "unexisting LU?",
3282                                 (long long unsigned int)cmd->lun);
3283                         __scst_put();
3284                 }
3285         } else {
3286                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3287                 __scst_put();
3288                 res = 1;
3289         }
3290
3291         TRACE_EXIT_RES(res);
3292         return res;
3293 }
3294
3295 /*
3296  * No locks, but might be on IRQ
3297  *
3298  * Returns 0 on success, > 0 when we need to wait for unblock,
3299  * < 0 if there is no device (lun) or device type handler.
3300  */
3301 static int __scst_init_cmd(struct scst_cmd *cmd)
3302 {
3303         int res = 0;
3304
3305         TRACE_ENTRY();
3306
3307         res = scst_translate_lun(cmd);
3308         if (likely(res == 0)) {
3309                 int cnt;
3310                 bool failure = false;
3311
3312                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3313
3314                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3315                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3316                         TRACE(TRACE_MGMT_MINOR,
3317                                 "Too many pending commands (%d) in "
3318                                 "session, returning BUSY to initiator \"%s\"",
3319                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3320                                   "Anonymous" : cmd->sess->initiator_name);
3321                         failure = true;
3322                 }
3323
3324                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3325                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3326                         if (!failure) {
3327                                 TRACE(TRACE_MGMT_MINOR,
3328                                         "Too many pending device "
3329                                         "commands (%d), returning BUSY to "
3330                                         "initiator \"%s\"", cnt,
3331                                         (cmd->sess->initiator_name[0] == '\0') ?
3332                                                 "Anonymous" :
3333                                                 cmd->sess->initiator_name);
3334                                 failure = true;
3335                         }
3336                 }
3337
3338                 /* If expected values not set, expected direction is UNKNOWN */
3339                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
3340                         atomic_inc(&cmd->dev->write_cmd_count);
3341
3342                 if (unlikely(failure))
3343                         goto out_busy;
3344
3345                 if (!cmd->set_sn_on_restart_cmd)
3346                         scst_cmd_set_sn(cmd);
3347         } else if (res < 0) {
3348                 TRACE_DBG("Finishing cmd %p", cmd);
3349                 scst_set_cmd_error(cmd,
3350                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3351                 scst_set_cmd_abnormal_done_state(cmd);
3352         } else
3353                 goto out;
3354
3355 out:
3356         TRACE_EXIT_RES(res);
3357         return res;
3358
3359 out_busy:
3360         scst_set_busy(cmd);
3361         scst_set_cmd_abnormal_done_state(cmd);
3362         goto out;
3363 }
3364
3365 /* Called under scst_init_lock and IRQs disabled */
3366 static void scst_do_job_init(void)
3367         __releases(&scst_init_lock)
3368         __acquires(&scst_init_lock)
3369 {
3370         struct scst_cmd *cmd;
3371         int susp;
3372
3373         TRACE_ENTRY();
3374
3375 restart:
3376         /*
3377          * There is no need for read barrier here, because we don't care where
3378          * this check will be done.
3379          */
3380         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3381         if (scst_init_poll_cnt > 0)
3382                 scst_init_poll_cnt--;
3383
3384         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3385                 int rc;
3386                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3387                         continue;
3388                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3389                         spin_unlock_irq(&scst_init_lock);
3390                         rc = __scst_init_cmd(cmd);
3391                         spin_lock_irq(&scst_init_lock);
3392                         if (rc > 0) {
3393                                 TRACE_MGMT_DBG("%s",
3394                                         "FLAG SUSPENDED set, restarting");
3395                                 goto restart;
3396                         }
3397                 } else {
3398                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3399                                        cmd, (long long unsigned int)cmd->tag);
3400                         scst_set_cmd_abnormal_done_state(cmd);
3401                 }
3402
3403                 /*
3404                  * Deleting cmd from init cmd list after __scst_init_cmd()
3405                  * is necessary to keep the check in scst_init_cmd() correct
3406                  * to preserve the commands order.
3407                  *
3408                  * We don't care about the race, when init cmd list is empty
3409                  * and one command detected that it just was not empty, so
3410                  * it's inserting to it, but another command at the same time
3411                  * seeing init cmd list empty and goes directly, because it
3412                  * could affect only commands from the same initiator to the
3413                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3414                  * the order in case of simultaneous such calls anyway.
3415                  */
3416                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3417                 smp_wmb(); /* enforce the required order */
3418                 list_del(&cmd->cmd_list_entry);
3419                 spin_unlock(&scst_init_lock);
3420
3421                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3422                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3423                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3424                         list_add(&cmd->cmd_list_entry,
3425                                 &cmd->cmd_lists->active_cmd_list);
3426                 else
3427                         list_add_tail(&cmd->cmd_list_entry,
3428                                 &cmd->cmd_lists->active_cmd_list);
3429                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3430                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3431
3432                 spin_lock(&scst_init_lock);
3433                 goto restart;
3434         }
3435
3436         /* It isn't really needed, but let's keep it */
3437         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3438                 goto restart;
3439
3440         TRACE_EXIT();
3441         return;
3442 }
3443
3444 static inline int test_init_cmd_list(void)
3445 {
3446         int res = (!list_empty(&scst_init_cmd_list) &&
3447                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3448                   unlikely(kthread_should_stop()) ||
3449                   (scst_init_poll_cnt > 0);
3450         return res;
3451 }
3452
3453 int scst_init_thread(void *arg)
3454 {
3455         TRACE_ENTRY();
3456
3457         PRINT_INFO("Init thread started, PID %d", current->pid);
3458
3459         current->flags |= PF_NOFREEZE;
3460
3461         set_user_nice(current, -10);
3462
3463         spin_lock_irq(&scst_init_lock);
3464         while (!kthread_should_stop()) {
3465                 wait_queue_t wait;
3466                 init_waitqueue_entry(&wait, current);
3467
3468                 if (!test_init_cmd_list()) {
3469                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3470                                                  &wait);
3471                         for (;;) {
3472                                 set_current_state(TASK_INTERRUPTIBLE);
3473                                 if (test_init_cmd_list())
3474                                         break;
3475                                 spin_unlock_irq(&scst_init_lock);
3476                                 schedule();
3477                                 spin_lock_irq(&scst_init_lock);
3478                         }
3479                         set_current_state(TASK_RUNNING);
3480                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3481                 }
3482                 scst_do_job_init();
3483         }
3484         spin_unlock_irq(&scst_init_lock);
3485
3486      &n