e6bac2b11dc53acdc13aef0dce7edd88a4783c75
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38
39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
40 {
41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
42         unsigned long flags;
43
44         spin_lock_irqsave(&t->tasklet_lock, flags);
45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
46                 smp_processor_id());
47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
49
50         tasklet_schedule(&t->tasklet);
51 }
52
53 /*
54  * Must not be called in parallel with scst_unregister_session_ex() for the
55  * same sess
56  */
57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
58                              const uint8_t *lun, int lun_len,
59                              const uint8_t *cdb, int cdb_len, int atomic)
60 {
61         struct scst_cmd *cmd;
62
63         TRACE_ENTRY();
64
65 #ifdef CONFIG_SCST_EXTRACHECKS
66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
67                 PRINT_CRIT_ERROR("%s",
68                         "New cmd while shutting down the session");
69                 sBUG();
70         }
71 #endif
72
73         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
74         if (cmd == NULL)
75                 goto out;
76
77         cmd->sess = sess;
78         cmd->tgt = sess->tgt;
79         cmd->tgtt = sess->tgt->tgtt;
80
81         /*
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100 EXPORT_SYMBOL(scst_rx_cmd);
101
102 /*
103  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
104  * this command should be stopped.
105  */
106 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
107 {
108         int rc, res = 0;
109
110         TRACE_ENTRY();
111
112         /* See the comment in scst_do_job_init() */
113         if (unlikely(!list_empty(&scst_init_cmd_list))) {
114                 TRACE_MGMT_DBG("%s", "init cmd list busy");
115                 goto out_redirect;
116         }
117         /*
118          * Memory barrier isn't necessary here, because CPU appears to
119          * be self-consistent
120          */
121
122         rc = __scst_init_cmd(cmd);
123         if (unlikely(rc > 0))
124                 goto out_redirect;
125         else if (unlikely(rc != 0))
126                 goto out;
127
128         /* Small context optimization */
129         if (((*context == SCST_CONTEXT_TASKLET) ||
130              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
131              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
132               scst_cmd_is_expected_set(cmd)) {
133                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
134                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
135                                         &cmd->tgt_dev->tgt_dev_flags))
136                                 *context = SCST_CONTEXT_THREAD;
137                 } else {
138                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
139                                         &cmd->tgt_dev->tgt_dev_flags))
140                                 *context = SCST_CONTEXT_THREAD;
141                 }
142         }
143
144 out:
145         TRACE_EXIT_RES(res);
146         return res;
147
148 out_redirect:
149         if (cmd->preprocessing_only) {
150                 /*
151                  * Poor man solution for single threaded targets, where
152                  * blocking receiver at least sometimes means blocking all.
153                  */
154                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
155                 scst_set_busy(cmd);
156                 scst_set_cmd_abnormal_done_state(cmd);
157                 /* Keep initiator away from too many BUSY commands */
158                 msleep(50);
159         } else {
160                 unsigned long flags;
161                 spin_lock_irqsave(&scst_init_lock, flags);
162                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
163                         "%d)", cmd, atomic_read(&scst_cmd_count));
164                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
165                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
166                         scst_init_poll_cnt++;
167                 spin_unlock_irqrestore(&scst_init_lock, flags);
168                 wake_up(&scst_init_cmd_list_waitQ);
169                 res = -1;
170         }
171         goto out;
172 }
173
174 #ifdef CONFIG_SCST_MEASURE_LATENCY
175 static inline uint64_t scst_sec_to_nsec(time_t sec)
176 {
177         return (uint64_t)sec * 1000000000;
178 }
179 #endif
180
181 void scst_cmd_init_done(struct scst_cmd *cmd,
182         enum scst_exec_context pref_context)
183 {
184         unsigned long flags;
185         struct scst_session *sess = cmd->sess;
186         int rc;
187
188         TRACE_ENTRY();
189
190 #ifdef CONFIG_SCST_MEASURE_LATENCY
191         {
192                 struct timespec ts;
193                 getnstimeofday(&ts);
194                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
195                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
196                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
197                         ts.tv_nsec);
198         }
199 #endif
200
201         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
202         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
203                 "(cmd %p)", (long long unsigned int)cmd->tag,
204                 (long long unsigned int)cmd->lun, cmd->cdb_len,
205                 cmd->queue_type, cmd);
206         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
207                 cmd->cdb, cmd->cdb_len);
208
209 #ifdef CONFIG_SCST_EXTRACHECKS
210         if (unlikely((in_irq() || irqs_disabled())) &&
211             ((pref_context == SCST_CONTEXT_DIRECT) ||
212              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
213                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
214                         "SCST_CONTEXT_THREAD instead\n", pref_context,
215                         cmd->tgtt->name);
216                 pref_context = SCST_CONTEXT_THREAD;
217         }
218 #endif
219
220         atomic_inc(&sess->sess_cmd_count);
221
222         spin_lock_irqsave(&sess->sess_list_lock, flags);
223
224         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
225                 /*
226                  * We have to always keep command in the search list from the
227                  * very beginning, because otherwise it can be missed during
228                  * TM processing. This check is needed because there might be
229                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
230                  */
231                 if (cmd->search_cmd_list_entry.next == NULL)
232                         list_add_tail(&cmd->search_cmd_list_entry,
233                                 &sess->search_cmd_list);
234                 switch (sess->init_phase) {
235                 case SCST_SESS_IPH_SUCCESS:
236                         break;
237                 case SCST_SESS_IPH_INITING:
238                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
239                                   cmd);
240                         list_add_tail(&cmd->cmd_list_entry,
241                                 &sess->init_deferred_cmd_list);
242                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
243                         goto out;
244                 case SCST_SESS_IPH_FAILED:
245                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
246                         scst_set_busy(cmd);
247                         scst_set_cmd_abnormal_done_state(cmd);
248                         goto active;
249                 default:
250                         sBUG();
251                 }
252         } else
253                 list_add_tail(&cmd->search_cmd_list_entry,
254                               &sess->search_cmd_list);
255
256         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
257
258         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
259                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
260                 scst_set_cmd_error(cmd,
261                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
262                 scst_set_cmd_abnormal_done_state(cmd);
263                 goto active;
264         }
265
266         if (unlikely(cmd->cdb_len == 0)) {
267                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
268                 scst_set_cmd_error(cmd,
269                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
270                 scst_set_cmd_abnormal_done_state(cmd);
271                 goto active;
272         }
273
274         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
275                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
276                 scst_set_cmd_error(cmd,
277                         SCST_LOAD_SENSE(scst_sense_invalid_message));
278                 scst_set_cmd_abnormal_done_state(cmd);
279                 goto active;
280         }
281
282         cmd->state = SCST_CMD_STATE_INIT;
283         /* cmd must be inited here to preserve the order */
284         rc = scst_init_cmd(cmd, &pref_context);
285         if (unlikely(rc < 0))
286                 goto out;
287
288 active:
289         /* Here cmd must not be in any cmd list, no locks */
290         switch (pref_context) {
291         case SCST_CONTEXT_TASKLET:
292                 scst_schedule_tasklet(cmd);
293                 break;
294
295         case SCST_CONTEXT_DIRECT:
296                 scst_process_active_cmd(cmd, false);
297                 /* For *NEED_THREAD wake_up() is already done */
298                 break;
299
300         case SCST_CONTEXT_DIRECT_ATOMIC:
301                 scst_process_active_cmd(cmd, true);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         default:
306                 PRINT_ERROR("Context %x is undefined, using the thread one",
307                         pref_context);
308                 /* go through */
309         case SCST_CONTEXT_THREAD:
310                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
311                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
312                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
313                         list_add(&cmd->cmd_list_entry,
314                                 &cmd->cmd_lists->active_cmd_list);
315                 else
316                         list_add_tail(&cmd->cmd_list_entry,
317                                 &cmd->cmd_lists->active_cmd_list);
318                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
319                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
320                 break;
321         }
322
323 out:
324         TRACE_EXIT();
325         return;
326 }
327 EXPORT_SYMBOL(scst_cmd_init_done);
328
329 static int scst_pre_parse(struct scst_cmd *cmd)
330 {
331         int res = SCST_CMD_STATE_RES_CONT_SAME;
332         struct scst_device *dev = cmd->dev;
333         int rc;
334
335         TRACE_ENTRY();
336
337         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
338              (!dev->has_own_order_mgmt &&
339               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
340                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
341
342         /*
343          * Expected transfer data supplied by the SCSI transport via the
344          * target driver are untrusted, so we prefer to fetch them from CDB.
345          * Additionally, not all transports support supplying the expected
346          * transfer data.
347          */
348
349         rc = scst_get_cdb_info(cmd);
350         if (unlikely(rc != 0)) {
351                 if (rc > 0) {
352                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
353                         goto out_xmit;
354                 }
355                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
356                         "Should you update scst_scsi_op_table?",
357                         cmd->cdb[0], dev->handler->name);
358                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
359 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
360                 if (scst_cmd_is_expected_set(cmd)) {
361                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
362                                 "direction %d, transfer_len %d",
363                                 cmd->expected_data_direction,
364                                 cmd->expected_transfer_len);
365                         cmd->data_direction = cmd->expected_data_direction;
366
367                         cmd->bufflen = cmd->expected_transfer_len;
368                         /* Restore (likely) lost CDB length */
369                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
370                         if (cmd->cdb_len == -1) {
371                                 PRINT_ERROR("Unable to get CDB length for "
372                                         "opcode 0x%02x. Returning INVALID "
373                                         "OPCODE", cmd->cdb[0]);
374                                 scst_set_cmd_error(cmd,
375                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
376                                 goto out_xmit;
377                         }
378                 } else {
379                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
380                              "target %s not supplied expected values",
381                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
382                         scst_set_cmd_error(cmd,
383                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
384                         goto out_xmit;
385                 }
386 #else
387                 scst_set_cmd_error(cmd,
388                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                 goto out_xmit;
390 #endif
391         } else {
392                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
393                         "(expected %d, set %s), transfer_len=%d (expected "
394                         "len %d), flags=%d", cmd->op_name, cmd,
395                         cmd->data_direction, cmd->expected_data_direction,
396                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
397                         cmd->bufflen, cmd->expected_transfer_len,
398                         cmd->op_flags);
399
400                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
401                         if (scst_cmd_is_expected_set(cmd)) {
402                                 /*
403                                  * Command data length can't be easily
404                                  * determined from the CDB. ToDo, all such
405                                  * commands processing should be fixed. Until
406                                  * it's done, get the length from the supplied
407                                  * expected value, but limit it to some
408                                  * reasonable value (15MB).
409                                  */
410                                 cmd->bufflen = min(cmd->expected_transfer_len,
411                                                         15*1024*1024);
412                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
413                         } else
414                                 cmd->bufflen = 0;
415                 }
416         }
417
418         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
419                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
420                             "(opcode 0x%02x)", cmd->cdb[0]);
421                 scst_set_cmd_error(cmd,
422                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
423                 goto out_xmit;
424         }
425
426         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
427                 PRINT_ERROR("Linked commands are not supported "
428                             "(opcode 0x%02x)", cmd->cdb[0]);
429                 scst_set_cmd_error(cmd,
430                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
431                 goto out_xmit;
432         }
433
434         cmd->state = SCST_CMD_STATE_DEV_PARSE;
435
436 out:
437         TRACE_EXIT_RES(res);
438         return res;
439
440 out_xmit:
441         scst_set_cmd_abnormal_done_state(cmd);
442         res = SCST_CMD_STATE_RES_CONT_SAME;
443         goto out;
444 }
445
446 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
447 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
448 {
449         bool res = false;
450
451         switch (cmd->cdb[0]) {
452         case TEST_UNIT_READY:
453                 /* Crazy VMware people sometimes do TUR with READ direction */
454                 res = true;
455                 break;
456         case VERIFY:
457         case VERIFY_6:
458         case VERIFY_12:
459         case VERIFY_16:
460                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
461                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
462                     (cmd->cdb[1] & BYTCHK) == 0)
463                         res = true;
464                 break;
465         }
466
467         return res;
468 }
469 #endif
470
471 static int scst_parse_cmd(struct scst_cmd *cmd)
472 {
473         int res = SCST_CMD_STATE_RES_CONT_SAME;
474         int state;
475         struct scst_device *dev = cmd->dev;
476         int orig_bufflen = cmd->bufflen;
477
478         TRACE_ENTRY();
479
480         if (likely(!scst_is_cmd_local(cmd))) {
481                 if (unlikely(!dev->handler->parse_atomic &&
482                              scst_cmd_atomic(cmd))) {
483                         /*
484                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
485                          * optimization.
486                          */
487                         TRACE_DBG("Dev handler %s parse() needs thread "
488                                 "context, rescheduling", dev->handler->name);
489                         res = SCST_CMD_STATE_RES_NEED_THREAD;
490                         goto out;
491                 }
492
493                 TRACE_DBG("Calling dev handler %s parse(%p)",
494                       dev->handler->name, cmd);
495                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
496                                 cmd->cdb, cmd->cdb_len);
497                 state = dev->handler->parse(cmd);
498                 /* Caution: cmd can be already dead here */
499                 TRACE_DBG("Dev handler %s parse() returned %d",
500                         dev->handler->name, state);
501
502                 switch (state) {
503                 case SCST_CMD_STATE_NEED_THREAD_CTX:
504                         TRACE_DBG("Dev handler %s parse() requested thread "
505                               "context, rescheduling", dev->handler->name);
506                         res = SCST_CMD_STATE_RES_NEED_THREAD;
507                         goto out;
508
509                 case SCST_CMD_STATE_STOP:
510                         TRACE_DBG("Dev handler %s parse() requested stop "
511                                 "processing", dev->handler->name);
512                         res = SCST_CMD_STATE_RES_CONT_NEXT;
513                         goto out;
514                 }
515
516                 if (state == SCST_CMD_STATE_DEFAULT)
517                         state = SCST_CMD_STATE_PREPARE_SPACE;
518         } else
519                 state = SCST_CMD_STATE_PREPARE_SPACE;
520
521         if (cmd->data_len == -1)
522                 cmd->data_len = cmd->bufflen;
523
524         if (cmd->dh_data_buf_alloced &&
525             unlikely((orig_bufflen > cmd->bufflen))) {
526                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
527                         "is less, than required (size %d)", cmd->bufflen,
528                         orig_bufflen);
529                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
530                 goto out_error;
531         }
532
533         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
534                 goto set_res;
535
536         if (unlikely((cmd->bufflen == 0) &&
537                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
538                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
539                         "(handler %s, target %s)", cmd->cdb[0],
540                         dev->handler->name, cmd->tgtt->name);
541                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
542                 goto out_error;
543         }
544
545 #ifdef CONFIG_SCST_EXTRACHECKS
546         if ((cmd->bufflen != 0) &&
547             ((cmd->data_direction == SCST_DATA_NONE) ||
548              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
549                 PRINT_ERROR("Dev handler %s parse() returned "
550                         "invalid cmd data_direction %d, bufflen %d, state %d "
551                         "or sg %p (opcode 0x%x)", dev->handler->name,
552                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
553                         cmd->cdb[0]);
554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
555                 goto out_error;
556         }
557 #endif
558
559         if (scst_cmd_is_expected_set(cmd)) {
560 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
561 #       ifdef CONFIG_SCST_EXTRACHECKS
562                 if ((cmd->data_direction != cmd->expected_data_direction) ||
563                     (cmd->bufflen != cmd->expected_transfer_len)) {
564                         PRINT_WARNING("Expected values don't match decoded "
565                                 "ones: data_direction %d, "
566                                 "expected_data_direction %d, "
567                                 "bufflen %d, expected_transfer_len %d",
568                                 cmd->data_direction,
569                                 cmd->expected_data_direction,
570                                 cmd->bufflen, cmd->expected_transfer_len);
571                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
572                 }
573 #       endif
574                 cmd->data_direction = cmd->expected_data_direction;
575                 cmd->bufflen = cmd->expected_transfer_len;
576 #else
577                 if (unlikely(cmd->data_direction !=
578                                 cmd->expected_data_direction)) {
579                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
580                              (cmd->bufflen != 0)) &&
581                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
582                                 PRINT_ERROR("Expected data direction %d for "
583                                         "opcode 0x%02x (handler %s, target %s) "
584                                         "doesn't match "
585                                         "decoded value %d",
586                                         cmd->expected_data_direction,
587                                         cmd->cdb[0], dev->handler->name,
588                                         cmd->tgtt->name, cmd->data_direction);
589                                 PRINT_BUFFER("Failed CDB",
590                                         cmd->cdb, cmd->cdb_len);
591                                 scst_set_cmd_error(cmd,
592                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
593                                 goto out_dev_done;
594                         }
595                 }
596                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
597                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
598                                 "transfer length %d for opcode 0x%02x "
599                                 "(handler %s, target %s) doesn't match "
600                                 "decoded value %d. Faulty initiator "
601                                 "(e.g. VMware is known to be such) or "
602                                 "scst_scsi_op_table should be updated?",
603                                 cmd->expected_transfer_len, cmd->cdb[0],
604                                 dev->handler->name, cmd->tgtt->name,
605                                 cmd->bufflen);
606                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
607                                 cmd->cdb, cmd->cdb_len);
608                 }
609 #endif
610         }
611
612         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
613                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
614                         "target %s", cmd->cdb[0], dev->handler->name,
615                         cmd->tgtt->name);
616                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
617                 goto out_error;
618         }
619
620 set_res:
621         switch (state) {
622         case SCST_CMD_STATE_PREPARE_SPACE:
623         case SCST_CMD_STATE_PRE_PARSE:
624         case SCST_CMD_STATE_DEV_PARSE:
625         case SCST_CMD_STATE_RDY_TO_XFER:
626         case SCST_CMD_STATE_TGT_PRE_EXEC:
627         case SCST_CMD_STATE_SEND_FOR_EXEC:
628         case SCST_CMD_STATE_LOCAL_EXEC:
629         case SCST_CMD_STATE_REAL_EXEC:
630         case SCST_CMD_STATE_PRE_DEV_DONE:
631         case SCST_CMD_STATE_DEV_DONE:
632         case SCST_CMD_STATE_PRE_XMIT_RESP:
633         case SCST_CMD_STATE_XMIT_RESP:
634         case SCST_CMD_STATE_FINISHED:
635                 cmd->state = state;
636                 res = SCST_CMD_STATE_RES_CONT_SAME;
637                 break;
638
639         default:
640                 if (state >= 0) {
641                         PRINT_ERROR("Dev handler %s parse() returned "
642                              "invalid cmd state %d (opcode %d)",
643                              dev->handler->name, state, cmd->cdb[0]);
644                 } else {
645                         PRINT_ERROR("Dev handler %s parse() returned "
646                                 "error %d (opcode %d)", dev->handler->name,
647                                 state, cmd->cdb[0]);
648                 }
649                 goto out_error;
650         }
651
652         if (cmd->resp_data_len == -1) {
653                 if (cmd->data_direction == SCST_DATA_READ)
654                         cmd->resp_data_len = cmd->bufflen;
655                 else
656                          cmd->resp_data_len = 0;
657         }
658
659 out:
660         TRACE_EXIT_HRES(res);
661         return res;
662
663 out_error:
664         /* dev_done() will be called as part of the regular cmd's finish */
665         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
666
667 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
668 out_dev_done:
669 #endif
670         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
671         res = SCST_CMD_STATE_RES_CONT_SAME;
672         goto out;
673 }
674
675 static int scst_prepare_space(struct scst_cmd *cmd)
676 {
677         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
678
679         TRACE_ENTRY();
680
681         if (cmd->data_direction == SCST_DATA_NONE)
682                 goto prep_done;
683
684         if (cmd->tgt_need_alloc_data_buf) {
685                 int orig_bufflen = cmd->bufflen;
686
687                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
688                         cmd);
689
690                 r = cmd->tgtt->alloc_data_buf(cmd);
691                 if (r > 0)
692                         goto alloc;
693                 else if (r == 0) {
694                         if (unlikely(cmd->bufflen == 0)) {
695                                 /* See comment in scst_alloc_space() */
696                                 if (cmd->sg == NULL)
697                                         goto alloc;
698                         }
699
700                         cmd->tgt_data_buf_alloced = 1;
701
702                         if (unlikely(orig_bufflen < cmd->bufflen)) {
703                                 PRINT_ERROR("Target driver allocated data "
704                                         "buffer (size %d), is less, than "
705                                         "required (size %d)", orig_bufflen,
706                                         cmd->bufflen);
707                                 goto out_error;
708                         }
709                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
710                 } else
711                         goto check;
712         }
713
714 alloc:
715         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
716                 r = scst_alloc_space(cmd);
717                 cmd->tgt_sg = cmd->sg;
718                 cmd->tgt_sg_cnt = cmd->sg_cnt;
719         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
720                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
721                 cmd->tgt_sg = cmd->sg;
722                 cmd->tgt_sg_cnt = cmd->sg_cnt;
723                 r = 0;
724         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
725                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
726                 cmd->sg = cmd->tgt_sg;
727                 cmd->sg_cnt = cmd->tgt_sg_cnt;
728                 r = 0;
729         } else {
730                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
731                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
732                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
733                 r = 0;
734         }
735
736 check:
737         if (r != 0) {
738                 if (scst_cmd_atomic(cmd)) {
739                         TRACE_MEM("%s", "Atomic memory allocation failed, "
740                               "rescheduling to the thread");
741                         res = SCST_CMD_STATE_RES_NEED_THREAD;
742                         goto out;
743                 } else
744                         goto out_no_space;
745         }
746
747 prep_done:
748         if (cmd->preprocessing_only) {
749                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
750                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
751                                 "cmd %p", cmd);
752                         scst_set_cmd_abnormal_done_state(cmd);
753                         res = SCST_CMD_STATE_RES_CONT_SAME;
754                         goto out;
755                 }
756
757                 res = SCST_CMD_STATE_RES_CONT_NEXT;
758                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
759
760                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
761                 cmd->tgtt->preprocessing_done(cmd);
762                 TRACE_DBG("%s", "preprocessing_done() returned");
763                 goto out;
764
765         }
766
767         switch (cmd->data_direction) {
768         case SCST_DATA_WRITE:
769                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
770                 break;
771
772         default:
773                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
774                 break;
775         }
776
777 out:
778         TRACE_EXIT_HRES(res);
779         return res;
780
781 out_no_space:
782         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
783                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
784         scst_set_busy(cmd);
785         scst_set_cmd_abnormal_done_state(cmd);
786         res = SCST_CMD_STATE_RES_CONT_SAME;
787         goto out;
788
789 out_error:
790         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
791         scst_set_cmd_abnormal_done_state(cmd);
792         res = SCST_CMD_STATE_RES_CONT_SAME;
793         goto out;
794 }
795
796 void scst_restart_cmd(struct scst_cmd *cmd, int status,
797         enum scst_exec_context pref_context)
798 {
799         TRACE_ENTRY();
800
801         TRACE_DBG("Preferred context: %d", pref_context);
802         TRACE_DBG("tag=%llu, status=%#x",
803                   (long long unsigned int)scst_cmd_get_tag(cmd),
804                   status);
805
806 #ifdef CONFIG_SCST_EXTRACHECKS
807         if ((in_irq() || irqs_disabled()) &&
808             ((pref_context == SCST_CONTEXT_DIRECT) ||
809              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
810                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
811                         "SCST_CONTEXT_THREAD instead\n", pref_context,
812                         cmd->tgtt->name);
813                 pref_context = SCST_CONTEXT_THREAD;
814         }
815 #endif
816
817         switch (status) {
818         case SCST_PREPROCESS_STATUS_SUCCESS:
819                 switch (cmd->data_direction) {
820                 case SCST_DATA_WRITE:
821                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
822                         break;
823                 default:
824                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
825                         break;
826                 }
827                 if (cmd->set_sn_on_restart_cmd)
828                         scst_cmd_set_sn(cmd);
829                 /* Small context optimization */
830                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
831                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
832                     ((pref_context == SCST_CONTEXT_SAME) &&
833                      scst_cmd_atomic(cmd))) {
834                         if (cmd->data_direction == SCST_DATA_WRITE) {
835                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
836                                                 &cmd->tgt_dev->tgt_dev_flags))
837                                         pref_context = SCST_CONTEXT_THREAD;
838                         } else {
839                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
840                                                 &cmd->tgt_dev->tgt_dev_flags))
841                                         pref_context = SCST_CONTEXT_THREAD;
842                         }
843                 }
844                 break;
845
846         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
847                 scst_set_cmd_abnormal_done_state(cmd);
848                 break;
849
850         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
851                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
852                 /* go through */
853         case SCST_PREPROCESS_STATUS_ERROR:
854                 scst_set_cmd_error(cmd,
855                            SCST_LOAD_SENSE(scst_sense_hardw_error));
856                 scst_set_cmd_abnormal_done_state(cmd);
857                 break;
858
859         default:
860                 PRINT_ERROR("%s() received unknown status %x", __func__,
861                         status);
862                 scst_set_cmd_abnormal_done_state(cmd);
863                 break;
864         }
865
866         scst_proccess_redirect_cmd(cmd, pref_context, 1);
867
868         TRACE_EXIT();
869         return;
870 }
871 EXPORT_SYMBOL(scst_restart_cmd);
872
873 /* No locks */
874 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
875 {
876         struct scst_tgt *tgt = cmd->sess->tgt;
877         int res = 0;
878         unsigned long flags;
879
880         TRACE_ENTRY();
881
882         spin_lock_irqsave(&tgt->tgt_lock, flags);
883         tgt->retry_cmds++;
884         smp_mb();
885         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
886               tgt->retry_cmds);
887         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
888                 /* At least one cmd finished, so try again */
889                 tgt->retry_cmds--;
890                 TRACE_RETRY("Some command(s) finished, direct retry "
891                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
892                       "retry_cmds=%d)", finished_cmds,
893                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
894                 res = -1;
895                 goto out_unlock_tgt;
896         }
897
898         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
899         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
900
901         if (!tgt->retry_timer_active) {
902                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
903                 add_timer(&tgt->retry_timer);
904                 tgt->retry_timer_active = 1;
905         }
906
907 out_unlock_tgt:
908         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
909
910         TRACE_EXIT_RES(res);
911         return res;
912 }
913
914 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
915 {
916         int res, rc;
917
918         TRACE_ENTRY();
919
920         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
921                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
922                 goto out_dev_done;
923         }
924
925         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
926                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
927                 res = SCST_CMD_STATE_RES_CONT_SAME;
928                 goto out;
929         }
930
931         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
932                 /*
933                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
934                  * optimization.
935                  */
936                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
937                               "context, rescheduling", cmd->tgtt->name);
938                 res = SCST_CMD_STATE_RES_NEED_THREAD;
939                 goto out;
940         }
941
942         while (1) {
943                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
944
945                 res = SCST_CMD_STATE_RES_CONT_NEXT;
946                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
947
948                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
949 #ifdef CONFIG_SCST_DEBUG_RETRY
950                 if (((scst_random() % 100) == 75))
951                         rc = SCST_TGT_RES_QUEUE_FULL;
952                 else
953 #endif
954                         rc = cmd->tgtt->rdy_to_xfer(cmd);
955                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
956
957                 if (likely(rc == SCST_TGT_RES_SUCCESS))
958                         goto out;
959
960                 /* Restore the previous state */
961                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
962
963                 switch (rc) {
964                 case SCST_TGT_RES_QUEUE_FULL:
965                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
966                                 break;
967                         else
968                                 continue;
969
970                 case SCST_TGT_RES_NEED_THREAD_CTX:
971                         TRACE_DBG("Target driver %s "
972                               "rdy_to_xfer() requested thread "
973                               "context, rescheduling", cmd->tgtt->name);
974                         res = SCST_CMD_STATE_RES_NEED_THREAD;
975                         break;
976
977                 default:
978                         goto out_error_rc;
979                 }
980                 break;
981         }
982
983 out:
984         TRACE_EXIT_HRES(res);
985         return res;
986
987 out_error_rc:
988         if (rc == SCST_TGT_RES_FATAL_ERROR) {
989                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
990                      "fatal error", cmd->tgtt->name);
991         } else {
992                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
993                             "value %d", cmd->tgtt->name, rc);
994         }
995         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
996
997 out_dev_done:
998         scst_set_cmd_abnormal_done_state(cmd);
999         res = SCST_CMD_STATE_RES_CONT_SAME;
1000         goto out;
1001 }
1002
1003 /* No locks, but might be in IRQ */
1004 void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1005         enum scst_exec_context context, int check_retries)
1006 {
1007         unsigned long flags;
1008
1009         TRACE_ENTRY();
1010
1011         TRACE_DBG("Context: %x", context);
1012
1013         if (context == SCST_CONTEXT_SAME)
1014                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1015                                                  SCST_CONTEXT_DIRECT;
1016
1017         switch (context) {
1018         case SCST_CONTEXT_DIRECT_ATOMIC:
1019                 scst_process_active_cmd(cmd, true);
1020                 break;
1021
1022         case SCST_CONTEXT_DIRECT:
1023                 if (check_retries)
1024                         scst_check_retries(cmd->tgt);
1025                 scst_process_active_cmd(cmd, false);
1026                 break;
1027
1028         default:
1029                 PRINT_ERROR("Context %x is unknown, using the thread one",
1030                             context);
1031                 /* go through */
1032         case SCST_CONTEXT_THREAD:
1033                 if (check_retries)
1034                         scst_check_retries(cmd->tgt);
1035                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1036                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1037                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1038                         list_add(&cmd->cmd_list_entry,
1039                                 &cmd->cmd_lists->active_cmd_list);
1040                 else
1041                         list_add_tail(&cmd->cmd_list_entry,
1042                                 &cmd->cmd_lists->active_cmd_list);
1043                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1044                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1045                 break;
1046
1047         case SCST_CONTEXT_TASKLET:
1048                 if (check_retries)
1049                         scst_check_retries(cmd->tgt);
1050                 scst_schedule_tasklet(cmd);
1051                 break;
1052         }
1053
1054         TRACE_EXIT();
1055         return;
1056 }
1057
1058 void scst_rx_data(struct scst_cmd *cmd, int status,
1059         enum scst_exec_context pref_context)
1060 {
1061         TRACE_ENTRY();
1062
1063         TRACE_DBG("Preferred context: %d", pref_context);
1064         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1065
1066 #ifdef CONFIG_SCST_EXTRACHECKS
1067         if ((in_irq() || irqs_disabled()) &&
1068             ((pref_context == SCST_CONTEXT_DIRECT) ||
1069              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1070                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1071                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1072                         cmd->tgtt->name);
1073                 pref_context = SCST_CONTEXT_THREAD;
1074         }
1075 #endif
1076
1077         switch (status) {
1078         case SCST_RX_STATUS_SUCCESS:
1079 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1080                 if (cmd->tgt_sg) {
1081                         int i;
1082                         struct scatterlist *sg = cmd->tgt_sg;
1083                         TRACE_RECV_BOT("RX data for cmd %p "
1084                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1085                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1086                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1087                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1088                                         sg_virt(&sg[i]), sg[i].length);
1089                         }
1090                 }
1091 #endif
1092                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1093                 /* Small context optimization */
1094                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1095                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1096                     ((pref_context == SCST_CONTEXT_SAME) &&
1097                      scst_cmd_atomic(cmd))) {
1098                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1099                                         &cmd->tgt_dev->tgt_dev_flags))
1100                                 pref_context = SCST_CONTEXT_THREAD;
1101                 }
1102                 break;
1103
1104         case SCST_RX_STATUS_ERROR_SENSE_SET:
1105                 scst_set_cmd_abnormal_done_state(cmd);
1106                 break;
1107
1108         case SCST_RX_STATUS_ERROR_FATAL:
1109                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1110                 /* go through */
1111         case SCST_RX_STATUS_ERROR:
1112                 scst_set_cmd_error(cmd,
1113                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1114                 scst_set_cmd_abnormal_done_state(cmd);
1115                 break;
1116
1117         default:
1118                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1119                         status);
1120                 scst_set_cmd_abnormal_done_state(cmd);
1121                 break;
1122         }
1123
1124         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1125
1126         TRACE_EXIT();
1127         return;
1128 }
1129 EXPORT_SYMBOL(scst_rx_data);
1130
1131 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1132 {
1133         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1134
1135         TRACE_ENTRY();
1136
1137         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1138
1139         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1140                 goto out;
1141
1142         TRACE_DBG("Calling pre_exec(%p)", cmd);
1143         rc = cmd->tgtt->pre_exec(cmd);
1144         TRACE_DBG("pre_exec() returned %d", rc);
1145
1146         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1147                 switch (rc) {
1148                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1149                         scst_set_cmd_abnormal_done_state(cmd);
1150                         break;
1151                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1152                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1153                         /* go through */
1154                 case SCST_PREPROCESS_STATUS_ERROR:
1155                         scst_set_cmd_error(cmd,
1156                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1157                         scst_set_cmd_abnormal_done_state(cmd);
1158                         break;
1159                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1160                         TRACE_DBG("Target driver's %s pre_exec() requested "
1161                                 "thread context, rescheduling",
1162                                 cmd->tgtt->name);
1163                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1164                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1165                         break;
1166                 default:
1167                         sBUG();
1168                         break;
1169                 }
1170         }
1171
1172 out:
1173         TRACE_EXIT_RES(res);
1174         return res;
1175 }
1176
1177 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1178         const uint8_t *rq_sense, int rq_sense_len, int resid)
1179 {
1180         TRACE_ENTRY();
1181
1182 #ifdef CONFIG_SCST_MEASURE_LATENCY
1183         {
1184                 struct timespec ts;
1185                 getnstimeofday(&ts);
1186                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1187                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1188                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1189                         ts.tv_sec, ts.tv_nsec);
1190         }
1191 #endif
1192
1193         cmd->status = result & 0xff;
1194         cmd->msg_status = msg_byte(result);
1195         cmd->host_status = host_byte(result);
1196         cmd->driver_status = driver_byte(result);
1197         if (unlikely(resid != 0)) {
1198 #ifdef CONFIG_SCST_EXTRACHECKS
1199                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1200                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1201                                 "op %x)", resid, cmd->resp_data_len,
1202                                 cmd->cdb[0]);
1203                 } else
1204 #endif
1205                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1206         }
1207
1208         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1209                 /* We might have double reset UA here */
1210                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1211                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1212
1213                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1214         }
1215
1216         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1217               "cmd->msg_status=%x, cmd->host_status=%x, "
1218               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1219               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1220
1221         cmd->completed = 1;
1222
1223         TRACE_EXIT();
1224         return;
1225 }
1226
1227 /* For small context optimization */
1228 static inline enum scst_exec_context scst_optimize_post_exec_context(
1229         struct scst_cmd *cmd, enum scst_exec_context context)
1230 {
1231         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1232             (context == SCST_CONTEXT_TASKLET) ||
1233             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1234                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1235                                 &cmd->tgt_dev->tgt_dev_flags))
1236                         context = SCST_CONTEXT_THREAD;
1237         }
1238         return context;
1239 }
1240
1241 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1242 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1243                                             struct scsi_request **req)
1244 {
1245         struct scst_cmd *cmd = NULL;
1246
1247         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1248                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1249
1250         if (cmd == NULL) {
1251                 PRINT_ERROR("%s", "Request with NULL cmd");
1252                 if (*req)
1253                         scsi_release_request(*req);
1254         }
1255
1256         return cmd;
1257 }
1258
1259 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1260 {
1261         struct scsi_request *req = NULL;
1262         struct scst_cmd *cmd;
1263         enum scst_exec_context context;
1264
1265         TRACE_ENTRY();
1266
1267         cmd = scst_get_cmd(scsi_cmd, &req);
1268         if (cmd == NULL)
1269                 goto out;
1270
1271         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1272                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1273
1274         /* Clear out request structure */
1275         req->sr_use_sg = 0;
1276         req->sr_sglist_len = 0;
1277         req->sr_bufflen = 0;
1278         req->sr_buffer = NULL;
1279         req->sr_underflow = 0;
1280         req->sr_request->rq_disk = NULL; /* disown request blk */
1281
1282         scst_release_request(cmd);
1283
1284         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1285
1286         scst_proccess_redirect_cmd(cmd,
1287                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1288                                                 0);
1289
1290 out:
1291         TRACE_EXIT();
1292         return;
1293 }
1294 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1295 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1296 {
1297         struct scst_cmd *cmd;
1298
1299         TRACE_ENTRY();
1300
1301         cmd = (struct scst_cmd *)data;
1302         if (cmd == NULL)
1303                 goto out;
1304
1305         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1306
1307         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1308
1309         scst_proccess_redirect_cmd(cmd,
1310                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1311                                                 0);
1312
1313 out:
1314         TRACE_EXIT();
1315         return;
1316 }
1317 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1318
1319 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1320         enum scst_exec_context pref_context)
1321 {
1322         TRACE_ENTRY();
1323
1324 #ifdef CONFIG_SCST_MEASURE_LATENCY
1325         {
1326                 struct timespec ts;
1327                 getnstimeofday(&ts);
1328                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1329                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1330                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1331                         ts.tv_sec, ts.tv_nsec);
1332         }
1333 #endif
1334
1335         if (next_state == SCST_CMD_STATE_DEFAULT)
1336                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1337
1338 #if defined(CONFIG_SCST_DEBUG)
1339         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1340                 if (cmd->sg) {
1341                         int i;
1342                         struct scatterlist *sg = cmd->sg;
1343                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1344                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1345                         for (i = 0; i < cmd->sg_cnt; ++i) {
1346                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1347                                         "Exec'd sg", sg_virt(&sg[i]),
1348                                         sg[i].length);
1349                         }
1350                 }
1351         }
1352 #endif
1353
1354         cmd->state = next_state;
1355
1356 #ifdef CONFIG_SCST_EXTRACHECKS
1357         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1358             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1359             (next_state != SCST_CMD_STATE_FINISHED)) {
1360                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1361                         __func__, next_state, cmd->cdb[0]);
1362                 scst_set_cmd_error(cmd,
1363                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1364                 scst_set_cmd_abnormal_done_state(cmd);
1365         }
1366 #endif
1367         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1368         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1369
1370         TRACE_EXIT();
1371         return;
1372 }
1373
1374 static int scst_report_luns_local(struct scst_cmd *cmd)
1375 {
1376         int rc;
1377         int dev_cnt = 0;
1378         int buffer_size;
1379         int i;
1380         struct scst_tgt_dev *tgt_dev = NULL;
1381         uint8_t *buffer;
1382         int offs, overflow = 0;
1383
1384         TRACE_ENTRY();
1385
1386         rc = scst_check_local_events(cmd);
1387         if (unlikely(rc != 0))
1388                 goto out_done;
1389
1390         cmd->status = 0;
1391         cmd->msg_status = 0;
1392         cmd->host_status = DID_OK;
1393         cmd->driver_status = 0;
1394
1395         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1396                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1397                         "LUNS command", cmd->cdb[2]);
1398                 goto out_err;
1399         }
1400
1401         buffer_size = scst_get_buf_first(cmd, &buffer);
1402         if (unlikely(buffer_size == 0))
1403                 goto out_compl;
1404         else if (unlikely(buffer_size < 0))
1405                 goto out_err;
1406
1407         if (buffer_size < 16)
1408                 goto out_put_err;
1409
1410         memset(buffer, 0, buffer_size);
1411         offs = 8;
1412
1413         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1414         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1415                 struct list_head *sess_tgt_dev_list_head =
1416                         &cmd->sess->sess_tgt_dev_list_hash[i];
1417                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1418                                 sess_tgt_dev_list_entry) {
1419                         if (!overflow) {
1420                                 if (offs >= buffer_size) {
1421                                         scst_put_buf(cmd, buffer);
1422                                         buffer_size = scst_get_buf_next(cmd,
1423                                                                        &buffer);
1424                                         if (buffer_size > 0) {
1425                                                 memset(buffer, 0, buffer_size);
1426                                                 offs = 0;
1427                                         } else {
1428                                                 overflow = 1;
1429                                                 goto inc_dev_cnt;
1430                                         }
1431                                 }
1432                                 if ((buffer_size - offs) < 8) {
1433                                         PRINT_ERROR("Buffer allocated for "
1434                                                 "REPORT LUNS command doesn't "
1435                                                 "allow to fit 8 byte entry "
1436                                                 "(buffer_size=%d)",
1437                                                 buffer_size);
1438                                         goto out_put_hw_err;
1439                                 }
1440                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1441                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1442                                 offs += 8;
1443                         }
1444 inc_dev_cnt:
1445                         dev_cnt++;
1446                 }
1447         }
1448         if (!overflow)
1449                 scst_put_buf(cmd, buffer);
1450
1451         /* Set the response header */
1452         buffer_size = scst_get_buf_first(cmd, &buffer);
1453         if (unlikely(buffer_size == 0))
1454                 goto out_compl;
1455         else if (unlikely(buffer_size < 0))
1456                 goto out_err;
1457
1458         dev_cnt *= 8;
1459         buffer[0] = (dev_cnt >> 24) & 0xff;
1460         buffer[1] = (dev_cnt >> 16) & 0xff;
1461         buffer[2] = (dev_cnt >> 8) & 0xff;
1462         buffer[3] = dev_cnt & 0xff;
1463
1464         scst_put_buf(cmd, buffer);
1465
1466         dev_cnt += 8;
1467         if (dev_cnt < cmd->resp_data_len)
1468                 scst_set_resp_data_len(cmd, dev_cnt);
1469
1470 out_compl:
1471         cmd->completed = 1;
1472
1473 out_done:
1474         /* Report the result */
1475         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1476
1477         TRACE_EXIT();
1478         return SCST_EXEC_COMPLETED;
1479
1480 out_put_err:
1481         scst_put_buf(cmd, buffer);
1482
1483 out_err:
1484         scst_set_cmd_error(cmd,
1485                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1486         goto out_compl;
1487
1488 out_put_hw_err:
1489         scst_put_buf(cmd, buffer);
1490         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1491         goto out_compl;
1492 }
1493
1494 static int scst_pre_select(struct scst_cmd *cmd)
1495 {
1496         int res = SCST_EXEC_NOT_COMPLETED;
1497
1498         TRACE_ENTRY();
1499
1500         if (scst_cmd_atomic(cmd)) {
1501                 res = SCST_EXEC_NEED_THREAD;
1502                 goto out;
1503         }
1504
1505         scst_block_dev_cmd(cmd, 1);
1506
1507         /* Check for local events will be done when cmd will be executed */
1508
1509 out:
1510         TRACE_EXIT_RES(res);
1511         return res;
1512 }
1513
1514 static int scst_reserve_local(struct scst_cmd *cmd)
1515 {
1516         int res = SCST_EXEC_NOT_COMPLETED, rc;
1517         struct scst_device *dev;
1518         struct scst_tgt_dev *tgt_dev_tmp;
1519
1520         TRACE_ENTRY();
1521
1522         if (scst_cmd_atomic(cmd)) {
1523                 res = SCST_EXEC_NEED_THREAD;
1524                 goto out;
1525         }
1526
1527         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1528                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1529                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1530                 scst_set_cmd_error(cmd,
1531                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1532                 goto out_done;
1533         }
1534
1535         dev = cmd->dev;
1536
1537         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1538                 scst_block_dev_cmd(cmd, 1);
1539
1540         rc = scst_check_local_events(cmd);
1541         if (unlikely(rc != 0))
1542                 goto out_done;
1543
1544         spin_lock_bh(&dev->dev_lock);
1545
1546         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1547                 spin_unlock_bh(&dev->dev_lock);
1548                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1549                 goto out_done;
1550         }
1551
1552         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1553                             dev_tgt_dev_list_entry) {
1554                 if (cmd->tgt_dev != tgt_dev_tmp)
1555                         set_bit(SCST_TGT_DEV_RESERVED,
1556                                 &tgt_dev_tmp->tgt_dev_flags);
1557         }
1558         dev->dev_reserved = 1;
1559
1560         spin_unlock_bh(&dev->dev_lock);
1561
1562 out:
1563         TRACE_EXIT_RES(res);
1564         return res;
1565
1566 out_done:
1567         /* Report the result */
1568         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1569         res = SCST_EXEC_COMPLETED;
1570         goto out;
1571 }
1572
1573 static int scst_release_local(struct scst_cmd *cmd)
1574 {
1575         int res = SCST_EXEC_NOT_COMPLETED, rc;
1576         struct scst_tgt_dev *tgt_dev_tmp;
1577         struct scst_device *dev;
1578
1579         TRACE_ENTRY();
1580
1581         if (scst_cmd_atomic(cmd)) {
1582                 res = SCST_EXEC_NEED_THREAD;
1583                 goto out;
1584         }
1585
1586         dev = cmd->dev;
1587
1588         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1589                 scst_block_dev_cmd(cmd, 1);
1590
1591         rc = scst_check_local_events(cmd);
1592         if (unlikely(rc != 0))
1593                 goto out_done;
1594
1595         spin_lock_bh(&dev->dev_lock);
1596
1597         /*
1598          * The device could be RELEASED behind us, if RESERVING session
1599          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1600          * matter, so use lock and no retest for DEV_RESERVED bits again
1601          */
1602         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1603                 res = SCST_EXEC_COMPLETED;
1604                 cmd->status = 0;
1605                 cmd->msg_status = 0;
1606                 cmd->host_status = DID_OK;
1607                 cmd->driver_status = 0;
1608                 cmd->completed = 1;
1609         } else {
1610                 list_for_each_entry(tgt_dev_tmp,
1611                                     &dev->dev_tgt_dev_list,
1612                                     dev_tgt_dev_list_entry) {
1613                         clear_bit(SCST_TGT_DEV_RESERVED,
1614                                 &tgt_dev_tmp->tgt_dev_flags);
1615                 }
1616                 dev->dev_reserved = 0;
1617         }
1618
1619         spin_unlock_bh(&dev->dev_lock);
1620
1621         if (res == SCST_EXEC_COMPLETED)
1622                 goto out_done;
1623
1624 out:
1625         TRACE_EXIT_RES(res);
1626         return res;
1627
1628 out_done:
1629         res = SCST_EXEC_COMPLETED;
1630         /* Report the result */
1631         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1632         goto out;
1633 }
1634
1635 /* No locks, no IRQ or IRQ-safe context allowed */
1636 int scst_check_local_events(struct scst_cmd *cmd)
1637 {
1638         int res, rc;
1639         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1640         struct scst_device *dev = cmd->dev;
1641
1642         TRACE_ENTRY();
1643
1644         /*
1645          * There's no race here, because we need to trace commands sent
1646          * *after* dev_double_ua_possible flag was set.
1647          */
1648         if (unlikely(dev->dev_double_ua_possible))
1649                 cmd->double_ua_possible = 1;
1650
1651         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1652                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1653                 goto out_uncomplete;
1654         }
1655
1656         /* Reserve check before Unit Attention */
1657         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1658                               &tgt_dev->tgt_dev_flags))) {
1659                 if (cmd->cdb[0] != INQUIRY &&
1660                     cmd->cdb[0] != REPORT_LUNS &&
1661                     cmd->cdb[0] != RELEASE &&
1662                     cmd->cdb[0] != RELEASE_10 &&
1663                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1664                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1665                      (cmd->cdb[4] & 3)) &&
1666                     cmd->cdb[0] != LOG_SENSE &&
1667                     cmd->cdb[0] != REQUEST_SENSE) {
1668                         scst_set_cmd_error_status(cmd,
1669                                 SAM_STAT_RESERVATION_CONFLICT);
1670                         goto out_complete;
1671                 }
1672         }
1673
1674         /* If we had internal bus reset, set the command error unit attention */
1675         if ((dev->scsi_dev != NULL) &&
1676             unlikely(dev->scsi_dev->was_reset)) {
1677                 if (scst_is_ua_command(cmd)) {
1678                         int done = 0;
1679                         /*
1680                          * Prevent more than 1 cmd to be triggered by
1681                          * was_reset.
1682                          */
1683                         spin_lock_bh(&dev->dev_lock);
1684                         barrier(); /* to reread was_reset */
1685                         if (dev->scsi_dev->was_reset) {
1686                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1687                                 scst_set_cmd_error(cmd,
1688                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1689                                 /*
1690                                  * It looks like it is safe to clear was_reset
1691                                  * here.
1692                                  */
1693                                 dev->scsi_dev->was_reset = 0;
1694                                 done = 1;
1695                         }
1696                         spin_unlock_bh(&dev->dev_lock);
1697
1698                         if (done)
1699                                 goto out_complete;
1700                 }
1701         }
1702
1703         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1704                         &cmd->tgt_dev->tgt_dev_flags))) {
1705                 if (scst_is_ua_command(cmd)) {
1706                         rc = scst_set_pending_UA(cmd);
1707                         if (rc == 0)
1708                                 goto out_complete;
1709                 }
1710         }
1711
1712         res = 0;
1713
1714 out:
1715         TRACE_EXIT_RES(res);
1716         return res;
1717
1718 out_complete:
1719         res = 1;
1720         sBUG_ON(!cmd->completed);
1721         goto out;
1722
1723 out_uncomplete:
1724         res = -1;
1725         goto out;
1726 }
1727 EXPORT_SYMBOL(scst_check_local_events);
1728
1729 /* No locks */
1730 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1731 {
1732         if (slot == NULL)
1733                 goto inc;
1734
1735         /* Optimized for lockless fast path */
1736
1737         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1738                 atomic_read(slot));
1739
1740         if (!atomic_dec_and_test(slot))
1741                 goto out;
1742
1743         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1744                 tgt_dev->num_free_sn_slots);
1745         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1746                 spin_lock_irq(&tgt_dev->sn_lock);
1747                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1748                         if (tgt_dev->num_free_sn_slots < 0)
1749                                 tgt_dev->cur_sn_slot = slot;
1750                         /*
1751                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1752                          */
1753                         smp_mb();
1754                         tgt_dev->num_free_sn_slots++;
1755                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1756                                 tgt_dev->num_free_sn_slots);
1757
1758                 }
1759                 spin_unlock_irq(&tgt_dev->sn_lock);
1760         }
1761
1762 inc:
1763         /*
1764          * No locks is needed, because only one thread at time can
1765          * be here (serialized by sn). Also it is supposed that there
1766          * could not be half-incremented halves.
1767          */
1768         tgt_dev->expected_sn++;
1769         smp_mb(); /* write must be before def_cmd_count read */
1770         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1771
1772 out:
1773         return;
1774 }
1775
1776 /* No locks */
1777 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1778         bool make_active)
1779 {
1780         /* For HQ commands SN is not set */
1781         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1782                                cmd->sn_set && !cmd->retry;
1783         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1784         struct scst_cmd *res;
1785
1786         TRACE_ENTRY();
1787
1788         if (inc_expected_sn)
1789                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1790
1791         if (make_active) {
1792                 scst_make_deferred_commands_active(tgt_dev);
1793                 res = NULL;
1794         } else
1795                 res = scst_check_deferred_commands(tgt_dev);
1796
1797         TRACE_EXIT_HRES(res);
1798         return res;
1799 }
1800
1801 /* cmd must be additionally referenced to not die inside */
1802 static int scst_do_real_exec(struct scst_cmd *cmd)
1803 {
1804         int res = SCST_EXEC_NOT_COMPLETED, rc;
1805         struct scst_device *dev = cmd->dev;
1806         struct scst_dev_type *handler = dev->handler;
1807
1808         TRACE_ENTRY();
1809
1810         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1811
1812         if (handler->exec) {
1813                 if (unlikely(!dev->handler->exec_atomic &&
1814                              scst_cmd_atomic(cmd))) {
1815                         /*
1816                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1817                          * optimization.
1818                          */
1819                         TRACE_DBG("Dev handler %s exec() needs thread "
1820                                 "context, rescheduling", dev->handler->name);
1821                         res = SCST_EXEC_NEED_THREAD;
1822                         goto out_restore;
1823                 }
1824
1825                 TRACE_DBG("Calling dev handler %s exec(%p)",
1826                       handler->name, cmd);
1827                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1828                         cmd->cdb_len);
1829                 res = handler->exec(cmd);
1830                 TRACE_DBG("Dev handler %s exec() returned %d",
1831                       handler->name, res);
1832
1833                 if (res == SCST_EXEC_COMPLETED)
1834                         goto out_complete;
1835                 else if (res == SCST_EXEC_NEED_THREAD)
1836                         goto out_restore;
1837
1838                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1839         }
1840
1841         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1842
1843         if (unlikely(dev->scsi_dev == NULL)) {
1844                 PRINT_ERROR("Command for virtual device must be "
1845                         "processed by device handler (lun %lld)!",
1846                         (long long unsigned int)cmd->lun);
1847                 goto out_error;
1848         }
1849
1850         res = scst_check_local_events(cmd);
1851         if (unlikely(res != 0))
1852                 goto out_done;
1853
1854 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1855         if (unlikely(scst_cmd_atomic(cmd))) {
1856                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1857                         "context, rescheduling to the thread (handler %s)",
1858                         handler->name);
1859                 res = SCST_EXEC_NEED_THREAD;
1860                 goto out_restore;
1861         }
1862 #endif
1863
1864 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1865         if (unlikely(scst_alloc_request(cmd) != 0)) {
1866                 if (scst_cmd_atomic(cmd)) {
1867                         res = SCST_EXEC_NEED_THREAD;
1868                         goto out_restore;
1869                 } else {
1870                         PRINT_INFO("%s", "Unable to allocate request, "
1871                                 "sending BUSY status");
1872                         goto out_busy;
1873                 }
1874         }
1875
1876         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1877                     (void *)cmd->scsi_req->sr_buffer,
1878                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1879                     cmd->retries);
1880 #else
1881         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1882                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1883                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1884                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1885         if (unlikely(rc != 0)) {
1886                 if (scst_cmd_atomic(cmd)) {
1887                         res = SCST_EXEC_NEED_THREAD;
1888                         goto out_restore;
1889                 } else {
1890                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1891                         goto out_error;
1892                 }
1893         }
1894 #endif
1895
1896 out_complete:
1897         res = SCST_EXEC_COMPLETED;
1898
1899 out:
1900         TRACE_EXIT();
1901         return res;
1902
1903 out_restore:
1904         /* Restore the state */
1905         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1906         goto out;
1907
1908 out_error:
1909         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1910         goto out_done;
1911
1912 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1913 out_busy:
1914         scst_set_busy(cmd);
1915         /* go through */
1916 #endif
1917
1918 out_done:
1919         res = SCST_EXEC_COMPLETED;
1920         /* Report the result */
1921         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1922         goto out_complete;
1923 }
1924
1925 static inline int scst_real_exec(struct scst_cmd *cmd)
1926 {
1927         int res;
1928
1929         TRACE_ENTRY();
1930
1931         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1932         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1933         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1934
1935         __scst_cmd_get(cmd);
1936
1937         res = scst_do_real_exec(cmd);
1938
1939         if (likely(res == SCST_EXEC_COMPLETED)) {
1940                 scst_post_exec_sn(cmd, true);
1941                 if (cmd->dev->scsi_dev != NULL)
1942                         generic_unplug_device(
1943                                 cmd->dev->scsi_dev->request_queue);
1944         } else
1945                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1946
1947         __scst_cmd_put(cmd);
1948
1949         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1950
1951         TRACE_EXIT_RES(res);
1952         return res;
1953 }
1954
1955 static int scst_do_local_exec(struct scst_cmd *cmd)
1956 {
1957         int res;
1958         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1959
1960         TRACE_ENTRY();
1961
1962         /* Check READ_ONLY device status */
1963         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1964             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1965              cmd->cdb[0] == WRITE_10 ||
1966              cmd->cdb[0] == WRITE_12 ||
1967              cmd->cdb[0] == WRITE_16 ||
1968              cmd->cdb[0] == WRITE_VERIFY ||
1969              cmd->cdb[0] == WRITE_VERIFY_12 ||
1970              cmd->cdb[0] == WRITE_VERIFY_16 ||
1971              (cmd->dev->handler->type == TYPE_TAPE &&
1972               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1973                 scst_set_cmd_error(cmd,
1974                            SCST_LOAD_SENSE(scst_sense_data_protect));
1975                 goto out_done;
1976         }
1977
1978         /*
1979          * Adding new commands here don't forget to update
1980          * scst_is_cmd_local() in scst.h, if necessary
1981          */
1982
1983         switch (cmd->cdb[0]) {
1984         case MODE_SELECT:
1985         case MODE_SELECT_10:
1986         case LOG_SELECT:
1987                 res = scst_pre_select(cmd);
1988                 break;
1989         case RESERVE:
1990         case RESERVE_10:
1991                 res = scst_reserve_local(cmd);
1992                 break;
1993         case RELEASE:
1994         case RELEASE_10:
1995                 res = scst_release_local(cmd);
1996                 break;
1997         case REPORT_LUNS:
1998                 res = scst_report_luns_local(cmd);
1999                 break;
2000         default:
2001                 res = SCST_EXEC_NOT_COMPLETED;
2002                 break;
2003         }
2004
2005 out:
2006         TRACE_EXIT_RES(res);
2007         return res;
2008
2009 out_done:
2010         /* Report the result */
2011         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2012         res = SCST_EXEC_COMPLETED;
2013         goto out;
2014 }
2015
2016 static int scst_local_exec(struct scst_cmd *cmd)
2017 {
2018         int res;
2019
2020         TRACE_ENTRY();
2021
2022         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2023         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2024         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2025
2026         __scst_cmd_get(cmd);
2027
2028         res = scst_do_local_exec(cmd);
2029         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2030                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2031         else if (res == SCST_EXEC_COMPLETED)
2032                 scst_post_exec_sn(cmd, true);
2033         else
2034                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2035
2036         __scst_cmd_put(cmd);
2037
2038         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2039         TRACE_EXIT_RES(res);
2040         return res;
2041 }
2042
2043 static int scst_exec(struct scst_cmd **active_cmd)
2044 {
2045         struct scst_cmd *cmd = *active_cmd;
2046         struct scst_cmd *ref_cmd;
2047         struct scst_device *dev = cmd->dev;
2048         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2049
2050         TRACE_ENTRY();
2051
2052         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2053                 goto out;
2054
2055         /* To protect tgt_dev */
2056         ref_cmd = cmd;
2057         __scst_cmd_get(ref_cmd);
2058
2059         count = 0;
2060         while (1) {
2061                 int rc;
2062
2063                 cmd->sent_for_exec = 1;
2064                 cmd->scst_cmd_done = scst_cmd_done_local;
2065                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2066
2067                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2068                     (cmd->data_direction == SCST_DATA_WRITE))
2069                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2070
2071                 rc = scst_do_local_exec(cmd);
2072                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2073                         /* Nothing to do */;
2074                 else if (rc == SCST_EXEC_NEED_THREAD) {
2075                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2076                                 "thread context, rescheduling");
2077                         scst_dec_on_dev_cmd(cmd);
2078                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2079                         break;
2080                 } else {
2081                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2082                         goto done;
2083                 }
2084
2085                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2086
2087                 rc = scst_do_real_exec(cmd);
2088                 if (likely(rc == SCST_EXEC_COMPLETED))
2089                         /* Nothing to do */;
2090                 else if (rc == SCST_EXEC_NEED_THREAD) {
2091                         TRACE_DBG("scst_real_exec() requested thread "
2092                                 "context, rescheduling (cmd %p)", cmd);
2093                         scst_dec_on_dev_cmd(cmd);
2094                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2095                         break;
2096                 } else
2097                         sBUG();
2098
2099 done:
2100                 count++;
2101
2102                 cmd = scst_post_exec_sn(cmd, false);
2103                 if (cmd == NULL)
2104                         break;
2105
2106                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2107                         break;
2108
2109                 __scst_cmd_put(ref_cmd);
2110                 ref_cmd = cmd;
2111                 __scst_cmd_get(ref_cmd);
2112         }
2113
2114         *active_cmd = cmd;
2115
2116         if (count == 0)
2117                 goto out_put;
2118
2119         if (dev->scsi_dev != NULL)
2120                 generic_unplug_device(dev->scsi_dev->request_queue);
2121
2122 out_put:
2123         __scst_cmd_put(ref_cmd);
2124         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2125
2126 out:
2127         TRACE_EXIT_RES(res);
2128         return res;
2129 }
2130
2131 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2132 {
2133         int res;
2134         struct scst_cmd *cmd = *active_cmd;
2135         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2136         typeof(tgt_dev->expected_sn) expected_sn;
2137
2138         TRACE_ENTRY();
2139
2140 #ifdef CONFIG_SCST_MEASURE_LATENCY
2141         if (cmd->pre_exec_finish == 0) {
2142                 struct timespec ts;
2143                 getnstimeofday(&ts);
2144                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2145                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2146                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2147                         ts.tv_sec, ts.tv_nsec);
2148         }
2149 #endif
2150
2151         if (unlikely(cmd->internal))
2152                 goto exec;
2153
2154         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2155                 goto exec;
2156
2157         sBUG_ON(!cmd->sn_set);
2158
2159         expected_sn = tgt_dev->expected_sn;
2160         /* Optimized for lockless fast path */
2161         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2162                 spin_lock_irq(&tgt_dev->sn_lock);
2163
2164                 tgt_dev->def_cmd_count++;
2165                 smp_mb();
2166
2167                 expected_sn = tgt_dev->expected_sn;
2168                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2169                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2170                                               &cmd->cmd_flags))) {
2171                                 /* Necessary to allow aborting out of sn cmds */
2172                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2173                                         "(tag %llu, sn %lu)", cmd,
2174                                         (long long unsigned)cmd->tag, cmd->sn);
2175                                 tgt_dev->def_cmd_count--;
2176                                 scst_set_cmd_abnormal_done_state(cmd);
2177                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2178                         } else {
2179                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2180                                         "expected_sn=%ld)", cmd, cmd->sn,
2181                                         cmd->sn_set, expected_sn);
2182                                 list_add_tail(&cmd->sn_cmd_list_entry,
2183                                               &tgt_dev->deferred_cmd_list);
2184                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2185                         }
2186                         spin_unlock_irq(&tgt_dev->sn_lock);
2187                         goto out;
2188                 } else {
2189                         TRACE_SN("Somebody incremented expected_sn %ld, "
2190                                 "continuing", expected_sn);
2191                         tgt_dev->def_cmd_count--;
2192                         spin_unlock_irq(&tgt_dev->sn_lock);
2193                 }
2194         }
2195
2196 exec:
2197         res = scst_exec(active_cmd);
2198
2199 out:
2200         TRACE_EXIT_HRES(res);
2201         return res;
2202 }
2203
2204 /* No locks supposed to be held */
2205 static int scst_check_sense(struct scst_cmd *cmd)
2206 {
2207         int res = 0;
2208         struct scst_device *dev = cmd->dev;
2209
2210         TRACE_ENTRY();
2211
2212         if (unlikely(cmd->ua_ignore))
2213                 goto out;
2214
2215         /* If we had internal bus reset behind us, set the command error UA */
2216         if ((dev->scsi_dev != NULL) &&
2217             unlikely(cmd->host_status == DID_RESET) &&
2218             scst_is_ua_command(cmd)) {
2219                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2220                       dev->scsi_dev->was_reset, cmd->host_status);
2221                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2222                 /* It looks like it is safe to clear was_reset here */
2223                 dev->scsi_dev->was_reset = 0;
2224         }
2225
2226         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2227             SCST_SENSE_VALID(cmd->sense)) {
2228                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2229                         SCST_SENSE_BUFFERSIZE);
2230
2231                 /* Check Unit Attention Sense Key */
2232                 if (scst_is_ua_sense(cmd->sense)) {
2233                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2234                                 if (cmd->double_ua_possible) {
2235                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2236                                                 "detected for device %p", dev);
2237                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2238                                                 " %p (tag %llu)", cmd,
2239                                                 (long long unsigned)cmd->tag);
2240
2241                                         cmd->status = 0;
2242                                         cmd->msg_status = 0;
2243                                         cmd->host_status = DID_OK;
2244                                         cmd->driver_status = 0;
2245
2246                                         mempool_free(cmd->sense,
2247                                                      scst_sense_mempool);
2248                                         cmd->sense = NULL;
2249
2250                                         scst_check_restore_sg_buff(cmd);
2251
2252                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2253                                         cmd->data_direction =
2254                                                 cmd->dbl_ua_orig_data_direction;
2255                                         cmd->resp_data_len =
2256                                                 cmd->dbl_ua_orig_resp_data_len;
2257
2258                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2259                                         cmd->retry = 1;
2260                                         res = 1;
2261                                         goto out;
2262                                 }
2263                         }
2264                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2265                                 SCST_SENSE_BUFFERSIZE);
2266                 }
2267         }
2268
2269         if (unlikely(cmd->double_ua_possible)) {
2270                 if (scst_is_ua_command(cmd)) {
2271                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2272                                 "cmd %p)", dev, cmd);
2273                         /*
2274                          * Lock used to protect other flags in the bitfield
2275                          * (just in case, actually). Those 2 flags can't be
2276                          * changed in parallel, because the device is
2277                          * serialized.
2278                          */
2279                         spin_lock_bh(&dev->dev_lock);
2280                         dev->dev_double_ua_possible = 0;
2281                         dev->dev_serialized = 0;
2282                         spin_unlock_bh(&dev->dev_lock);
2283                 }
2284         }
2285
2286 out:
2287         TRACE_EXIT_RES(res);
2288         return res;
2289 }
2290
2291 static int scst_check_auto_sense(struct scst_cmd *cmd)
2292 {
2293         int res = 0;
2294
2295         TRACE_ENTRY();
2296
2297         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2298             (!SCST_SENSE_VALID(cmd->sense) ||
2299              SCST_NO_SENSE(cmd->sense))) {
2300                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2301                       "cmd->status=%x, cmd->msg_status=%x, "
2302                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2303                       cmd->status, cmd->msg_status, cmd->host_status,
2304                       cmd->driver_status, cmd);
2305                 res = 1;
2306         } else if (unlikely(cmd->host_status)) {
2307                 if ((cmd->host_status == DID_REQUEUE) ||
2308                     (cmd->host_status == DID_IMM_RETRY) ||
2309                     (cmd->host_status == DID_SOFT_ERROR) ||
2310                     (cmd->host_status == DID_ABORT)) {
2311                         scst_set_busy(cmd);
2312                 } else {
2313                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2314                                 "received, returning HARDWARE ERROR instead "
2315                                 "(cmd %p)", cmd->host_status, cmd);
2316                         scst_set_cmd_error(cmd,
2317                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2318                 }
2319         }
2320
2321         TRACE_EXIT_RES(res);
2322         return res;
2323 }
2324
2325 static int scst_pre_dev_done(struct scst_cmd *cmd)
2326 {
2327         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2328
2329         TRACE_ENTRY();
2330
2331         if (unlikely(scst_check_auto_sense(cmd))) {
2332                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2333                             "without sense data (opcode 0x%x), issuing "
2334                             "REQUEST SENSE", cmd->cdb[0]);
2335                 rc = scst_prepare_request_sense(cmd);
2336                 if (rc == 0)
2337                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2338                 else {
2339                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2340                                     "returning HARDWARE ERROR");
2341                         scst_set_cmd_error(cmd,
2342                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2343                 }
2344                 goto out;
2345         } else if (unlikely(scst_check_sense(cmd)))
2346                 goto out;
2347
2348         if (likely(scsi_status_is_good(cmd->status))) {
2349                 unsigned char type = cmd->dev->handler->type;
2350                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2351                               cmd->cdb[0] == MODE_SENSE_10)) &&
2352                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2353                     (type == TYPE_DISK ||
2354                      type == TYPE_WORM ||
2355                      type == TYPE_MOD ||
2356                      type == TYPE_TAPE)) {
2357                         int32_t length;
2358                         uint8_t *address;
2359                         bool err = false;
2360
2361                         length = scst_get_buf_first(cmd, &address);
2362                         if (length < 0) {
2363                                 PRINT_ERROR("%s", "Unable to get "
2364                                         "MODE_SENSE buffer");
2365                                 scst_set_cmd_error(cmd,
2366                                         SCST_LOAD_SENSE(
2367                                                 scst_sense_hardw_error));
2368                                 err = true;
2369                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2370                                 address[2] |= 0x80;   /* Write Protect*/
2371                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2372                                 address[3] |= 0x80;   /* Write Protect*/
2373                         scst_put_buf(cmd, address);
2374
2375                         if (err)
2376                                 goto out;
2377                 }
2378
2379                 /*
2380                  * Check and clear NormACA option for the device, if necessary,
2381                  * since we don't support ACA
2382                  */
2383                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2384                     /* Std INQUIRY data (no EVPD) */
2385                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2386                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2387                         uint8_t *buffer;
2388                         int buflen;
2389                         bool err = false;
2390
2391                         /* ToDo: all pages ?? */
2392                         buflen = scst_get_buf_first(cmd, &buffer);
2393                         if (buflen > SCST_INQ_BYTE3) {
2394 #ifdef CONFIG_SCST_EXTRACHECKS
2395                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2396                                         PRINT_INFO("NormACA set for device: "
2397                                             "lun=%lld, type 0x%02x. Clear it, "
2398                                             "since it's unsupported.",
2399                                             (long long unsigned int)cmd->lun,
2400                                             buffer[0]);
2401                                 }
2402 #endif
2403                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2404                         } else if (buflen != 0) {
2405                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2406                                     "buffer");
2407                                 scst_set_cmd_error(cmd,
2408                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2409                                 err = true;
2410                         }
2411                         if (buflen > 0)
2412                                 scst_put_buf(cmd, buffer);
2413
2414                         if (err)
2415                                 goto out;
2416                 }
2417
2418                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2419                     (cmd->cdb[0] == MODE_SELECT_10) ||
2420                     (cmd->cdb[0] == LOG_SELECT))) {
2421                         TRACE(TRACE_SCSI,
2422                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2423                                 (long long unsigned int)cmd->lun);
2424                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2425                         goto out;
2426                 }
2427         } else {
2428                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2429                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2430                                         &cmd->tgt_dev->tgt_dev_flags)) {
2431                                 struct scst_tgt_dev *tgt_dev_tmp;
2432                                 struct scst_device *dev = cmd->dev;
2433
2434                                 TRACE(TRACE_SCSI,
2435                                         "Real RESERVE failed lun=%lld, "
2436                                         "status=%x",
2437                                         (long long unsigned int)cmd->lun,
2438                                         cmd->status);
2439                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2440                                         SCST_SENSE_BUFFERSIZE);
2441
2442                                 /* Clearing the reservation */
2443                                 spin_lock_bh(&dev->dev_lock);
2444                                 list_for_each_entry(tgt_dev_tmp,
2445                                                     &dev->dev_tgt_dev_list,
2446                                                     dev_tgt_dev_list_entry) {
2447                                         clear_bit(SCST_TGT_DEV_RESERVED,
2448                                                 &tgt_dev_tmp->tgt_dev_flags);
2449                                 }
2450                                 dev->dev_reserved = 0;
2451                                 spin_unlock_bh(&dev->dev_lock);
2452                         }
2453                 }
2454
2455                 /* Check for MODE PARAMETERS CHANGED UA */
2456                 if ((cmd->dev->scsi_dev != NULL) &&
2457                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2458                     SCST_SENSE_VALID(cmd->sense) &&
2459                     scst_is_ua_sense(cmd->sense) &&
2460                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2461                         TRACE(TRACE_SCSI,
2462                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2463                               (long long unsigned int)cmd->lun);
2464                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2465                         goto out;
2466                 }
2467         }
2468
2469         cmd->state = SCST_CMD_STATE_DEV_DONE;
2470
2471 out:
2472         TRACE_EXIT_RES(res);
2473         return res;
2474 }
2475
2476 static int scst_mode_select_checks(struct scst_cmd *cmd)
2477 {
2478         int res = SCST_CMD_STATE_RES_CONT_SAME;
2479         int atomic = scst_cmd_atomic(cmd);
2480
2481         TRACE_ENTRY();
2482
2483         if (likely(scsi_status_is_good(cmd->status))) {
2484                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2485                     (cmd->cdb[0] == MODE_SELECT_10) ||
2486                     (cmd->cdb[0] == LOG_SELECT))) {
2487                         struct scst_device *dev = cmd->dev;
2488                         if (atomic && (dev->scsi_dev != NULL)) {
2489                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2490                                         "context required");
2491                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2492                                 goto out;
2493                         }
2494
2495                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2496                                 "setting the SELECT UA (lun=%lld)",
2497                                 (long long unsigned int)cmd->lun);
2498
2499                         spin_lock_bh(&dev->dev_lock);
2500                         spin_lock(&scst_temp_UA_lock);
2501                         if (cmd->cdb[0] == LOG_SELECT) {
2502                                 scst_set_sense(scst_temp_UA,
2503                                         sizeof(scst_temp_UA),
2504                                         UNIT_ATTENTION, 0x2a, 0x02);
2505                         } else {
2506                                 scst_set_sense(scst_temp_UA,
2507                                         sizeof(scst_temp_UA),
2508                                         UNIT_ATTENTION, 0x2a, 0x01);
2509                         }
2510                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2511                                 sizeof(scst_temp_UA));
2512                         spin_unlock(&scst_temp_UA_lock);
2513                         spin_unlock_bh(&dev->dev_lock);
2514
2515                         if (dev->scsi_dev != NULL)
2516                                 scst_obtain_device_parameters(dev);
2517                 }
2518         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2519                     SCST_SENSE_VALID(cmd->sense) &&
2520                     scst_is_ua_sense(cmd->sense) &&
2521                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2522                      (cmd->sense[12] == 0x29) /* reset */ ||
2523                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2524                      /* cleared by another ini (just in case) */
2525                      (cmd->sense[12] == 0x2F))) {
2526                 if (atomic) {
2527                         TRACE_DBG("Possible parameters changed UA %x: "
2528                                 "thread context required", cmd->sense[12]);
2529                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2530                         goto out;
2531                 }
2532
2533                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2534                         "(lun %lld): getting new parameters", cmd->sense[12],
2535                         (long long unsigned int)cmd->lun);
2536
2537                 scst_obtain_device_parameters(cmd->dev);
2538         } else
2539                 sBUG();
2540
2541         cmd->state = SCST_CMD_STATE_DEV_DONE;
2542
2543 out:
2544         TRACE_EXIT_HRES(res);
2545         return res;
2546 }
2547
2548 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2549 {
2550         if (likely(cmd->sn_set))
2551                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2552
2553         scst_make_deferred_commands_active(cmd->tgt_dev);
2554 }
2555
2556 static int scst_dev_done(struct scst_cmd **pcmd)
2557 {
2558         int res = SCST_CMD_STATE_RES_CONT_SAME;
2559         struct scst_cmd *cmd = *pcmd;
2560         int state;
2561         struct scst_device *dev = cmd->dev;
2562
2563         TRACE_ENTRY();
2564
2565         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2566
2567         if (likely(!scst_is_cmd_local(cmd)) &&
2568             likely(dev->handler->dev_done != NULL)) {
2569                 int rc;
2570
2571                 if (unlikely(!dev->handler->dev_done_atomic &&
2572                              scst_cmd_atomic(cmd))) {
2573                         /*
2574                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2575                          * optimization.
2576                          */
2577                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2578                               "context, rescheduling", dev->handler->name);
2579                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2580                         goto out;
2581                 }
2582
2583                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2584                       dev->handler->name, cmd);
2585                 rc = dev->handler->dev_done(cmd);
2586                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2587                       dev->handler->name, rc);
2588                 if (rc != SCST_CMD_STATE_DEFAULT)
2589                         state = rc;
2590         }
2591
2592         switch (state) {
2593         case SCST_CMD_STATE_PRE_XMIT_RESP:
2594         case SCST_CMD_STATE_DEV_PARSE:
2595         case SCST_CMD_STATE_PRE_PARSE:
2596         case SCST_CMD_STATE_PREPARE_SPACE:
2597         case SCST_CMD_STATE_RDY_TO_XFER:
2598         case SCST_CMD_STATE_TGT_PRE_EXEC:
2599         case SCST_CMD_STATE_SEND_FOR_EXEC:
2600         case SCST_CMD_STATE_LOCAL_EXEC:
2601         case SCST_CMD_STATE_REAL_EXEC:
2602         case SCST_CMD_STATE_PRE_DEV_DONE:
2603         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2604         case SCST_CMD_STATE_DEV_DONE:
2605         case SCST_CMD_STATE_XMIT_RESP:
2606         case SCST_CMD_STATE_FINISHED:
2607                 cmd->state = state;
2608                 break;
2609
2610         case SCST_CMD_STATE_NEED_THREAD_CTX:
2611                 TRACE_DBG("Dev handler %s dev_done() requested "
2612                       "thread context, rescheduling",
2613                       dev->handler->name);
2614                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2615                 break;
2616
2617         default:
2618                 if (state >= 0) {
2619                         PRINT_ERROR("Dev handler %s dev_done() returned "
2620                                 "invalid cmd state %d",
2621                                 dev->handler->name, state);
2622                 } else {
2623                         PRINT_ERROR("Dev handler %s dev_done() returned "
2624                                 "error %d", dev->handler->name,
2625                                 state);
2626                 }
2627                 scst_set_cmd_error(cmd,
2628                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2629                 scst_set_cmd_abnormal_done_state(cmd);
2630                 break;
2631         }
2632
2633         if (cmd->needs_unblocking)
2634                 scst_unblock_dev_cmd(cmd);
2635
2636         if (likely(cmd->dec_on_dev_needed))
2637                 scst_dec_on_dev_cmd(cmd);
2638
2639         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2640                 scst_inc_check_expected_sn(cmd);
2641
2642         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2643                 *pcmd = scst_complete_request_sense(cmd);
2644
2645 out:
2646         TRACE_EXIT_HRES(res);
2647         return res;
2648 }
2649
2650 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2651 {
2652         int res;
2653
2654         TRACE_ENTRY();
2655
2656 #ifdef CONFIG_SCST_DEBUG_TM
2657         if (cmd->tm_dbg_delayed &&
2658                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2659                 if (scst_cmd_atomic(cmd)) {
2660                         TRACE_MGMT_DBG("%s",
2661                                 "DEBUG_TM delayed cmd needs a thread");
2662                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2663                         return res;
2664                 }
2665                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2666                         cmd, cmd->tag);
2667                 schedule_timeout_uninterruptible(HZ);
2668         }
2669 #endif
2670
2671         if (likely(cmd->tgt_dev != NULL)) {
2672                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2673                 atomic_dec(&cmd->dev->dev_cmd_count);
2674                 /* If expected values not set, expected direction is UNKNOWN */
2675                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2676                         atomic_dec(&cmd->dev->write_cmd_count);
2677
2678                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2679                         scst_on_hq_cmd_response(cmd);
2680
2681                 if (unlikely(!cmd->sent_for_exec)) {
2682                         TRACE_SN("cmd %p was not sent to mid-lev"
2683                                 " (sn %ld, set %d)",
2684                                 cmd, cmd->sn, cmd->sn_set);
2685                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2686                         cmd->sent_for_exec = 1;
2687                 }
2688         }
2689
2690         /*
2691          * If we don't remove cmd from the search list here, before
2692          * submitting it for transmittion, we will have a race, when for
2693          * some reason cmd's release is delayed after transmittion and
2694          * initiator sends cmd with the same tag => it is possible that
2695          * a wrong cmd will be found by find() functions.
2696          */
2697         spin_lock_irq(&cmd->sess->sess_list_lock);
2698         list_del(&cmd->search_cmd_list_entry);
2699         spin_unlock_irq(&cmd->sess->sess_list_lock);
2700
2701         cmd->done = 1;
2702         smp_mb(); /* to sync with scst_abort_cmd() */
2703
2704         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2705                 scst_xmit_process_aborted_cmd(cmd);
2706
2707         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2708                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2709                                 " skipping",
2710                                 cmd, (long long unsigned int)cmd->tag);
2711                 cmd->state = SCST_CMD_STATE_FINISHED;
2712                 res = SCST_CMD_STATE_RES_CONT_SAME;
2713                 goto out;
2714         }
2715
2716         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2717             (cmd->data_direction == SCST_DATA_READ))
2718                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2719
2720         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2721         res = SCST_CMD_STATE_RES_CONT_SAME;
2722
2723 out:
2724 #ifdef CONFIG_SCST_MEASURE_LATENCY
2725         {
2726                 struct timespec ts;
2727                 uint64_t finish, scst_time, proc_time;
2728                 struct scst_session *sess = cmd->sess;
2729
2730                 getnstimeofday(&ts);
2731                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2732
2733                 spin_lock_bh(&sess->meas_lock);
2734
2735                 scst_time = cmd->pre_exec_finish - cmd->start;
2736                 scst_time += finish - cmd->post_exec_start;
2737                 proc_time = finish - cmd->start;
2738
2739                 sess->scst_time += scst_time;
2740                 sess->processing_time += proc_time;
2741                 sess->processed_cmds++;
2742
2743                 spin_unlock_bh(&sess->meas_lock);
2744
2745                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2746                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2747                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2748                         proc_time);
2749         }
2750 #endif
2751         TRACE_EXIT_HRES(res);
2752         return res;
2753 }
2754
2755 static int scst_xmit_response(struct scst_cmd *cmd)
2756 {
2757         int res, rc;
2758
2759         TRACE_ENTRY();
2760
2761         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2762                      scst_cmd_atomic(cmd))) {
2763                 /*
2764                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2765                  * optimization.
2766                  */
2767                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2768                               "context, rescheduling", cmd->tgtt->name);
2769                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2770                 goto out;
2771         }
2772
2773         while (1) {
2774                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2775
2776                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2777                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2778
2779                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2780
2781 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2782                 if (cmd->tgt_sg) {
2783                         int i;
2784                         struct scatterlist *sg = cmd->tgt_sg;
2785                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2786                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2787                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2788                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2789                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2790                                         sg_virt(&sg[i]), sg[i].length);
2791                         }
2792                 }
2793 #endif
2794
2795 #ifdef CONFIG_SCST_DEBUG_RETRY
2796                 if (((scst_random() % 100) == 77))
2797                         rc = SCST_TGT_RES_QUEUE_FULL;
2798                 else
2799 #endif
2800                         rc = cmd->tgtt->xmit_response(cmd);
2801                 TRACE_DBG("xmit_response() returned %d", rc);
2802
2803                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2804                         goto out;
2805
2806                 /* Restore the previous state */
2807                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2808
2809                 switch (rc) {
2810                 case SCST_TGT_RES_QUEUE_FULL:
2811                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2812                                 break;
2813                         else
2814                                 continue;
2815
2816                 case SCST_TGT_RES_NEED_THREAD_CTX:
2817                         TRACE_DBG("Target driver %s xmit_response() "
2818                               "requested thread context, rescheduling",
2819                               cmd->tgtt->name);
2820                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2821                         break;
2822
2823                 default:
2824                         goto out_error;
2825                 }
2826                 break;
2827         }
2828
2829 out:
2830         /* Caution: cmd can be already dead here */
2831         TRACE_EXIT_HRES(res);
2832         return res;
2833
2834 out_error:
2835         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2836                 PRINT_ERROR("Target driver %s xmit_response() returned "
2837                         "fatal error", cmd->tgtt->name);
2838         } else {
2839                 PRINT_ERROR("Target driver %s xmit_response() returned "
2840                         "invalid value %d", cmd->tgtt->name, rc);
2841         }
2842         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2843         cmd->state = SCST_CMD_STATE_FINISHED;
2844         res = SCST_CMD_STATE_RES_CONT_SAME;
2845         goto out;
2846 }
2847
2848 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2849         enum scst_exec_context pref_context)
2850 {
2851         TRACE_ENTRY();
2852
2853         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2854
2855         cmd->state = SCST_CMD_STATE_FINISHED;
2856         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2857
2858         TRACE_EXIT();
2859         return;
2860 }
2861 EXPORT_SYMBOL(scst_tgt_cmd_done);
2862
2863 static int scst_finish_cmd(struct scst_cmd *cmd)
2864 {
2865         int res;
2866
2867         TRACE_ENTRY();
2868
2869         atomic_dec(&cmd->sess->sess_cmd_count);
2870
2871         cmd->finished = 1;
2872         smp_mb(); /* to sync with scst_abort_cmd() */
2873
2874         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2875                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2876                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2877                         atomic_read(&scst_cmd_count));
2878
2879                 scst_finish_cmd_mgmt(cmd);
2880         }
2881
2882         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2883                 if ((cmd->tgt_dev != NULL) &&
2884                     scst_is_ua_sense(cmd->sense)) {
2885                         /* This UA delivery failed, so requeue it */
2886                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2887                                 "%p", cmd);
2888                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2889                                         SCST_SENSE_BUFFERSIZE, 1);
2890                 }
2891         }
2892
2893         __scst_cmd_put(cmd);
2894
2895         res = SCST_CMD_STATE_RES_CONT_NEXT;
2896
2897         TRACE_EXIT_HRES(res);
2898         return res;
2899 }
2900
2901 /*
2902  * No locks, but it must be externally serialized (see comment for
2903  * scst_cmd_init_done() in scst.h)
2904  */
2905 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2906 {
2907         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2908         unsigned long flags;
2909
2910         TRACE_ENTRY();
2911
2912         if (scst_is_implicit_hq(cmd)) {
2913                 TRACE_SN("Implicit HQ cmd %p", cmd);
2914                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2915         }
2916
2917         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2918
2919         /* Optimized for lockless fast path */
2920
2921         scst_check_debug_sn(cmd);
2922
2923         if (cmd->dev->queue_alg ==
2924                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2925                 /*
2926                  * Not the best way, but well enough until there will be a
2927                  * possibility to specify queue type during pass-through
2928                  * commands submission.
2929                  */
2930                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2931         }
2932
2933         switch (cmd->queue_type) {
2934         case SCST_CMD_QUEUE_SIMPLE:
2935         case SCST_CMD_QUEUE_UNTAGGED:
2936 #if 1 /* temporary, ToDo */
2937                 if (scst_cmd_is_expected_set(cmd)) {
2938                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2939                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2940                                 goto ordered;
2941                 } else
2942                         goto ordered;
2943 #endif
2944                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2945                         /*
2946                          * atomic_inc_return() implies memory barrier to sync
2947                          * with scst_inc_expected_sn()
2948                          */
2949                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2950                                 tgt_dev->curr_sn++;
2951                                 TRACE_SN("Incremented curr_sn %ld",
2952                                         tgt_dev->curr_sn);
2953                         }
2954                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2955                         cmd->sn = tgt_dev->curr_sn;
2956
2957                         tgt_dev->prev_cmd_ordered = 0;
2958                 } else {
2959                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2960                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2961                         goto ordered;
2962                 }
2963                 break;
2964
2965         case SCST_CMD_QUEUE_ORDERED:
2966                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
2967 ordered:
2968                 if (!tgt_dev->prev_cmd_ordered) {
2969                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2970                         if (tgt_dev->num_free_sn_slots >= 0) {
2971                                 tgt_dev->num_free_sn_slots--;
2972                                 if (tgt_dev->num_free_sn_slots >= 0) {
2973                                         int i = 0;
2974                                         /* Commands can finish in any order, so
2975                                          * we don't know which slot is empty.
2976                                          */
2977                                         while (1) {
2978                                                 tgt_dev->cur_sn_slot++;
2979                                                 if (tgt_dev->cur_sn_slot ==
2980                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
2981                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2982
2983                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2984                                                         break;
2985
2986                                                 i++;
2987                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2988                                         }
2989                                         TRACE_SN("New cur SN slot %zd",
2990                                                 tgt_dev->cur_sn_slot -
2991                                                 tgt_dev->sn_slots);
2992                                 }
2993                         }
2994                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2995                 }
2996                 tgt_dev->prev_cmd_ordered = 1;
2997                 tgt_dev->curr_sn++;
2998                 cmd->sn = tgt_dev->curr_sn;
2999                 break;
3000
3001         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3002                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3003                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3004                 tgt_dev->hq_cmd_count++;
3005                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3006                 cmd->hq_cmd_inced = 1;
3007                 goto out;
3008
3009         default:
3010                 sBUG();
3011         }
3012
3013         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3014                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3015                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3016                 atomic_read(tgt_dev->cur_sn_slot),
3017                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3018                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3019
3020         cmd->sn_set = 1;
3021
3022 out:
3023         TRACE_EXIT();
3024         return;
3025 }
3026
3027 /*
3028  * Returns 0 on success, > 0 when we need to wait for unblock,
3029  * < 0 if there is no device (lun) or device type handler.
3030  *
3031  * No locks, but might be on IRQ, protection is done by the
3032  * suspended activity.
3033  */
3034 static int scst_translate_lun(struct scst_cmd *cmd)
3035 {
3036         struct scst_tgt_dev *tgt_dev = NULL;
3037         int res;
3038
3039         TRACE_ENTRY();
3040
3041         __scst_get(1);
3042
3043         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3044                 struct list_head *sess_tgt_dev_list_head =
3045                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3046                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3047                         (long long unsigned int)cmd->lun);
3048                 res = -1;
3049                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3050                                 sess_tgt_dev_list_entry) {
3051                         if (tgt_dev->lun == cmd->lun) {
3052                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3053
3054                                 if (unlikely(tgt_dev->dev->handler ==
3055                                                 &scst_null_devtype)) {
3056                                         PRINT_INFO("Dev handler for device "
3057                                           "%lld is NULL, the device will not "
3058                                           "be visible remotely",
3059                                            (long long unsigned int)cmd->lun);
3060                                         break;
3061                                 }
3062
3063                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3064                                 cmd->tgt_dev = tgt_dev;
3065                                 cmd->dev = tgt_dev->dev;
3066
3067                                 res = 0;
3068                                 break;
3069                         }
3070                 }
3071                 if (res != 0) {
3072                         TRACE(TRACE_MINOR,
3073                                 "tgt_dev for lun %lld not found, command to "
3074                                 "unexisting LU?",
3075                                 (long long unsigned int)cmd->lun);
3076                         __scst_put();
3077                 }
3078         } else {
3079                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3080                 __scst_put();
3081                 res = 1;
3082         }
3083
3084         TRACE_EXIT_RES(res);
3085         return res;
3086 }
3087
3088 /*
3089  * No locks, but might be on IRQ
3090  *
3091  * Returns 0 on success, > 0 when we need to wait for unblock,
3092  * < 0 if there is no device (lun) or device type handler.
3093  */
3094 static int __scst_init_cmd(struct scst_cmd *cmd)
3095 {
3096         int res = 0;
3097
3098         TRACE_ENTRY();
3099
3100         res = scst_translate_lun(cmd);
3101         if (likely(res == 0)) {
3102                 int cnt;
3103                 bool failure = false;
3104
3105                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3106
3107                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3108                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3109                         TRACE(TRACE_MGMT_MINOR,
3110                                 "Too many pending commands (%d) in "
3111                                 "session, returning BUSY to initiator \"%s\"",
3112                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3113                                   "Anonymous" : cmd->sess->initiator_name);
3114                         failure = true;
3115                 }
3116
3117                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3118                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3119                         if (!failure) {
3120                                 TRACE(TRACE_MGMT_MINOR,
3121                                         "Too many pending device "
3122                                         "commands (%d), returning BUSY to "
3123                                         "initiator \"%s\"", cnt,
3124                                         (cmd->sess->initiator_name[0] == '\0') ?
3125                                                 "Anonymous" :
3126                                                 cmd->sess->initiator_name);
3127                                 failure = true;
3128                         }
3129                 }
3130
3131                 /* If expected values not set, expected direction is UNKNOWN */
3132                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3133                         atomic_inc(&cmd->dev->write_cmd_count);
3134
3135                 if (unlikely(failure))
3136                         goto out_busy;
3137
3138                 if (!cmd->set_sn_on_restart_cmd)
3139                         scst_cmd_set_sn(cmd);
3140         } else if (res < 0) {
3141                 TRACE_DBG("Finishing cmd %p", cmd);
3142                 scst_set_cmd_error(cmd,
3143                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3144                 scst_set_cmd_abnormal_done_state(cmd);
3145         } else
3146                 goto out;
3147
3148 out:
3149         TRACE_EXIT_RES(res);
3150         return res;
3151
3152 out_busy:
3153         scst_set_busy(cmd);
3154         scst_set_cmd_abnormal_done_state(cmd);
3155         goto out;
3156 }
3157
3158 /* Called under scst_init_lock and IRQs disabled */
3159 static void scst_do_job_init(void)
3160         __releases(&scst_init_lock)
3161         __acquires(&scst_init_lock)
3162 {
3163         struct scst_cmd *cmd;
3164         int susp;
3165
3166         TRACE_ENTRY();
3167
3168 restart:
3169         /*
3170          * There is no need for read barrier here, because we don't care where
3171          * this check will be done.
3172          */
3173         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3174         if (scst_init_poll_cnt > 0)
3175                 scst_init_poll_cnt--;
3176
3177         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3178                 int rc;
3179                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3180                         continue;
3181                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3182                         spin_unlock_irq(&scst_init_lock);
3183                         rc = __scst_init_cmd(cmd);
3184                         spin_lock_irq(&scst_init_lock);
3185                         if (rc > 0) {
3186                                 TRACE_MGMT_DBG("%s",
3187                                         "FLAG SUSPENDED set, restarting");
3188                                 goto restart;
3189                         }
3190                 } else {
3191                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3192                                        cmd, (long long unsigned int)cmd->tag);
3193                         scst_set_cmd_abnormal_done_state(cmd);
3194                 }
3195
3196                 /*
3197                  * Deleting cmd from init cmd list after __scst_init_cmd()
3198                  * is necessary to keep the check in scst_init_cmd() correct
3199                  * to preserve the commands order.
3200                  *
3201                  * We don't care about the race, when init cmd list is empty
3202                  * and one command detected that it just was not empty, so
3203                  * it's inserting to it, but another command at the same time
3204                  * seeing init cmd list empty and goes directly, because it
3205                  * could affect only commands from the same initiator to the
3206                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3207                  * in case of simultaneous such calls anyway.
3208                  */
3209                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3210                 smp_wmb();
3211                 list_del(&cmd->cmd_list_entry);
3212                 spin_unlock(&scst_init_lock);
3213
3214                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3215                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3216                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3217                         list_add(&cmd->cmd_list_entry,
3218                                 &cmd->cmd_lists->active_cmd_list);
3219                 else
3220                         list_add_tail(&cmd->cmd_list_entry,
3221                                 &cmd->cmd_lists->active_cmd_list);
3222                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3223                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3224
3225                 spin_lock(&scst_init_lock);
3226                 goto restart;
3227         }
3228
3229         /* It isn't really needed, but let's keep it */
3230         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3231                 goto restart;
3232
3233         TRACE_EXIT();
3234         return;
3235 }
3236
3237 static inline int test_init_cmd_list(void)
3238 {
3239         int res = (!list_empty(&scst_init_cmd_list) &&
3240                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3241                   unlikely(kthread_should_stop()) ||
3242                   (scst_init_poll_cnt > 0);
3243         return res;
3244 }
3245
3246 int scst_init_cmd_thread(void *arg)
3247 {
3248         TRACE_ENTRY();
3249
3250         PRINT_INFO("Init thread started, PID %d", current->pid);
3251
3252         current->flags |= PF_NOFREEZE;
3253
3254         set_user_nice(current, -10);
3255
3256         spin_lock_irq(&scst_init_lock);
3257         while (!kthread_should_stop()) {
3258                 wait_queue_t wait;
3259                 init_waitqueue_entry(&wait, current);
3260
3261                 if (!test_init_cmd_list()) {
3262                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3263                                                  &wait);
3264                         for (;;) {
3265                                 set_current_state(TASK_INTERRUPTIBLE);
3266                                 if (test_init_cmd_list())
3267                                         break;
3268                                 spin_unlock_irq(&scst_init_lock);
3269                                 schedule();
3270                                 spin_lock_irq(&scst_init_lock);
3271                         }
3272                         set_current_state(TASK_RUNNING);
3273                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3274                 }
3275                 scst_do_job_init();
3276         }
3277         spin_unlock_irq(&scst_init_lock);
3278
3279         /*
3280          * If kthread_should_stop() is true, we are guaranteed to be
3281          * on the module unload, so scst_init_cmd_list must be empty.
3282          */
3283         sBUG_ON(!list_empty(&scst_init_cmd_list));
3284
3285         PRINT_INFO("Init thread PID %d finished", current->pid);
3286
3287         TRACE_EXIT();
3288         return 0;
3289 }
3290
3291 /* Called with no locks held */
3292 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3293 {
3294         int res;
3295
3296         TRACE_ENTRY();
3297
3298         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3299
3300         cmd->atomic = atomic;
3301
3302         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3303
3304         do {
3305                 switch (cmd->state) {
3306                 case SCST_CMD_STATE_PRE_PARSE:
3307                         res = scst_pre_parse(cmd);
3308                         EXTRACHECKS_BUG_ON(res ==
3309                                 SCST_CMD_STATE_RES_NEED_THREAD);
3310                         break;
3311
3312                 case SCST_CMD_STATE_DEV_PARSE:
3313                         res = scst_parse_cmd(cmd);
3314                         break;
3315
3316                 case SCST_CMD_STATE_PREPARE_SPACE:
3317                         res = scst_prepare_space(cmd);
3318                         break;
3319
3320                 case SCST_CMD_STATE_RDY_TO_XFER:
3321                         res = scst_rdy_to_xfer(cmd);
3322                         break;
3323
3324                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3325                         res = scst_tgt_pre_exec(cmd);
3326                         break;
3327
3328                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3329                         if (tm_dbg_check_cmd(cmd) != 0) {
3330                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3331                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3332                                         "because of TM DBG delay", cmd,
3333                                         (long long unsigned int)cmd->tag);
3334                                 break;
3335                         }
3336                         res = scst_send_for_exec(&cmd);
3337                         /*
3338                          * !! At this point cmd, sess & tgt_dev can already be
3339                          * freed !!
3340                          */
3341                         break;
3342
3343                 case SCST_CMD_STATE_LOCAL_EXEC:
3344                         res = scst_local_exec(cmd);
3345                         /*
3346                          * !! At this point cmd, sess & tgt_dev can already be
3347                          * freed !!
3348                          */
3349                         break;
3350
3351                 case SCST_CMD_STATE_REAL_EXEC:
3352                         res = scst_real_exec(cmd);
3353                         /*
3354                          * !! At this point cmd, sess & tgt_dev can already be
3355                          * freed !!
3356                          */
3357                         break;
3358
3359                 case SCST_CMD_STATE_PRE_DEV_DONE:
3360                         res = scst_pre_dev_done(cmd);
3361                         EXTRACHECKS_BUG_ON(res ==
3362                                 SCST_CMD_STATE_RES_NEED_THREAD);
3363                         break;
3364
3365                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3366                         res = scst_mode_select_checks(cmd);
3367                         break;
3368
3369                 case SCST_CMD_STATE_DEV_DONE:
3370                         res = scst_dev_done(&cmd);
3371                         break;
3372
3373                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3374                         res = scst_pre_xmit_response(cmd);
3375                         EXTRACHECKS_BUG_ON(res ==
3376                                 SCST_CMD_STATE_RES_NEED_THREAD);
3377                         break;
3378
3379                 case SCST_CMD_STATE_XMIT_RESP:
3380                         res = scst_xmit_response(cmd);
3381                         break;
3382
3383                 case SCST_CMD_STATE_FINISHED:
3384                         res = scst_finish_cmd(cmd);
3385                         EXTRACHECKS_BUG_ON(res ==
3386                                 SCST_CMD_STATE_RES_NEED_THREAD);
3387                         break;
3388
3389                 default:
3390                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3391                                 "be", cmd, cmd->state);
3392                         sBUG();
3393                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3394                         break;
3395                 }
3396         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3397
3398         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3399                 /* None */
3400         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3401                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3402                 switch (cmd->state) {
3403                 case SCST_CMD_STATE_PRE_PARSE:
3404                 case SCST_CMD_STATE_DEV_PARSE:
3405                 case SCST_CMD_STATE_PREPARE_SPACE:
3406                 case SCST_CMD_STATE_RDY_TO_XFER:
3407                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3408                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3409                 case SCST_CMD_STATE_LOCAL_EXEC:
3410                 case SCST_CMD_STATE_REAL_EXEC:
3411                 case SCST_CMD_STATE_PRE_DEV_DONE:
3412                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3413                 case SCST_CMD_STATE_DEV_DONE:
3414                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3415                 case SCST_CMD_STATE_XMIT_RESP:
3416                 case SCST_CMD_STATE_FINISHED:
3417                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3418                                   cmd);
3419                         list_add(&cmd->cmd_list_entry,
3420                                 &cmd->cmd_lists->active_cmd_list);
3421                         break;
3422 #ifdef CONFIG_SCST_EXTRACHECKS
3423                 /* not very valid commands */
3424                 case SCST_CMD_STATE_DEFAULT:
3425                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3426                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3427                                 cmd->state);
3428                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3429                         sBUG();
3430                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3431                         break;
3432 #endif
3433                 default:
3434                         break;
3435                 }
3436                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3437                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3438         } else
3439                 sBUG();
3440
3441         TRACE_EXIT();
3442         return;
3443 }
3444 EXPORT_SYMBOL(scst_process_active_cmd);
3445
3446 /* Called under cmd_list_lock and IRQs disabled */
3447 static void scst_do_job_active(struct list_head *cmd_list,
3448         spinlock_t *cmd_list_lock, bool atomic)
3449         __releases(cmd_list_lock)
3450         __acquires(cmd_list_lock)
3451 {
3452         TRACE_ENTRY();
3453
3454         while (!list_empty(cmd_list)) {
3455                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3456                                         cmd_list_entry);
3457                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3458                 list_del(&cmd->cmd_list_entry);
3459                 spin_unlock_irq(cmd_list_lock);
3460                 scst_process_active_cmd(cmd, atomic);
3461                 spin_lock_irq(cmd_list_lock);
3462         }
3463
3464         TRACE_EXIT();
3465         return;
3466 }
3467
3468 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3469 {
3470         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3471             unlikely(kthread_should_stop()) ||
3472             tm_dbg_is_release();
3473         return res;
3474 }
3475
3476 int scst_cmd_thread(void *arg)
3477 {
3478         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3479
3480         TRACE_ENTRY();
3481
3482         PRINT_INFO("Processing thread started, PID %d", current->pid);
3483
3484 #if 0
3485         set_user_nice(current, 10);
3486 #endif
3487         current->flags |= PF_NOFREEZE;
3488
3489         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3490         while (!kthread_should_stop()) {
3491                 wait_queue_t wait;
3492                 init_waitqueue_entry(&wait, current);
3493
3494                 if (!test_cmd_lists(p_cmd_lists)) {
3495                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3496                                 &wait);
3497                         for (;;) {
3498                                 set_current_state(TASK_INTERRUPTIBLE);