Fexes for processing internal REQUEST SENSE with scst_user handler
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
39         uint64_t tag);
40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
41         enum scst_exec_context context, int check_retries);
42
43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
44 {
45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
46         unsigned long flags;
47
48         spin_lock_irqsave(&t->tasklet_lock, flags);
49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
50                 smp_processor_id());
51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
53
54         tasklet_schedule(&t->tasklet);
55 }
56
57 /*
58  * Must not be called in parallel with scst_unregister_session_ex() for the
59  * same sess
60  */
61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
62                              const uint8_t *lun, int lun_len,
63                              const uint8_t *cdb, int cdb_len, int atomic)
64 {
65         struct scst_cmd *cmd;
66
67         TRACE_ENTRY();
68
69 #ifdef CONFIG_SCST_EXTRACHECKS
70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
71                 PRINT_CRIT_ERROR("%s",
72                         "New cmd while shutting down the session");
73                 sBUG();
74         }
75 #endif
76
77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
78         if (cmd == NULL)
79                 goto out;
80
81         cmd->sess = sess;
82         cmd->tgt = sess->tgt;
83         cmd->tgtt = sess->tgt->tgtt;
84
85         /*
86          * For both wrong lun and CDB defer the error reporting for
87          * scst_cmd_init_done()
88          */
89
90         cmd->lun = scst_unpack_lun(lun, lun_len);
91
92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
93                 memcpy(cmd->cdb, cdb, cdb_len);
94                 cmd->cdb_len = cdb_len;
95         }
96
97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
98         scst_sess_get(sess);
99
100 out:
101         TRACE_EXIT();
102         return cmd;
103 }
104 EXPORT_SYMBOL(scst_rx_cmd);
105
106 /*
107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
108  * this command should be stopped.
109  */
110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
111 {
112         int rc, res = 0;
113
114         TRACE_ENTRY();
115
116         /* See the comment in scst_do_job_init() */
117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
119                 goto out_redirect;
120         }
121         /*
122          * Memory barrier isn't necessary here, because CPU appears to
123          * be self-consistent and we don't care about the race, described
124          * in comment in scst_do_job_init().
125          */
126
127         rc = __scst_init_cmd(cmd);
128         if (unlikely(rc > 0))
129                 goto out_redirect;
130         else if (unlikely(rc != 0))
131                 goto out;
132
133         /* Small context optimization */
134         if (((*context == SCST_CONTEXT_TASKLET) ||
135              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
136              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
137               scst_cmd_is_expected_set(cmd)) {
138                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
139                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
140                                         &cmd->tgt_dev->tgt_dev_flags))
141                                 *context = SCST_CONTEXT_THREAD;
142                 } else {
143                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
144                                         &cmd->tgt_dev->tgt_dev_flags))
145                                 *context = SCST_CONTEXT_THREAD;
146                 }
147         }
148
149 out:
150         TRACE_EXIT_RES(res);
151         return res;
152
153 out_redirect:
154         if (cmd->preprocessing_only) {
155                 /*
156                  * Poor man solution for single threaded targets, where
157                  * blocking receiver at least sometimes means blocking all.
158                  */
159                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
160                 scst_set_busy(cmd);
161                 scst_set_cmd_abnormal_done_state(cmd);
162                 /* Keep initiator away from too many BUSY commands */
163                 msleep(50);
164         } else {
165                 unsigned long flags;
166                 spin_lock_irqsave(&scst_init_lock, flags);
167                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
168                         "%d)", cmd, atomic_read(&scst_cmd_count));
169                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
170                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
171                         scst_init_poll_cnt++;
172                 spin_unlock_irqrestore(&scst_init_lock, flags);
173                 wake_up(&scst_init_cmd_list_waitQ);
174                 res = -1;
175         }
176         goto out;
177 }
178
179 #ifdef CONFIG_SCST_MEASURE_LATENCY
180 static inline uint64_t scst_sec_to_nsec(time_t sec)
181 {
182         return (uint64_t)sec * 1000000000;
183 }
184 #endif
185
186 void scst_cmd_init_done(struct scst_cmd *cmd,
187         enum scst_exec_context pref_context)
188 {
189         unsigned long flags;
190         struct scst_session *sess = cmd->sess;
191         int rc;
192
193         TRACE_ENTRY();
194
195 #ifdef CONFIG_SCST_MEASURE_LATENCY
196         {
197                 struct timespec ts;
198                 getnstimeofday(&ts);
199                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
200                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
201                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
202                         ts.tv_nsec);
203         }
204 #endif
205
206         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
207         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
208                 "(cmd %p)", (long long unsigned int)cmd->tag,
209                 (long long unsigned int)cmd->lun, cmd->cdb_len,
210                 cmd->queue_type, cmd);
211         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
212                 cmd->cdb, cmd->cdb_len);
213
214 #ifdef CONFIG_SCST_EXTRACHECKS
215         if (unlikely((in_irq() || irqs_disabled())) &&
216             ((pref_context == SCST_CONTEXT_DIRECT) ||
217              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
218                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
219                         "SCST_CONTEXT_THREAD instead\n", pref_context,
220                         cmd->tgtt->name);
221                 pref_context = SCST_CONTEXT_THREAD;
222         }
223 #endif
224
225         atomic_inc(&sess->sess_cmd_count);
226
227         spin_lock_irqsave(&sess->sess_list_lock, flags);
228
229         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
230                 /*
231                  * We have to always keep command in the search list from the
232                  * very beginning, because otherwise it can be missed during
233                  * TM processing. This check is needed because there might be
234                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
235                  */
236                 if (cmd->search_cmd_list_entry.next == NULL)
237                         list_add_tail(&cmd->search_cmd_list_entry,
238                                 &sess->search_cmd_list);
239                 switch (sess->init_phase) {
240                 case SCST_SESS_IPH_SUCCESS:
241                         break;
242                 case SCST_SESS_IPH_INITING:
243                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
244                                   cmd);
245                         list_add_tail(&cmd->cmd_list_entry,
246                                 &sess->init_deferred_cmd_list);
247                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
248                         goto out;
249                 case SCST_SESS_IPH_FAILED:
250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
251                         scst_set_busy(cmd);
252                         scst_set_cmd_abnormal_done_state(cmd);
253                         goto active;
254                 default:
255                         sBUG();
256                 }
257         } else
258                 list_add_tail(&cmd->search_cmd_list_entry,
259                               &sess->search_cmd_list);
260
261         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
262
263         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
264                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
265                 scst_set_cmd_error(cmd,
266                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
267                 scst_set_cmd_abnormal_done_state(cmd);
268                 goto active;
269         }
270
271         if (unlikely(cmd->cdb_len == 0)) {
272                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
273                 scst_set_cmd_error(cmd,
274                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
275                 scst_set_cmd_abnormal_done_state(cmd);
276                 goto active;
277         }
278
279         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
280                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
281                 scst_set_cmd_error(cmd,
282                         SCST_LOAD_SENSE(scst_sense_invalid_message));
283                 scst_set_cmd_abnormal_done_state(cmd);
284                 goto active;
285         }
286
287         cmd->state = SCST_CMD_STATE_INIT;
288         /* cmd must be inited here to preserve the order */
289         rc = scst_init_cmd(cmd, &pref_context);
290         if (unlikely(rc < 0))
291                 goto out;
292
293 active:
294         /* Here cmd must not be in any cmd list, no locks */
295         switch (pref_context) {
296         case SCST_CONTEXT_TASKLET:
297                 scst_schedule_tasklet(cmd);
298                 break;
299
300         case SCST_CONTEXT_DIRECT:
301                 scst_process_active_cmd(cmd, false);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         case SCST_CONTEXT_DIRECT_ATOMIC:
306                 scst_process_active_cmd(cmd, true);
307                 /* For *NEED_THREAD wake_up() is already done */
308                 break;
309
310         default:
311                 PRINT_ERROR("Context %x is undefined, using the thread one",
312                         pref_context);
313                 /* go through */
314         case SCST_CONTEXT_THREAD:
315                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
316                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
317                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
318                         list_add(&cmd->cmd_list_entry,
319                                 &cmd->cmd_lists->active_cmd_list);
320                 else
321                         list_add_tail(&cmd->cmd_list_entry,
322                                 &cmd->cmd_lists->active_cmd_list);
323                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
324                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
325                 break;
326         }
327
328 out:
329         TRACE_EXIT();
330         return;
331 }
332 EXPORT_SYMBOL(scst_cmd_init_done);
333
334 static int scst_pre_parse(struct scst_cmd *cmd)
335 {
336         int res = SCST_CMD_STATE_RES_CONT_SAME;
337         struct scst_device *dev = cmd->dev;
338         int rc;
339
340         TRACE_ENTRY();
341
342         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
343              (!dev->has_own_order_mgmt &&
344               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
345                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
346
347         /*
348          * Expected transfer data supplied by the SCSI transport via the
349          * target driver are untrusted, so we prefer to fetch them from CDB.
350          * Additionally, not all transports support supplying the expected
351          * transfer data.
352          */
353
354         rc = scst_get_cdb_info(cmd);
355         if (unlikely(rc != 0)) {
356                 if (rc > 0) {
357                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
358                         goto out_xmit;
359                 }
360                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
361                         "Should you update scst_scsi_op_table?",
362                         cmd->cdb[0], dev->handler->name);
363                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
364 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
365                 if (scst_cmd_is_expected_set(cmd)) {
366                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
367                                 "direction %d, transfer_len %d",
368                                 cmd->expected_data_direction,
369                                 cmd->expected_transfer_len);
370                         cmd->data_direction = cmd->expected_data_direction;
371
372                         cmd->bufflen = cmd->expected_transfer_len;
373                         /* Restore (likely) lost CDB length */
374                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
375                         if (cmd->cdb_len == -1) {
376                                 PRINT_ERROR("Unable to get CDB length for "
377                                         "opcode 0x%02x. Returning INVALID "
378                                         "OPCODE", cmd->cdb[0]);
379                                 scst_set_cmd_error(cmd,
380                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
381                                 goto out_xmit;
382                         }
383                 } else {
384                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
385                              "target %s not supplied expected values",
386                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
387                         scst_set_cmd_error(cmd,
388                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                         goto out_xmit;
390                 }
391 #else
392                 scst_set_cmd_error(cmd,
393                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
394                 goto out_xmit;
395 #endif
396         } else {
397                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
398                         "(expected %d, set %s), transfer_len=%d (expected "
399                         "len %d), flags=%d", cmd->op_name, cmd,
400                         cmd->data_direction, cmd->expected_data_direction,
401                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
402                         cmd->bufflen, cmd->expected_transfer_len,
403                         cmd->op_flags);
404
405                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
406                         if (scst_cmd_is_expected_set(cmd)) {
407                                 /*
408                                  * Command data length can't be easily
409                                  * determined from the CDB. ToDo, all such
410                                  * commands processing should be fixed. Until
411                                  * it's done, get the length from the supplied
412                                  * expected value, but limit it to some
413                                  * reasonable value (15MB).
414                                  */
415                                 cmd->bufflen = min(cmd->expected_transfer_len,
416                                                         15*1024*1024);
417                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
418                         } else
419                                 cmd->bufflen = 0;
420                 }
421         }
422
423         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
424                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
425                             "(opcode 0x%02x)", cmd->cdb[0]);
426                 scst_set_cmd_error(cmd,
427                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
428                 goto out_xmit;
429         }
430
431         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
432                 PRINT_ERROR("Linked commands are not supported "
433                             "(opcode 0x%02x)", cmd->cdb[0]);
434                 scst_set_cmd_error(cmd,
435                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
436                 goto out_xmit;
437         }
438
439         cmd->state = SCST_CMD_STATE_DEV_PARSE;
440
441 out:
442         TRACE_EXIT_RES(res);
443         return res;
444
445 out_xmit:
446         scst_set_cmd_abnormal_done_state(cmd);
447         res = SCST_CMD_STATE_RES_CONT_SAME;
448         goto out;
449 }
450
451 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
452 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
453 {
454         bool res = false;
455
456         switch (cmd->cdb[0]) {
457         case TEST_UNIT_READY:
458                 /* Crazy VMware people sometimes do TUR with READ direction */
459                 res = true;
460                 break;
461         case VERIFY:
462         case VERIFY_6:
463         case VERIFY_12:
464         case VERIFY_16:
465                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
466                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
467                     (cmd->cdb[1] & BYTCHK) == 0)
468                         res = true;
469                 break;
470         }
471
472         return res;
473 }
474 #endif
475
476 static int scst_parse_cmd(struct scst_cmd *cmd)
477 {
478         int res = SCST_CMD_STATE_RES_CONT_SAME;
479         int state;
480         struct scst_device *dev = cmd->dev;
481         int orig_bufflen = cmd->bufflen;
482
483         TRACE_ENTRY();
484
485         if (likely(!scst_is_cmd_local(cmd))) {
486                 if (unlikely(!dev->handler->parse_atomic &&
487                              scst_cmd_atomic(cmd))) {
488                         /*
489                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
490                          * optimization.
491                          */
492                         TRACE_DBG("Dev handler %s parse() needs thread "
493                                 "context, rescheduling", dev->handler->name);
494                         res = SCST_CMD_STATE_RES_NEED_THREAD;
495                         goto out;
496                 }
497
498                 TRACE_DBG("Calling dev handler %s parse(%p)",
499                       dev->handler->name, cmd);
500                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
501                                 cmd->cdb, cmd->cdb_len);
502                 state = dev->handler->parse(cmd);
503                 /* Caution: cmd can be already dead here */
504                 TRACE_DBG("Dev handler %s parse() returned %d",
505                         dev->handler->name, state);
506
507                 switch (state) {
508                 case SCST_CMD_STATE_NEED_THREAD_CTX:
509                         TRACE_DBG("Dev handler %s parse() requested thread "
510                               "context, rescheduling", dev->handler->name);
511                         res = SCST_CMD_STATE_RES_NEED_THREAD;
512                         goto out;
513
514                 case SCST_CMD_STATE_STOP:
515                         TRACE_DBG("Dev handler %s parse() requested stop "
516                                 "processing", dev->handler->name);
517                         res = SCST_CMD_STATE_RES_CONT_NEXT;
518                         goto out;
519                 }
520
521                 if (state == SCST_CMD_STATE_DEFAULT)
522                         state = SCST_CMD_STATE_PREPARE_SPACE;
523         } else
524                 state = SCST_CMD_STATE_PREPARE_SPACE;
525
526         if (cmd->data_len == -1)
527                 cmd->data_len = cmd->bufflen;
528
529         if (cmd->bufflen == 0) {
530                 /*
531                  * According to SPC bufflen 0 for data transfer commands isn't
532                  * an error, so we need to fix the transfer direction.
533                  */
534                 cmd->data_direction = SCST_DATA_NONE;
535         }
536
537         if (cmd->dh_data_buf_alloced &&
538             unlikely((orig_bufflen > cmd->bufflen))) {
539                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
540                         "is less, than required (size %d)", cmd->bufflen,
541                         orig_bufflen);
542                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
543                 goto out_error;
544         }
545
546         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
547                 goto set_res;
548
549         if (unlikely((cmd->bufflen == 0) &&
550                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
551                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
552                         "(handler %s, target %s)", cmd->cdb[0],
553                         dev->handler->name, cmd->tgtt->name);
554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
555                 goto out_error;
556         }
557
558 #ifdef CONFIG_SCST_EXTRACHECKS
559         if ((cmd->bufflen != 0) &&
560             ((cmd->data_direction == SCST_DATA_NONE) ||
561              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
562                 PRINT_ERROR("Dev handler %s parse() returned "
563                         "invalid cmd data_direction %d, bufflen %d, state %d "
564                         "or sg %p (opcode 0x%x)", dev->handler->name,
565                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
566                         cmd->cdb[0]);
567                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
568                 goto out_error;
569         }
570 #endif
571
572         if (scst_cmd_is_expected_set(cmd)) {
573 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
574 #       ifdef CONFIG_SCST_EXTRACHECKS
575                 if ((cmd->data_direction != cmd->expected_data_direction) ||
576                     (cmd->bufflen != cmd->expected_transfer_len)) {
577                         PRINT_WARNING("Expected values don't match decoded "
578                                 "ones: data_direction %d, "
579                                 "expected_data_direction %d, "
580                                 "bufflen %d, expected_transfer_len %d",
581                                 cmd->data_direction,
582                                 cmd->expected_data_direction,
583                                 cmd->bufflen, cmd->expected_transfer_len);
584                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
585                 }
586 #       endif
587                 cmd->data_direction = cmd->expected_data_direction;
588                 cmd->bufflen = cmd->expected_transfer_len;
589 #else
590                 if (unlikely(cmd->data_direction !=
591                                 cmd->expected_data_direction)) {
592                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
593                              (cmd->bufflen != 0)) &&
594                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
595                                 PRINT_ERROR("Expected data direction %d for "
596                                         "opcode 0x%02x (handler %s, target %s) "
597                                         "doesn't match "
598                                         "decoded value %d",
599                                         cmd->expected_data_direction,
600                                         cmd->cdb[0], dev->handler->name,
601                                         cmd->tgtt->name, cmd->data_direction);
602                                 PRINT_BUFFER("Failed CDB",
603                                         cmd->cdb, cmd->cdb_len);
604                                 scst_set_cmd_error(cmd,
605                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
606                                 goto out_dev_done;
607                         }
608                 }
609                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
610                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
611                                 "transfer length %d for opcode 0x%02x "
612                                 "(handler %s, target %s) doesn't match "
613                                 "decoded value %d. Faulty initiator "
614                                 "(e.g. VMware is known to be such) or "
615                                 "scst_scsi_op_table should be updated?",
616                                 cmd->expected_transfer_len, cmd->cdb[0],
617                                 dev->handler->name, cmd->tgtt->name,
618                                 cmd->bufflen);
619                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
620                                 cmd->cdb, cmd->cdb_len);
621                 }
622 #endif
623         }
624
625         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
626                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
627                         "target %s", cmd->cdb[0], dev->handler->name,
628                         cmd->tgtt->name);
629                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
630                 goto out_error;
631         }
632
633 set_res:
634         switch (state) {
635         case SCST_CMD_STATE_PREPARE_SPACE:
636         case SCST_CMD_STATE_PRE_PARSE:
637         case SCST_CMD_STATE_DEV_PARSE:
638         case SCST_CMD_STATE_RDY_TO_XFER:
639         case SCST_CMD_STATE_TGT_PRE_EXEC:
640         case SCST_CMD_STATE_SEND_FOR_EXEC:
641         case SCST_CMD_STATE_LOCAL_EXEC:
642         case SCST_CMD_STATE_REAL_EXEC:
643         case SCST_CMD_STATE_PRE_DEV_DONE:
644         case SCST_CMD_STATE_DEV_DONE:
645         case SCST_CMD_STATE_PRE_XMIT_RESP:
646         case SCST_CMD_STATE_XMIT_RESP:
647         case SCST_CMD_STATE_FINISHED:
648         case SCST_CMD_STATE_FINISHED_INTERNAL:
649                 cmd->state = state;
650                 res = SCST_CMD_STATE_RES_CONT_SAME;
651                 break;
652
653         default:
654                 if (state >= 0) {
655                         PRINT_ERROR("Dev handler %s parse() returned "
656                              "invalid cmd state %d (opcode %d)",
657                              dev->handler->name, state, cmd->cdb[0]);
658                 } else {
659                         PRINT_ERROR("Dev handler %s parse() returned "
660                                 "error %d (opcode %d)", dev->handler->name,
661                                 state, cmd->cdb[0]);
662                 }
663                 goto out_error;
664         }
665
666         if (cmd->resp_data_len == -1) {
667                 if (cmd->data_direction == SCST_DATA_READ)
668                         cmd->resp_data_len = cmd->bufflen;
669                 else
670                          cmd->resp_data_len = 0;
671         }
672
673 out:
674         TRACE_EXIT_HRES(res);
675         return res;
676
677 out_error:
678         /* dev_done() will be called as part of the regular cmd's finish */
679         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
680
681 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
682 out_dev_done:
683 #endif
684         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
685         res = SCST_CMD_STATE_RES_CONT_SAME;
686         goto out;
687 }
688
689 static int scst_prepare_space(struct scst_cmd *cmd)
690 {
691         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
692
693         TRACE_ENTRY();
694
695         if (cmd->data_direction == SCST_DATA_NONE)
696                 goto prep_done;
697
698         if (cmd->tgt_need_alloc_data_buf) {
699                 int orig_bufflen = cmd->bufflen;
700
701                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
702                         cmd);
703
704                 r = cmd->tgtt->alloc_data_buf(cmd);
705                 if (r > 0)
706                         goto alloc;
707                 else if (r == 0) {
708                         if (unlikely(cmd->bufflen == 0)) {
709                                 /* See comment in scst_alloc_space() */
710                                 if (cmd->sg == NULL)
711                                         goto alloc;
712                         }
713
714                         cmd->tgt_data_buf_alloced = 1;
715
716                         if (unlikely(orig_bufflen < cmd->bufflen)) {
717                                 PRINT_ERROR("Target driver allocated data "
718                                         "buffer (size %d), is less, than "
719                                         "required (size %d)", orig_bufflen,
720                                         cmd->bufflen);
721                                 goto out_error;
722                         }
723                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
724                 } else
725                         goto check;
726         }
727
728 alloc:
729         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
730                 r = scst_alloc_space(cmd);
731                 cmd->tgt_sg = cmd->sg;
732                 cmd->tgt_sg_cnt = cmd->sg_cnt;
733         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
734                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
735                 cmd->tgt_sg = cmd->sg;
736                 cmd->tgt_sg_cnt = cmd->sg_cnt;
737                 r = 0;
738         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
739                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
740                 cmd->sg = cmd->tgt_sg;
741                 cmd->sg_cnt = cmd->tgt_sg_cnt;
742                 r = 0;
743         } else {
744                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
745                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
746                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
747                 r = 0;
748         }
749
750 check:
751         if (r != 0) {
752                 if (scst_cmd_atomic(cmd)) {
753                         TRACE_MEM("%s", "Atomic memory allocation failed, "
754                               "rescheduling to the thread");
755                         res = SCST_CMD_STATE_RES_NEED_THREAD;
756                         goto out;
757                 } else
758                         goto out_no_space;
759         }
760
761 prep_done:
762         if (cmd->preprocessing_only) {
763                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
764                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
765                                 "cmd %p", cmd);
766                         scst_set_cmd_abnormal_done_state(cmd);
767                         res = SCST_CMD_STATE_RES_CONT_SAME;
768                         goto out;
769                 }
770
771                 res = SCST_CMD_STATE_RES_CONT_NEXT;
772                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
773
774                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
775                 cmd->tgtt->preprocessing_done(cmd);
776                 TRACE_DBG("%s", "preprocessing_done() returned");
777                 goto out;
778
779         }
780
781         switch (cmd->data_direction) {
782         case SCST_DATA_WRITE:
783                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
784                 break;
785
786         default:
787                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
788                 break;
789         }
790
791 out:
792         TRACE_EXIT_HRES(res);
793         return res;
794
795 out_no_space:
796         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
797                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
798         scst_set_busy(cmd);
799         scst_set_cmd_abnormal_done_state(cmd);
800         res = SCST_CMD_STATE_RES_CONT_SAME;
801         goto out;
802
803 out_error:
804         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
805         scst_set_cmd_abnormal_done_state(cmd);
806         res = SCST_CMD_STATE_RES_CONT_SAME;
807         goto out;
808 }
809
810 void scst_restart_cmd(struct scst_cmd *cmd, int status,
811         enum scst_exec_context pref_context)
812 {
813         TRACE_ENTRY();
814
815         TRACE_DBG("Preferred context: %d", pref_context);
816         TRACE_DBG("tag=%llu, status=%#x",
817                   (long long unsigned int)scst_cmd_get_tag(cmd),
818                   status);
819
820 #ifdef CONFIG_SCST_EXTRACHECKS
821         if ((in_irq() || irqs_disabled()) &&
822             ((pref_context == SCST_CONTEXT_DIRECT) ||
823              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
824                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
825                         "SCST_CONTEXT_THREAD instead\n", pref_context,
826                         cmd->tgtt->name);
827                 pref_context = SCST_CONTEXT_THREAD;
828         }
829 #endif
830
831         switch (status) {
832         case SCST_PREPROCESS_STATUS_SUCCESS:
833                 switch (cmd->data_direction) {
834                 case SCST_DATA_WRITE:
835                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
836                         break;
837                 default:
838                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
839                         break;
840                 }
841                 if (cmd->set_sn_on_restart_cmd)
842                         scst_cmd_set_sn(cmd);
843                 /* Small context optimization */
844                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
845                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
846                     ((pref_context == SCST_CONTEXT_SAME) &&
847                      scst_cmd_atomic(cmd))) {
848                         if (cmd->data_direction == SCST_DATA_WRITE) {
849                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
850                                                 &cmd->tgt_dev->tgt_dev_flags))
851                                         pref_context = SCST_CONTEXT_THREAD;
852                         } else {
853                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
854                                                 &cmd->tgt_dev->tgt_dev_flags))
855                                         pref_context = SCST_CONTEXT_THREAD;
856                         }
857                 }
858                 break;
859
860         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
861                 scst_set_cmd_abnormal_done_state(cmd);
862                 break;
863
864         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
865                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
866                 /* go through */
867         case SCST_PREPROCESS_STATUS_ERROR:
868                 scst_set_cmd_error(cmd,
869                            SCST_LOAD_SENSE(scst_sense_hardw_error));
870                 scst_set_cmd_abnormal_done_state(cmd);
871                 break;
872
873         default:
874                 PRINT_ERROR("%s() received unknown status %x", __func__,
875                         status);
876                 scst_set_cmd_abnormal_done_state(cmd);
877                 break;
878         }
879
880         scst_proccess_redirect_cmd(cmd, pref_context, 1);
881
882         TRACE_EXIT();
883         return;
884 }
885 EXPORT_SYMBOL(scst_restart_cmd);
886
887 /* No locks */
888 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
889 {
890         struct scst_tgt *tgt = cmd->sess->tgt;
891         int res = 0;
892         unsigned long flags;
893
894         TRACE_ENTRY();
895
896         spin_lock_irqsave(&tgt->tgt_lock, flags);
897         tgt->retry_cmds++;
898         /*
899          * Memory barrier is needed here, because we need the exact order
900          * between the read and write between retry_cmds and finished_cmds to
901          * not miss the case when a command finished while we queuing it for
902          * retry after the finished_cmds check.
903          */
904         smp_mb();
905         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
906               tgt->retry_cmds);
907         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
908                 /* At least one cmd finished, so try again */
909                 tgt->retry_cmds--;
910                 TRACE_RETRY("Some command(s) finished, direct retry "
911                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
912                       "retry_cmds=%d)", finished_cmds,
913                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
914                 res = -1;
915                 goto out_unlock_tgt;
916         }
917
918         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
919         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
920
921         if (!tgt->retry_timer_active) {
922                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
923                 add_timer(&tgt->retry_timer);
924                 tgt->retry_timer_active = 1;
925         }
926
927 out_unlock_tgt:
928         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
929
930         TRACE_EXIT_RES(res);
931         return res;
932 }
933
934 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
935 {
936         int res, rc;
937
938         TRACE_ENTRY();
939
940         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
941                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
942                 goto out_dev_done;
943         }
944
945         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
946                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
947                 res = SCST_CMD_STATE_RES_CONT_SAME;
948                 goto out;
949         }
950
951         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
952                 /*
953                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
954                  * optimization.
955                  */
956                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
957                               "context, rescheduling", cmd->tgtt->name);
958                 res = SCST_CMD_STATE_RES_NEED_THREAD;
959                 goto out;
960         }
961
962         while (1) {
963                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
964
965                 res = SCST_CMD_STATE_RES_CONT_NEXT;
966                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
967
968                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
969 #ifdef CONFIG_SCST_DEBUG_RETRY
970                 if (((scst_random() % 100) == 75))
971                         rc = SCST_TGT_RES_QUEUE_FULL;
972                 else
973 #endif
974                         rc = cmd->tgtt->rdy_to_xfer(cmd);
975                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
976
977                 if (likely(rc == SCST_TGT_RES_SUCCESS))
978                         goto out;
979
980                 /* Restore the previous state */
981                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
982
983                 switch (rc) {
984                 case SCST_TGT_RES_QUEUE_FULL:
985                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
986                                 break;
987                         else
988                                 continue;
989
990                 case SCST_TGT_RES_NEED_THREAD_CTX:
991                         TRACE_DBG("Target driver %s "
992                               "rdy_to_xfer() requested thread "
993                               "context, rescheduling", cmd->tgtt->name);
994                         res = SCST_CMD_STATE_RES_NEED_THREAD;
995                         break;
996
997                 default:
998                         goto out_error_rc;
999                 }
1000                 break;
1001         }
1002
1003 out:
1004         TRACE_EXIT_HRES(res);
1005         return res;
1006
1007 out_error_rc:
1008         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1009                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1010                      "fatal error", cmd->tgtt->name);
1011         } else {
1012                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1013                             "value %d", cmd->tgtt->name, rc);
1014         }
1015         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1016
1017 out_dev_done:
1018         scst_set_cmd_abnormal_done_state(cmd);
1019         res = SCST_CMD_STATE_RES_CONT_SAME;
1020         goto out;
1021 }
1022
1023 /* No locks, but might be in IRQ */
1024 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1025         enum scst_exec_context context, int check_retries)
1026 {
1027         unsigned long flags;
1028
1029         TRACE_ENTRY();
1030
1031         TRACE_DBG("Context: %x", context);
1032
1033         if (context == SCST_CONTEXT_SAME)
1034                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1035                                                  SCST_CONTEXT_DIRECT;
1036
1037         switch (context) {
1038         case SCST_CONTEXT_DIRECT_ATOMIC:
1039                 scst_process_active_cmd(cmd, true);
1040                 break;
1041
1042         case SCST_CONTEXT_DIRECT:
1043                 if (check_retries)
1044                         scst_check_retries(cmd->tgt);
1045                 scst_process_active_cmd(cmd, false);
1046                 break;
1047
1048         default:
1049                 PRINT_ERROR("Context %x is unknown, using the thread one",
1050                             context);
1051                 /* go through */
1052         case SCST_CONTEXT_THREAD:
1053                 if (check_retries)
1054                         scst_check_retries(cmd->tgt);
1055                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1056                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1057                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1058                         list_add(&cmd->cmd_list_entry,
1059                                 &cmd->cmd_lists->active_cmd_list);
1060                 else
1061                         list_add_tail(&cmd->cmd_list_entry,
1062                                 &cmd->cmd_lists->active_cmd_list);
1063                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1064                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1065                 break;
1066
1067         case SCST_CONTEXT_TASKLET:
1068                 if (check_retries)
1069                         scst_check_retries(cmd->tgt);
1070                 scst_schedule_tasklet(cmd);
1071                 break;
1072         }
1073
1074         TRACE_EXIT();
1075         return;
1076 }
1077
1078 void scst_rx_data(struct scst_cmd *cmd, int status,
1079         enum scst_exec_context pref_context)
1080 {
1081         TRACE_ENTRY();
1082
1083         TRACE_DBG("Preferred context: %d", pref_context);
1084         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1085
1086 #ifdef CONFIG_SCST_EXTRACHECKS
1087         if ((in_irq() || irqs_disabled()) &&
1088             ((pref_context == SCST_CONTEXT_DIRECT) ||
1089              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1090                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1091                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1092                         cmd->tgtt->name);
1093                 pref_context = SCST_CONTEXT_THREAD;
1094         }
1095 #endif
1096
1097         switch (status) {
1098         case SCST_RX_STATUS_SUCCESS:
1099 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1100                 if (cmd->tgt_sg) {
1101                         int i;
1102                         struct scatterlist *sg = cmd->tgt_sg;
1103                         TRACE_RECV_BOT("RX data for cmd %p "
1104                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1105                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1106                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1107                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1108                                         sg_virt(&sg[i]), sg[i].length);
1109                         }
1110                 }
1111 #endif
1112                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1113                 /* Small context optimization */
1114                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1115                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1116                     ((pref_context == SCST_CONTEXT_SAME) &&
1117                      scst_cmd_atomic(cmd))) {
1118                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1119                                         &cmd->tgt_dev->tgt_dev_flags))
1120                                 pref_context = SCST_CONTEXT_THREAD;
1121                 }
1122                 break;
1123
1124         case SCST_RX_STATUS_ERROR_SENSE_SET:
1125                 scst_set_cmd_abnormal_done_state(cmd);
1126                 break;
1127
1128         case SCST_RX_STATUS_ERROR_FATAL:
1129                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1130                 /* go through */
1131         case SCST_RX_STATUS_ERROR:
1132                 scst_set_cmd_error(cmd,
1133                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1134                 scst_set_cmd_abnormal_done_state(cmd);
1135                 break;
1136
1137         default:
1138                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1139                         status);
1140                 scst_set_cmd_abnormal_done_state(cmd);
1141                 break;
1142         }
1143
1144         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1145
1146         TRACE_EXIT();
1147         return;
1148 }
1149 EXPORT_SYMBOL(scst_rx_data);
1150
1151 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1152 {
1153         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1154
1155         TRACE_ENTRY();
1156
1157         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1158
1159         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1160                 goto out;
1161
1162         TRACE_DBG("Calling pre_exec(%p)", cmd);
1163         rc = cmd->tgtt->pre_exec(cmd);
1164         TRACE_DBG("pre_exec() returned %d", rc);
1165
1166         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1167                 switch (rc) {
1168                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1169                         scst_set_cmd_abnormal_done_state(cmd);
1170                         break;
1171                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1172                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1173                         /* go through */
1174                 case SCST_PREPROCESS_STATUS_ERROR:
1175                         scst_set_cmd_error(cmd,
1176                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1177                         scst_set_cmd_abnormal_done_state(cmd);
1178                         break;
1179                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1180                         TRACE_DBG("Target driver's %s pre_exec() requested "
1181                                 "thread context, rescheduling",
1182                                 cmd->tgtt->name);
1183                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1184                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1185                         break;
1186                 default:
1187                         sBUG();
1188                         break;
1189                 }
1190         }
1191
1192 out:
1193         TRACE_EXIT_RES(res);
1194         return res;
1195 }
1196
1197 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1198         const uint8_t *rq_sense, int rq_sense_len, int resid)
1199 {
1200         TRACE_ENTRY();
1201
1202 #ifdef CONFIG_SCST_MEASURE_LATENCY
1203         {
1204                 struct timespec ts;
1205                 getnstimeofday(&ts);
1206                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1207                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1208                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1209                         ts.tv_sec, ts.tv_nsec);
1210         }
1211 #endif
1212
1213         cmd->status = result & 0xff;
1214         cmd->msg_status = msg_byte(result);
1215         cmd->host_status = host_byte(result);
1216         cmd->driver_status = driver_byte(result);
1217         if (unlikely(resid != 0)) {
1218 #ifdef CONFIG_SCST_EXTRACHECKS
1219                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1220                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1221                                 "op %x)", resid, cmd->resp_data_len,
1222                                 cmd->cdb[0]);
1223                 } else
1224 #endif
1225                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1226         }
1227
1228         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1229                 /* We might have double reset UA here */
1230                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1231                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1232
1233                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1234         }
1235
1236         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1237               "cmd->msg_status=%x, cmd->host_status=%x, "
1238               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1239               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1240
1241         cmd->completed = 1;
1242
1243         TRACE_EXIT();
1244         return;
1245 }
1246
1247 /* For small context optimization */
1248 static inline enum scst_exec_context scst_optimize_post_exec_context(
1249         struct scst_cmd *cmd, enum scst_exec_context context)
1250 {
1251         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1252             (context == SCST_CONTEXT_TASKLET) ||
1253             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1254                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1255                                 &cmd->tgt_dev->tgt_dev_flags))
1256                         context = SCST_CONTEXT_THREAD;
1257         }
1258         return context;
1259 }
1260
1261 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1262 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1263                                             struct scsi_request **req)
1264 {
1265         struct scst_cmd *cmd = NULL;
1266
1267         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1268                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1269
1270         if (cmd == NULL) {
1271                 PRINT_ERROR("%s", "Request with NULL cmd");
1272                 if (*req)
1273                         scsi_release_request(*req);
1274         }
1275
1276         return cmd;
1277 }
1278
1279 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1280 {
1281         struct scsi_request *req = NULL;
1282         struct scst_cmd *cmd;
1283
1284         TRACE_ENTRY();
1285
1286         cmd = scst_get_cmd(scsi_cmd, &req);
1287         if (cmd == NULL)
1288                 goto out;
1289
1290         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1291                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1292
1293         /* Clear out request structure */
1294         req->sr_use_sg = 0;
1295         req->sr_sglist_len = 0;
1296         req->sr_bufflen = 0;
1297         req->sr_buffer = NULL;
1298         req->sr_underflow = 0;
1299         req->sr_request->rq_disk = NULL; /* disown request blk */
1300
1301         scst_release_request(cmd);
1302
1303         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1304
1305         scst_proccess_redirect_cmd(cmd,
1306                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1307                                                 0);
1308
1309 out:
1310         TRACE_EXIT();
1311         return;
1312 }
1313 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1314 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1315 {
1316         struct scst_cmd *cmd;
1317
1318         TRACE_ENTRY();
1319
1320         cmd = (struct scst_cmd *)data;
1321         if (cmd == NULL)
1322                 goto out;
1323
1324         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1325
1326         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1327
1328         scst_proccess_redirect_cmd(cmd,
1329                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1330                                                 0);
1331
1332 out:
1333         TRACE_EXIT();
1334         return;
1335 }
1336 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1337
1338 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1339         enum scst_exec_context pref_context)
1340 {
1341         TRACE_ENTRY();
1342
1343 #ifdef CONFIG_SCST_MEASURE_LATENCY
1344         {
1345                 struct timespec ts;
1346                 getnstimeofday(&ts);
1347                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1348                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1349                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1350                         ts.tv_sec, ts.tv_nsec);
1351         }
1352 #endif
1353
1354         if (next_state == SCST_CMD_STATE_DEFAULT)
1355                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1356
1357 #if defined(CONFIG_SCST_DEBUG)
1358         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1359                 if (cmd->sg) {
1360                         int i;
1361                         struct scatterlist *sg = cmd->sg;
1362                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1363                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1364                         for (i = 0; i < cmd->sg_cnt; ++i) {
1365                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1366                                         "Exec'd sg", sg_virt(&sg[i]),
1367                                         sg[i].length);
1368                         }
1369                 }
1370         }
1371 #endif
1372
1373         cmd->state = next_state;
1374
1375 #ifdef CONFIG_SCST_EXTRACHECKS
1376         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1377             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1378             (next_state != SCST_CMD_STATE_FINISHED) &&
1379             (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
1380                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1381                         __func__, next_state, cmd->cdb[0]);
1382                 scst_set_cmd_error(cmd,
1383                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1384                 scst_set_cmd_abnormal_done_state(cmd);
1385         }
1386 #endif
1387         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1388         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1389
1390         TRACE_EXIT();
1391         return;
1392 }
1393
1394 static int scst_report_luns_local(struct scst_cmd *cmd)
1395 {
1396         int rc;
1397         int dev_cnt = 0;
1398         int buffer_size;
1399         int i;
1400         struct scst_tgt_dev *tgt_dev = NULL;
1401         uint8_t *buffer;
1402         int offs, overflow = 0;
1403
1404         TRACE_ENTRY();
1405
1406         rc = scst_check_local_events(cmd);
1407         if (unlikely(rc != 0))
1408                 goto out_done;
1409
1410         cmd->status = 0;
1411         cmd->msg_status = 0;
1412         cmd->host_status = DID_OK;
1413         cmd->driver_status = 0;
1414
1415         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1416                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1417                         "LUNS command", cmd->cdb[2]);
1418                 goto out_err;
1419         }
1420
1421         buffer_size = scst_get_buf_first(cmd, &buffer);
1422         if (unlikely(buffer_size == 0))
1423                 goto out_compl;
1424         else if (unlikely(buffer_size < 0))
1425                 goto out_err;
1426
1427         if (buffer_size < 16)
1428                 goto out_put_err;
1429
1430         memset(buffer, 0, buffer_size);
1431         offs = 8;
1432
1433         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1434         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1435                 struct list_head *sess_tgt_dev_list_head =
1436                         &cmd->sess->sess_tgt_dev_list_hash[i];
1437                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1438                                 sess_tgt_dev_list_entry) {
1439                         if (!overflow) {
1440                                 if (offs >= buffer_size) {
1441                                         scst_put_buf(cmd, buffer);
1442                                         buffer_size = scst_get_buf_next(cmd,
1443                                                                        &buffer);
1444                                         if (buffer_size > 0) {
1445                                                 memset(buffer, 0, buffer_size);
1446                                                 offs = 0;
1447                                         } else {
1448                                                 overflow = 1;
1449                                                 goto inc_dev_cnt;
1450                                         }
1451                                 }
1452                                 if ((buffer_size - offs) < 8) {
1453                                         PRINT_ERROR("Buffer allocated for "
1454                                                 "REPORT LUNS command doesn't "
1455                                                 "allow to fit 8 byte entry "
1456                                                 "(buffer_size=%d)",
1457                                                 buffer_size);
1458                                         goto out_put_hw_err;
1459                                 }
1460                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1461                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1462                                 offs += 8;
1463                         }
1464 inc_dev_cnt:
1465                         dev_cnt++;
1466                 }
1467         }
1468         if (!overflow)
1469                 scst_put_buf(cmd, buffer);
1470
1471         /* Set the response header */
1472         buffer_size = scst_get_buf_first(cmd, &buffer);
1473         if (unlikely(buffer_size == 0))
1474                 goto out_compl;
1475         else if (unlikely(buffer_size < 0))
1476                 goto out_err;
1477
1478         dev_cnt *= 8;
1479         buffer[0] = (dev_cnt >> 24) & 0xff;
1480         buffer[1] = (dev_cnt >> 16) & 0xff;
1481         buffer[2] = (dev_cnt >> 8) & 0xff;
1482         buffer[3] = dev_cnt & 0xff;
1483
1484         scst_put_buf(cmd, buffer);
1485
1486         dev_cnt += 8;
1487         if (dev_cnt < cmd->resp_data_len)
1488                 scst_set_resp_data_len(cmd, dev_cnt);
1489
1490 out_compl:
1491         cmd->completed = 1;
1492
1493 out_done:
1494         /* Report the result */
1495         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1496
1497         TRACE_EXIT();
1498         return SCST_EXEC_COMPLETED;
1499
1500 out_put_err:
1501         scst_put_buf(cmd, buffer);
1502
1503 out_err:
1504         scst_set_cmd_error(cmd,
1505                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1506         goto out_compl;
1507
1508 out_put_hw_err:
1509         scst_put_buf(cmd, buffer);
1510         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1511         goto out_compl;
1512 }
1513
1514 static int scst_pre_select(struct scst_cmd *cmd)
1515 {
1516         int res = SCST_EXEC_NOT_COMPLETED;
1517
1518         TRACE_ENTRY();
1519
1520         if (scst_cmd_atomic(cmd)) {
1521                 res = SCST_EXEC_NEED_THREAD;
1522                 goto out;
1523         }
1524
1525         scst_block_dev_cmd(cmd, 1);
1526
1527         /* Check for local events will be done when cmd will be executed */
1528
1529 out:
1530         TRACE_EXIT_RES(res);
1531         return res;
1532 }
1533
1534 static int scst_reserve_local(struct scst_cmd *cmd)
1535 {
1536         int res = SCST_EXEC_NOT_COMPLETED, rc;
1537         struct scst_device *dev;
1538         struct scst_tgt_dev *tgt_dev_tmp;
1539
1540         TRACE_ENTRY();
1541
1542         if (scst_cmd_atomic(cmd)) {
1543                 res = SCST_EXEC_NEED_THREAD;
1544                 goto out;
1545         }
1546
1547         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1548                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1549                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1550                 scst_set_cmd_error(cmd,
1551                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1552                 goto out_done;
1553         }
1554
1555         dev = cmd->dev;
1556
1557         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1558                 scst_block_dev_cmd(cmd, 1);
1559
1560         rc = scst_check_local_events(cmd);
1561         if (unlikely(rc != 0))
1562                 goto out_done;
1563
1564         spin_lock_bh(&dev->dev_lock);
1565
1566         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1567                 spin_unlock_bh(&dev->dev_lock);
1568                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1569                 goto out_done;
1570         }
1571
1572         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1573                             dev_tgt_dev_list_entry) {
1574                 if (cmd->tgt_dev != tgt_dev_tmp)
1575                         set_bit(SCST_TGT_DEV_RESERVED,
1576                                 &tgt_dev_tmp->tgt_dev_flags);
1577         }
1578         dev->dev_reserved = 1;
1579
1580         spin_unlock_bh(&dev->dev_lock);
1581
1582 out:
1583         TRACE_EXIT_RES(res);
1584         return res;
1585
1586 out_done:
1587         /* Report the result */
1588         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1589         res = SCST_EXEC_COMPLETED;
1590         goto out;
1591 }
1592
1593 static int scst_release_local(struct scst_cmd *cmd)
1594 {
1595         int res = SCST_EXEC_NOT_COMPLETED, rc;
1596         struct scst_tgt_dev *tgt_dev_tmp;
1597         struct scst_device *dev;
1598
1599         TRACE_ENTRY();
1600
1601         if (scst_cmd_atomic(cmd)) {
1602                 res = SCST_EXEC_NEED_THREAD;
1603                 goto out;
1604         }
1605
1606         dev = cmd->dev;
1607
1608         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1609                 scst_block_dev_cmd(cmd, 1);
1610
1611         rc = scst_check_local_events(cmd);
1612         if (unlikely(rc != 0))
1613                 goto out_done;
1614
1615         spin_lock_bh(&dev->dev_lock);
1616
1617         /*
1618          * The device could be RELEASED behind us, if RESERVING session
1619          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1620          * matter, so use lock and no retest for DEV_RESERVED bits again
1621          */
1622         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1623                 res = SCST_EXEC_COMPLETED;
1624                 cmd->status = 0;
1625                 cmd->msg_status = 0;
1626                 cmd->host_status = DID_OK;
1627                 cmd->driver_status = 0;
1628                 cmd->completed = 1;
1629         } else {
1630                 list_for_each_entry(tgt_dev_tmp,
1631                                     &dev->dev_tgt_dev_list,
1632                                     dev_tgt_dev_list_entry) {
1633                         clear_bit(SCST_TGT_DEV_RESERVED,
1634                                 &tgt_dev_tmp->tgt_dev_flags);
1635                 }
1636                 dev->dev_reserved = 0;
1637         }
1638
1639         spin_unlock_bh(&dev->dev_lock);
1640
1641         if (res == SCST_EXEC_COMPLETED)
1642                 goto out_done;
1643
1644 out:
1645         TRACE_EXIT_RES(res);
1646         return res;
1647
1648 out_done:
1649         res = SCST_EXEC_COMPLETED;
1650         /* Report the result */
1651         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1652         goto out;
1653 }
1654
1655 /* No locks, no IRQ or IRQ-safe context allowed */
1656 int scst_check_local_events(struct scst_cmd *cmd)
1657 {
1658         int res, rc;
1659         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1660         struct scst_device *dev = cmd->dev;
1661
1662         TRACE_ENTRY();
1663
1664         /*
1665          * There's no race here, because we need to trace commands sent
1666          * *after* dev_double_ua_possible flag was set.
1667          */
1668         if (unlikely(dev->dev_double_ua_possible))
1669                 cmd->double_ua_possible = 1;
1670
1671         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1672                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1673                 goto out_uncomplete;
1674         }
1675
1676         /* Reserve check before Unit Attention */
1677         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1678                               &tgt_dev->tgt_dev_flags))) {
1679                 if (cmd->cdb[0] != INQUIRY &&
1680                     cmd->cdb[0] != REPORT_LUNS &&
1681                     cmd->cdb[0] != RELEASE &&
1682                     cmd->cdb[0] != RELEASE_10 &&
1683                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1684                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1685                      (cmd->cdb[4] & 3)) &&
1686                     cmd->cdb[0] != LOG_SENSE &&
1687                     cmd->cdb[0] != REQUEST_SENSE) {
1688                         scst_set_cmd_error_status(cmd,
1689                                 SAM_STAT_RESERVATION_CONFLICT);
1690                         goto out_complete;
1691                 }
1692         }
1693
1694         /* If we had internal bus reset, set the command error unit attention */
1695         if ((dev->scsi_dev != NULL) &&
1696             unlikely(dev->scsi_dev->was_reset)) {
1697                 if (scst_is_ua_command(cmd)) {
1698                         int done = 0;
1699                         /*
1700                          * Prevent more than 1 cmd to be triggered by
1701                          * was_reset.
1702                          */
1703                         spin_lock_bh(&dev->dev_lock);
1704                         if (dev->scsi_dev->was_reset) {
1705                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1706                                 scst_set_cmd_error(cmd,
1707                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1708                                 /*
1709                                  * It looks like it is safe to clear was_reset
1710                                  * here.
1711                                  */
1712                                 dev->scsi_dev->was_reset = 0;
1713                                 done = 1;
1714                         }
1715                         spin_unlock_bh(&dev->dev_lock);
1716
1717                         if (done)
1718                                 goto out_complete;
1719                 }
1720         }
1721
1722         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1723                         &cmd->tgt_dev->tgt_dev_flags))) {
1724                 if (scst_is_ua_command(cmd)) {
1725                         rc = scst_set_pending_UA(cmd);
1726                         if (rc == 0)
1727                                 goto out_complete;
1728                 }
1729         }
1730
1731         res = 0;
1732
1733 out:
1734         TRACE_EXIT_RES(res);
1735         return res;
1736
1737 out_complete:
1738         res = 1;
1739         sBUG_ON(!cmd->completed);
1740         goto out;
1741
1742 out_uncomplete:
1743         res = -1;
1744         goto out;
1745 }
1746 EXPORT_SYMBOL(scst_check_local_events);
1747
1748 /* No locks */
1749 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1750 {
1751         if (slot == NULL)
1752                 goto inc;
1753
1754         /* Optimized for lockless fast path */
1755
1756         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1757                 atomic_read(slot));
1758
1759         if (!atomic_dec_and_test(slot))
1760                 goto out;
1761
1762         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1763                 tgt_dev->num_free_sn_slots);
1764         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1765                 spin_lock_irq(&tgt_dev->sn_lock);
1766                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1767                         if (tgt_dev->num_free_sn_slots < 0)
1768                                 tgt_dev->cur_sn_slot = slot;
1769                         /*
1770                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1771                          */
1772                         smp_mb();
1773                         tgt_dev->num_free_sn_slots++;
1774                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1775                                 tgt_dev->num_free_sn_slots);
1776
1777                 }
1778                 spin_unlock_irq(&tgt_dev->sn_lock);
1779         }
1780
1781 inc:
1782         /*
1783          * No protection of expected_sn is needed, because only one thread
1784          * at time can be here (serialized by sn). Also it is supposed that
1785          * there could not be half-incremented halves.
1786          */
1787         tgt_dev->expected_sn++;
1788         /*
1789          * Write must be before def_cmd_count read to be in sync. with
1790          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1791          */
1792         smp_mb();
1793         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1794
1795 out:
1796         return;
1797 }
1798
1799 /* No locks */
1800 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1801         bool make_active)
1802 {
1803         /* For HQ commands SN is not set */
1804         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1805                                cmd->sn_set && !cmd->retry;
1806         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1807         struct scst_cmd *res;
1808
1809         TRACE_ENTRY();
1810
1811         if (inc_expected_sn)
1812                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1813
1814         if (make_active) {
1815                 scst_make_deferred_commands_active(tgt_dev);
1816                 res = NULL;
1817         } else
1818                 res = scst_check_deferred_commands(tgt_dev);
1819
1820         TRACE_EXIT_HRES(res);
1821         return res;
1822 }
1823
1824 /* cmd must be additionally referenced to not die inside */
1825 static int scst_do_real_exec(struct scst_cmd *cmd)
1826 {
1827         int res = SCST_EXEC_NOT_COMPLETED;
1828 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1829         int rc;
1830 #endif
1831         struct scst_device *dev = cmd->dev;
1832         struct scst_dev_type *handler = dev->handler;
1833
1834         TRACE_ENTRY();
1835
1836         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1837
1838         if (handler->exec) {
1839                 if (unlikely(!dev->handler->exec_atomic &&
1840                              scst_cmd_atomic(cmd))) {
1841                         /*
1842                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1843                          * optimization.
1844                          */
1845                         TRACE_DBG("Dev handler %s exec() needs thread "
1846                                 "context, rescheduling", dev->handler->name);
1847                         res = SCST_EXEC_NEED_THREAD;
1848                         goto out_restore;
1849                 }
1850
1851                 TRACE_DBG("Calling dev handler %s exec(%p)",
1852                       handler->name, cmd);
1853                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1854                         cmd->cdb_len);
1855                 res = handler->exec(cmd);
1856                 TRACE_DBG("Dev handler %s exec() returned %d",
1857                       handler->name, res);
1858
1859                 if (res == SCST_EXEC_COMPLETED)
1860                         goto out_complete;
1861                 else if (res == SCST_EXEC_NEED_THREAD)
1862                         goto out_restore;
1863
1864                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1865         }
1866
1867         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1868
1869         if (unlikely(dev->scsi_dev == NULL)) {
1870                 PRINT_ERROR("Command for virtual device must be "
1871                         "processed by device handler (lun %lld)!",
1872                         (long long unsigned int)cmd->lun);
1873                 goto out_error;
1874         }
1875
1876         res = scst_check_local_events(cmd);
1877         if (unlikely(res != 0))
1878                 goto out_done;
1879
1880 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1881         if (unlikely(scst_cmd_atomic(cmd))) {
1882                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1883                         "context, rescheduling to the thread (handler %s)",
1884                         handler->name);
1885                 res = SCST_EXEC_NEED_THREAD;
1886                 goto out_restore;
1887         }
1888 #endif
1889
1890 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1891         if (unlikely(scst_alloc_request(cmd) != 0)) {
1892                 if (scst_cmd_atomic(cmd)) {
1893                         res = SCST_EXEC_NEED_THREAD;
1894                         goto out_restore;
1895                 } else {
1896                         PRINT_INFO("%s", "Unable to allocate request, "
1897                                 "sending BUSY status");
1898                         goto out_busy;
1899                 }
1900         }
1901
1902         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1903                     (void *)cmd->scsi_req->sr_buffer,
1904                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1905                     cmd->retries);
1906 #else
1907         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1908                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1909                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1910                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1911         if (unlikely(rc != 0)) {
1912                 if (scst_cmd_atomic(cmd)) {
1913                         res = SCST_EXEC_NEED_THREAD;
1914                         goto out_restore;
1915                 } else {
1916                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1917                         goto out_error;
1918                 }
1919         }
1920 #endif
1921
1922 out_complete:
1923         res = SCST_EXEC_COMPLETED;
1924
1925 out:
1926         TRACE_EXIT();
1927         return res;
1928
1929 out_restore:
1930         /* Restore the state */
1931         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1932         goto out;
1933
1934 out_error:
1935         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1936         goto out_done;
1937
1938 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1939 out_busy:
1940         scst_set_busy(cmd);
1941         /* go through */
1942 #endif
1943
1944 out_done:
1945         res = SCST_EXEC_COMPLETED;
1946         /* Report the result */
1947         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1948         goto out_complete;
1949 }
1950
1951 static inline int scst_real_exec(struct scst_cmd *cmd)
1952 {
1953         int res;
1954
1955         TRACE_ENTRY();
1956
1957         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1958         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1959         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1960
1961         __scst_cmd_get(cmd);
1962
1963         res = scst_do_real_exec(cmd);
1964
1965         if (likely(res == SCST_EXEC_COMPLETED)) {
1966                 scst_post_exec_sn(cmd, true);
1967                 if (cmd->dev->scsi_dev != NULL)
1968                         generic_unplug_device(
1969                                 cmd->dev->scsi_dev->request_queue);
1970         } else
1971                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1972
1973         __scst_cmd_put(cmd);
1974
1975         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1976
1977         TRACE_EXIT_RES(res);
1978         return res;
1979 }
1980
1981 static int scst_do_local_exec(struct scst_cmd *cmd)
1982 {
1983         int res;
1984         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1985
1986         TRACE_ENTRY();
1987
1988         /* Check READ_ONLY device status */
1989         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1990             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1991              cmd->cdb[0] == WRITE_10 ||
1992              cmd->cdb[0] == WRITE_12 ||
1993              cmd->cdb[0] == WRITE_16 ||
1994              cmd->cdb[0] == WRITE_VERIFY ||
1995              cmd->cdb[0] == WRITE_VERIFY_12 ||
1996              cmd->cdb[0] == WRITE_VERIFY_16 ||
1997              (cmd->dev->handler->type == TYPE_TAPE &&
1998               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1999                 scst_set_cmd_error(cmd,
2000                            SCST_LOAD_SENSE(scst_sense_data_protect));
2001                 goto out_done;
2002         }
2003
2004         /*
2005          * Adding new commands here don't forget to update
2006          * scst_is_cmd_local() in scst.h, if necessary
2007          */
2008
2009         switch (cmd->cdb[0]) {
2010         case MODE_SELECT:
2011         case MODE_SELECT_10:
2012         case LOG_SELECT:
2013                 res = scst_pre_select(cmd);
2014                 break;
2015         case RESERVE:
2016         case RESERVE_10:
2017                 res = scst_reserve_local(cmd);
2018                 break;
2019         case RELEASE:
2020         case RELEASE_10:
2021                 res = scst_release_local(cmd);
2022                 break;
2023         case REPORT_LUNS:
2024                 res = scst_report_luns_local(cmd);
2025                 break;
2026         default:
2027                 res = SCST_EXEC_NOT_COMPLETED;
2028                 break;
2029         }
2030
2031 out:
2032         TRACE_EXIT_RES(res);
2033         return res;
2034
2035 out_done:
2036         /* Report the result */
2037         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2038         res = SCST_EXEC_COMPLETED;
2039         goto out;
2040 }
2041
2042 static int scst_local_exec(struct scst_cmd *cmd)
2043 {
2044         int res;
2045
2046         TRACE_ENTRY();
2047
2048         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2049         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2050         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2051
2052         __scst_cmd_get(cmd);
2053
2054         res = scst_do_local_exec(cmd);
2055         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2056                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2057         else if (res == SCST_EXEC_COMPLETED)
2058                 scst_post_exec_sn(cmd, true);
2059         else
2060                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2061
2062         __scst_cmd_put(cmd);
2063
2064         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2065         TRACE_EXIT_RES(res);
2066         return res;
2067 }
2068
2069 static int scst_exec(struct scst_cmd **active_cmd)
2070 {
2071         struct scst_cmd *cmd = *active_cmd;
2072         struct scst_cmd *ref_cmd;
2073         struct scst_device *dev = cmd->dev;
2074         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2075
2076         TRACE_ENTRY();
2077
2078         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2079                 goto out;
2080
2081         /* To protect tgt_dev */
2082         ref_cmd = cmd;
2083         __scst_cmd_get(ref_cmd);
2084
2085         count = 0;
2086         while (1) {
2087                 int rc;
2088
2089                 cmd->sent_for_exec = 1;
2090                 cmd->scst_cmd_done = scst_cmd_done_local;
2091                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2092
2093                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2094                     (cmd->data_direction == SCST_DATA_WRITE))
2095                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2096
2097                 rc = scst_do_local_exec(cmd);
2098                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2099                         /* Nothing to do */;
2100                 else if (rc == SCST_EXEC_NEED_THREAD) {
2101                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2102                                 "thread context, rescheduling");
2103                         scst_dec_on_dev_cmd(cmd);
2104                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2105                         break;
2106                 } else {
2107                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2108                         goto done;
2109                 }
2110
2111                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2112
2113                 rc = scst_do_real_exec(cmd);
2114                 if (likely(rc == SCST_EXEC_COMPLETED))
2115                         /* Nothing to do */;
2116                 else if (rc == SCST_EXEC_NEED_THREAD) {
2117                         TRACE_DBG("scst_real_exec() requested thread "
2118                                 "context, rescheduling (cmd %p)", cmd);
2119                         scst_dec_on_dev_cmd(cmd);
2120                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2121                         break;
2122                 } else
2123                         sBUG();
2124
2125 done:
2126                 count++;
2127
2128                 cmd = scst_post_exec_sn(cmd, false);
2129                 if (cmd == NULL)
2130                         break;
2131
2132                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2133                         break;
2134
2135                 __scst_cmd_put(ref_cmd);
2136                 ref_cmd = cmd;
2137                 __scst_cmd_get(ref_cmd);
2138         }
2139
2140         *active_cmd = cmd;
2141
2142         if (count == 0)
2143                 goto out_put;
2144
2145         if (dev->scsi_dev != NULL)
2146                 generic_unplug_device(dev->scsi_dev->request_queue);
2147
2148 out_put:
2149         __scst_cmd_put(ref_cmd);
2150         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2151
2152 out:
2153         TRACE_EXIT_RES(res);
2154         return res;
2155 }
2156
2157 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2158 {
2159         int res;
2160         struct scst_cmd *cmd = *active_cmd;
2161         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2162         typeof(tgt_dev->expected_sn) expected_sn;
2163
2164         TRACE_ENTRY();
2165
2166 #ifdef CONFIG_SCST_MEASURE_LATENCY
2167         if (cmd->pre_exec_finish == 0) {
2168                 struct timespec ts;
2169                 getnstimeofday(&ts);
2170                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2171                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2172                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2173                         ts.tv_sec, ts.tv_nsec);
2174         }
2175 #endif
2176
2177         if (unlikely(cmd->internal))
2178                 goto exec;
2179
2180         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2181                 goto exec;
2182
2183         sBUG_ON(!cmd->sn_set);
2184
2185         expected_sn = tgt_dev->expected_sn;
2186         /* Optimized for lockless fast path */
2187         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2188                 spin_lock_irq(&tgt_dev->sn_lock);
2189
2190                 tgt_dev->def_cmd_count++;
2191                 /*
2192                  * Memory barrier is needed here to implement lockless fast
2193                  * path. We need the exact order of read and write between
2194                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2195                  * when expected_sn was changed to be equal to cmd->sn while
2196                  * we are queuing cmd the deferred list after the expected_sn
2197                  * below. It will lead to a forever stuck command. But with
2198                  * the barrier in such case __scst_check_deferred_commands()
2199                  * will be called and it will take sn_lock, so we will be
2200                  * synchronized.
2201                  */
2202                 smp_mb();
2203
2204                 expected_sn = tgt_dev->expected_sn;
2205                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2206                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2207                                               &cmd->cmd_flags))) {
2208                                 /* Necessary to allow aborting out of sn cmds */
2209                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2210                                         "(tag %llu, sn %lu)", cmd,
2211                                         (long long unsigned)cmd->tag, cmd->sn);
2212                                 tgt_dev->def_cmd_count--;
2213                                 scst_set_cmd_abnormal_done_state(cmd);
2214                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2215                         } else {
2216                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2217                                         "expected_sn=%ld)", cmd, cmd->sn,
2218                                         cmd->sn_set, expected_sn);
2219                                 list_add_tail(&cmd->sn_cmd_list_entry,
2220                                               &tgt_dev->deferred_cmd_list);
2221                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2222                         }
2223                         spin_unlock_irq(&tgt_dev->sn_lock);
2224                         goto out;
2225                 } else {
2226                         TRACE_SN("Somebody incremented expected_sn %ld, "
2227                                 "continuing", expected_sn);
2228                         tgt_dev->def_cmd_count--;
2229                         spin_unlock_irq(&tgt_dev->sn_lock);
2230                 }
2231         }
2232
2233 exec:
2234         res = scst_exec(active_cmd);
2235
2236 out:
2237         TRACE_EXIT_HRES(res);
2238         return res;
2239 }
2240
2241 /* No locks supposed to be held */
2242 static int scst_check_sense(struct scst_cmd *cmd)
2243 {
2244         int res = 0;
2245         struct scst_device *dev = cmd->dev;
2246
2247         TRACE_ENTRY();
2248
2249         if (unlikely(cmd->ua_ignore))
2250                 goto out;
2251
2252         /* If we had internal bus reset behind us, set the command error UA */
2253         if ((dev->scsi_dev != NULL) &&
2254             unlikely(cmd->host_status == DID_RESET) &&
2255             scst_is_ua_command(cmd)) {
2256                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2257                       dev->scsi_dev->was_reset, cmd->host_status);
2258                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2259                 /* It looks like it is safe to clear was_reset here */
2260                 dev->scsi_dev->was_reset = 0;
2261         }
2262
2263         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2264             SCST_SENSE_VALID(cmd->sense)) {
2265                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2266                         SCST_SENSE_BUFFERSIZE);
2267
2268                 /* Check Unit Attention Sense Key */
2269                 if (scst_is_ua_sense(cmd->sense)) {
2270                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2271                                 if (cmd->double_ua_possible) {
2272                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2273                                                 "detected for device %p", dev);
2274                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2275                                                 " %p (tag %llu)", cmd,
2276                                                 (long long unsigned)cmd->tag);
2277
2278                                         cmd->status = 0;
2279                                         cmd->msg_status = 0;
2280                                         cmd->host_status = DID_OK;
2281                                         cmd->driver_status = 0;
2282
2283                                         mempool_free(cmd->sense,
2284                                                      scst_sense_mempool);
2285                                         cmd->sense = NULL;
2286
2287                                         scst_check_restore_sg_buff(cmd);
2288
2289                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2290                                         cmd->data_direction =
2291                                                 cmd->dbl_ua_orig_data_direction;
2292                                         cmd->resp_data_len =
2293                                                 cmd->dbl_ua_orig_resp_data_len;
2294
2295                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2296                                         cmd->retry = 1;
2297                                         res = 1;
2298                                         goto out;
2299                                 }
2300                         }
2301                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2302                                 SCST_SENSE_BUFFERSIZE);
2303                 }
2304         }
2305
2306         if (unlikely(cmd->double_ua_possible)) {
2307                 if (scst_is_ua_command(cmd)) {
2308                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2309                                 "cmd %p)", dev, cmd);
2310                         /*
2311                          * Lock used to protect other flags in the bitfield
2312                          * (just in case, actually). Those flags can't be
2313                          * changed in parallel, because the device is
2314                          * serialized.
2315                          */
2316                         spin_lock_bh(&dev->dev_lock);
2317                         dev->dev_double_ua_possible = 0;
2318                         spin_unlock_bh(&dev->dev_lock);
2319                 }
2320         }
2321
2322 out:
2323         TRACE_EXIT_RES(res);
2324         return res;
2325 }
2326
2327 static int scst_check_auto_sense(struct scst_cmd *cmd)
2328 {
2329         int res = 0;
2330
2331         TRACE_ENTRY();
2332
2333         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2334             (!SCST_SENSE_VALID(cmd->sense) ||
2335              SCST_NO_SENSE(cmd->sense))) {
2336                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2337                       "cmd->status=%x, cmd->msg_status=%x, "
2338                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2339                       cmd->status, cmd->msg_status, cmd->host_status,
2340                       cmd->driver_status, cmd);
2341                 res = 1;
2342         } else if (unlikely(cmd->host_status)) {
2343                 if ((cmd->host_status == DID_REQUEUE) ||
2344                     (cmd->host_status == DID_IMM_RETRY) ||
2345                     (cmd->host_status == DID_SOFT_ERROR) ||
2346                     (cmd->host_status == DID_ABORT)) {
2347                         scst_set_busy(cmd);
2348                 } else {
2349                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2350                                 "received, returning HARDWARE ERROR instead "
2351                                 "(cmd %p)", cmd->host_status, cmd);
2352                         scst_set_cmd_error(cmd,
2353                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2354                 }
2355         }
2356
2357         TRACE_EXIT_RES(res);
2358         return res;
2359 }
2360
2361 static int scst_pre_dev_done(struct scst_cmd *cmd)
2362 {
2363         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2364
2365         TRACE_ENTRY();
2366
2367         if (unlikely(scst_check_auto_sense(cmd))) {
2368                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2369                             "without sense data (opcode 0x%x), issuing "
2370                             "REQUEST SENSE", cmd->cdb[0]);
2371                 rc = scst_prepare_request_sense(cmd);
2372                 if (rc == 0)
2373                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2374                 else {
2375                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2376                                     "returning HARDWARE ERROR");
2377                         scst_set_cmd_error(cmd,
2378                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2379                 }
2380                 goto out;
2381         } else if (unlikely(scst_check_sense(cmd)))
2382                 goto out;
2383
2384         if (likely(scsi_status_is_good(cmd->status))) {
2385                 unsigned char type = cmd->dev->handler->type;
2386                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2387                               cmd->cdb[0] == MODE_SENSE_10)) &&
2388                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2389                     (type == TYPE_DISK ||
2390                      type == TYPE_WORM ||
2391                      type == TYPE_MOD ||
2392                      type == TYPE_TAPE)) {
2393                         int32_t length;
2394                         uint8_t *address;
2395                         bool err = false;
2396
2397                         length = scst_get_buf_first(cmd, &address);
2398                         if (length < 0) {
2399                                 PRINT_ERROR("%s", "Unable to get "
2400                                         "MODE_SENSE buffer");
2401                                 scst_set_cmd_error(cmd,
2402                                         SCST_LOAD_SENSE(
2403                                                 scst_sense_hardw_error));
2404                                 err = true;
2405                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2406                                 address[2] |= 0x80;   /* Write Protect*/
2407                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2408                                 address[3] |= 0x80;   /* Write Protect*/
2409                         scst_put_buf(cmd, address);
2410
2411                         if (err)
2412                                 goto out;
2413                 }
2414
2415                 /*
2416                  * Check and clear NormACA option for the device, if necessary,
2417                  * since we don't support ACA
2418                  */
2419                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2420                     /* Std INQUIRY data (no EVPD) */
2421                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2422                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2423                         uint8_t *buffer;
2424                         int buflen;
2425                         bool err = false;
2426
2427                         /* ToDo: all pages ?? */
2428                         buflen = scst_get_buf_first(cmd, &buffer);
2429                         if (buflen > SCST_INQ_BYTE3) {
2430 #ifdef CONFIG_SCST_EXTRACHECKS
2431                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2432                                         PRINT_INFO("NormACA set for device: "
2433                                             "lun=%lld, type 0x%02x. Clear it, "
2434                                             "since it's unsupported.",
2435                                             (long long unsigned int)cmd->lun,
2436                                             buffer[0]);
2437                                 }
2438 #endif
2439                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2440                         } else if (buflen != 0) {
2441                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2442                                     "buffer");
2443                                 scst_set_cmd_error(cmd,
2444                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2445                                 err = true;
2446                         }
2447                         if (buflen > 0)
2448                                 scst_put_buf(cmd, buffer);
2449
2450                         if (err)
2451                                 goto out;
2452                 }
2453
2454                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2455                     (cmd->cdb[0] == MODE_SELECT_10) ||
2456                     (cmd->cdb[0] == LOG_SELECT))) {
2457                         TRACE(TRACE_SCSI,
2458                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2459                                 (long long unsigned int)cmd->lun);
2460                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2461                         goto out;
2462                 }
2463         } else {
2464                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2465                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2466                                         &cmd->tgt_dev->tgt_dev_flags)) {
2467                                 struct scst_tgt_dev *tgt_dev_tmp;
2468                                 struct scst_device *dev = cmd->dev;
2469
2470                                 TRACE(TRACE_SCSI,
2471                                         "Real RESERVE failed lun=%lld, "
2472                                         "status=%x",
2473                                         (long long unsigned int)cmd->lun,
2474                                         cmd->status);
2475                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2476                                         SCST_SENSE_BUFFERSIZE);
2477
2478                                 /* Clearing the reservation */
2479                                 spin_lock_bh(&dev->dev_lock);
2480                                 list_for_each_entry(tgt_dev_tmp,
2481                                                     &dev->dev_tgt_dev_list,
2482                                                     dev_tgt_dev_list_entry) {
2483                                         clear_bit(SCST_TGT_DEV_RESERVED,
2484                                                 &tgt_dev_tmp->tgt_dev_flags);
2485                                 }
2486                                 dev->dev_reserved = 0;
2487                                 spin_unlock_bh(&dev->dev_lock);
2488                         }
2489                 }
2490
2491                 /* Check for MODE PARAMETERS CHANGED UA */
2492                 if ((cmd->dev->scsi_dev != NULL) &&
2493                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2494                     SCST_SENSE_VALID(cmd->sense) &&
2495                     scst_is_ua_sense(cmd->sense) &&
2496                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2497                         TRACE(TRACE_SCSI,
2498                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2499                               (long long unsigned int)cmd->lun);
2500                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2501                         goto out;
2502                 }
2503         }
2504
2505         cmd->state = SCST_CMD_STATE_DEV_DONE;
2506
2507 out:
2508         TRACE_EXIT_RES(res);
2509         return res;
2510 }
2511
2512 static int scst_mode_select_checks(struct scst_cmd *cmd)
2513 {
2514         int res = SCST_CMD_STATE_RES_CONT_SAME;
2515         int atomic = scst_cmd_atomic(cmd);
2516
2517         TRACE_ENTRY();
2518
2519         if (likely(scsi_status_is_good(cmd->status))) {
2520                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2521                     (cmd->cdb[0] == MODE_SELECT_10) ||
2522                     (cmd->cdb[0] == LOG_SELECT))) {
2523                         struct scst_device *dev = cmd->dev;
2524                         if (atomic && (dev->scsi_dev != NULL)) {
2525                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2526                                         "context required");
2527                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2528                                 goto out;
2529                         }
2530
2531                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2532                                 "setting the SELECT UA (lun=%lld)",
2533                                 (long long unsigned int)cmd->lun);
2534
2535                         spin_lock_bh(&dev->dev_lock);
2536                         spin_lock(&scst_temp_UA_lock);
2537                         if (cmd->cdb[0] == LOG_SELECT) {
2538                                 scst_set_sense(scst_temp_UA,
2539                                         sizeof(scst_temp_UA),
2540                                         UNIT_ATTENTION, 0x2a, 0x02);
2541                         } else {
2542                                 scst_set_sense(scst_temp_UA,
2543                                         sizeof(scst_temp_UA),
2544                                         UNIT_ATTENTION, 0x2a, 0x01);
2545                         }
2546                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2547                                 sizeof(scst_temp_UA));
2548                         spin_unlock(&scst_temp_UA_lock);
2549                         spin_unlock_bh(&dev->dev_lock);
2550
2551                         if (dev->scsi_dev != NULL)
2552                                 scst_obtain_device_parameters(dev);
2553                 }
2554         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2555                     SCST_SENSE_VALID(cmd->sense) &&
2556                     scst_is_ua_sense(cmd->sense) &&
2557                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2558                      (cmd->sense[12] == 0x29) /* reset */ ||
2559                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2560                      /* cleared by another ini (just in case) */
2561                      (cmd->sense[12] == 0x2F))) {
2562                 if (atomic) {
2563                         TRACE_DBG("Possible parameters changed UA %x: "
2564                                 "thread context required", cmd->sense[12]);
2565                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2566                         goto out;
2567                 }
2568
2569                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2570                         "(lun %lld): getting new parameters", cmd->sense[12],
2571                         (long long unsigned int)cmd->lun);
2572
2573                 scst_obtain_device_parameters(cmd->dev);
2574         } else
2575                 sBUG();
2576
2577         cmd->state = SCST_CMD_STATE_DEV_DONE;
2578
2579 out:
2580         TRACE_EXIT_HRES(res);
2581         return res;
2582 }
2583
2584 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2585 {
2586         if (likely(cmd->sn_set))
2587                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2588
2589         scst_make_deferred_commands_active(cmd->tgt_dev);
2590 }
2591
2592 static int scst_dev_done(struct scst_cmd *cmd)
2593 {
2594         int res = SCST_CMD_STATE_RES_CONT_SAME;
2595         int state;
2596         struct scst_device *dev = cmd->dev;
2597
2598         TRACE_ENTRY();
2599
2600         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2601
2602         if (likely(!scst_is_cmd_local(cmd)) &&
2603             likely(dev->handler->dev_done != NULL)) {
2604                 int rc;
2605
2606                 if (unlikely(!dev->handler->dev_done_atomic &&
2607                              scst_cmd_atomic(cmd))) {
2608                         /*
2609                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2610                          * optimization.
2611                          */
2612                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2613                               "context, rescheduling", dev->handler->name);
2614                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2615                         goto out;
2616                 }
2617
2618                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2619                       dev->handler->name, cmd);
2620                 rc = dev->handler->dev_done(cmd);
2621                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2622                       dev->handler->name, rc);
2623                 if (rc != SCST_CMD_STATE_DEFAULT)
2624                         state = rc;
2625         }
2626
2627         switch (state) {
2628         case SCST_CMD_STATE_PRE_XMIT_RESP:
2629         case SCST_CMD_STATE_DEV_PARSE:
2630         case SCST_CMD_STATE_PRE_PARSE:
2631         case SCST_CMD_STATE_PREPARE_SPACE:
2632         case SCST_CMD_STATE_RDY_TO_XFER:
2633         case SCST_CMD_STATE_TGT_PRE_EXEC:
2634         case SCST_CMD_STATE_SEND_FOR_EXEC:
2635         case SCST_CMD_STATE_LOCAL_EXEC:
2636         case SCST_CMD_STATE_REAL_EXEC:
2637         case SCST_CMD_STATE_PRE_DEV_DONE:
2638         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2639         case SCST_CMD_STATE_DEV_DONE:
2640         case SCST_CMD_STATE_XMIT_RESP:
2641         case SCST_CMD_STATE_FINISHED:
2642         case SCST_CMD_STATE_FINISHED_INTERNAL:
2643                 cmd->state = state;
2644                 break;
2645
2646         case SCST_CMD_STATE_NEED_THREAD_CTX:
2647                 TRACE_DBG("Dev handler %s dev_done() requested "
2648                       "thread context, rescheduling",
2649                       dev->handler->name);
2650                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2651                 break;
2652
2653         default:
2654                 if (state >= 0) {
2655                         PRINT_ERROR("Dev handler %s dev_done() returned "
2656                                 "invalid cmd state %d",
2657                                 dev->handler->name, state);
2658                 } else {
2659                         PRINT_ERROR("Dev handler %s dev_done() returned "
2660                                 "error %d", dev->handler->name,
2661                                 state);
2662                 }
2663                 scst_set_cmd_error(cmd,
2664                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2665                 scst_set_cmd_abnormal_done_state(cmd);
2666                 break;
2667         }
2668
2669         if (cmd->needs_unblocking)
2670                 scst_unblock_dev_cmd(cmd);
2671
2672         if (likely(cmd->dec_on_dev_needed))
2673                 scst_dec_on_dev_cmd(cmd);
2674
2675         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2676                 scst_inc_check_expected_sn(cmd);
2677
2678         if (unlikely(cmd->internal))
2679                 cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
2680
2681 out:
2682         TRACE_EXIT_HRES(res);
2683         return res;
2684 }
2685
2686 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2687 {
2688         int res;
2689
2690         TRACE_ENTRY();
2691
2692         EXTRACHECKS_BUG_ON(cmd->internal);
2693
2694 #ifdef CONFIG_SCST_DEBUG_TM
2695         if (cmd->tm_dbg_delayed &&
2696                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2697                 if (scst_cmd_atomic(cmd)) {
2698                         TRACE_MGMT_DBG("%s",
2699                                 "DEBUG_TM delayed cmd needs a thread");
2700                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2701                         return res;
2702                 }
2703                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2704                         cmd, cmd->tag);
2705                 schedule_timeout_uninterruptible(HZ);
2706         }
2707 #endif
2708
2709         if (likely(cmd->tgt_dev != NULL)) {
2710                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2711                 atomic_dec(&cmd->dev->dev_cmd_count);
2712                 /* If expected values not set, expected direction is UNKNOWN */
2713                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2714                         atomic_dec(&cmd->dev->write_cmd_count);
2715
2716                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2717                         scst_on_hq_cmd_response(cmd);
2718
2719                 if (unlikely(!cmd->sent_for_exec)) {
2720                         TRACE_SN("cmd %p was not sent to mid-lev"
2721                                 " (sn %ld, set %d)",
2722                                 cmd, cmd->sn, cmd->sn_set);
2723                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2724                         cmd->sent_for_exec = 1;
2725                 }
2726         }
2727
2728         /*
2729          * If we don't remove cmd from the search list here, before
2730          * submitting it for transmittion, we will have a race, when for
2731          * some reason cmd's release is delayed after transmittion and
2732          * initiator sends cmd with the same tag => it is possible that
2733          * a wrong cmd will be found by find() functions.
2734          */
2735         spin_lock_irq(&cmd->sess->sess_list_lock);
2736         list_del(&cmd->search_cmd_list_entry);
2737         spin_unlock_irq(&cmd->sess->sess_list_lock);
2738
2739         cmd->done = 1;
2740         smp_mb(); /* to sync with scst_abort_cmd() */
2741
2742         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2743                 scst_xmit_process_aborted_cmd(cmd);
2744
2745         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2746                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2747                                 " skipping",
2748                                 cmd, (long long unsigned int)cmd->tag);
2749                 cmd->state = SCST_CMD_STATE_FINISHED;
2750                 res = SCST_CMD_STATE_RES_CONT_SAME;
2751                 goto out;
2752         }
2753
2754         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2755             (cmd->data_direction == SCST_DATA_READ))
2756                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2757
2758         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2759         res = SCST_CMD_STATE_RES_CONT_SAME;
2760
2761 out:
2762 #ifdef CONFIG_SCST_MEASURE_LATENCY
2763         {
2764                 struct timespec ts;
2765                 uint64_t finish, scst_time, proc_time;
2766                 struct scst_session *sess = cmd->sess;
2767
2768                 getnstimeofday(&ts);
2769                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2770
2771                 spin_lock_bh(&sess->meas_lock);
2772
2773                 scst_time = cmd->pre_exec_finish - cmd->start;
2774                 scst_time += finish - cmd->post_exec_start;
2775                 proc_time = finish - cmd->start;
2776
2777                 sess->scst_time += scst_time;
2778                 sess->processing_time += proc_time;
2779                 sess->processed_cmds++;
2780
2781                 spin_unlock_bh(&sess->meas_lock);
2782
2783                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2784                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2785                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2786                         proc_time);
2787         }
2788 #endif
2789         TRACE_EXIT_HRES(res);
2790         return res;
2791 }
2792
2793 static int scst_xmit_response(struct scst_cmd *cmd)
2794 {
2795         int res, rc;
2796
2797         TRACE_ENTRY();
2798
2799         EXTRACHECKS_BUG_ON(cmd->internal);
2800
2801         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2802                      scst_cmd_atomic(cmd))) {
2803                 /*
2804                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2805                  * optimization.
2806                  */
2807                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2808                               "context, rescheduling", cmd->tgtt->name);
2809                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2810                 goto out;
2811         }
2812
2813         while (1) {
2814                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2815
2816                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2817                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2818
2819                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2820
2821 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2822                 if (cmd->tgt_sg) {
2823                         int i;
2824                         struct scatterlist *sg = cmd->tgt_sg;
2825                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2826                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2827                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2828                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2829                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2830                                         sg_virt(&sg[i]), sg[i].length);
2831                         }
2832                 }
2833 #endif
2834
2835 #ifdef CONFIG_SCST_DEBUG_RETRY
2836                 if (((scst_random() % 100) == 77))
2837                         rc = SCST_TGT_RES_QUEUE_FULL;
2838                 else
2839 #endif
2840                         rc = cmd->tgtt->xmit_response(cmd);
2841                 TRACE_DBG("xmit_response() returned %d", rc);
2842
2843                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2844                         goto out;
2845
2846                 /* Restore the previous state */
2847                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2848
2849                 switch (rc) {
2850                 case SCST_TGT_RES_QUEUE_FULL:
2851                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2852                                 break;
2853                         else
2854                                 continue;
2855
2856                 case SCST_TGT_RES_NEED_THREAD_CTX:
2857                         TRACE_DBG("Target driver %s xmit_response() "
2858                               "requested thread context, rescheduling",
2859                               cmd->tgtt->name);
2860                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2861                         break;
2862
2863                 default:
2864                         goto out_error;
2865                 }
2866                 break;
2867         }
2868
2869 out:
2870         /* Caution: cmd can be already dead here */
2871         TRACE_EXIT_HRES(res);
2872         return res;
2873
2874 out_error:
2875         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2876                 PRINT_ERROR("Target driver %s xmit_response() returned "
2877                         "fatal error", cmd->tgtt->name);
2878         } else {
2879                 PRINT_ERROR("Target driver %s xmit_response() returned "
2880                         "invalid value %d", cmd->tgtt->name, rc);
2881         }
2882         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2883         cmd->state = SCST_CMD_STATE_FINISHED;
2884         res = SCST_CMD_STATE_RES_CONT_SAME;
2885         goto out;
2886 }
2887
2888 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2889         enum scst_exec_context pref_context)
2890 {
2891         TRACE_ENTRY();
2892
2893         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2894
2895         cmd->state = SCST_CMD_STATE_FINISHED;
2896         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2897
2898         TRACE_EXIT();
2899         return;
2900 }
2901 EXPORT_SYMBOL(scst_tgt_cmd_done);
2902
2903 static int scst_finish_cmd(struct scst_cmd *cmd)
2904 {
2905         int res;
2906
2907         TRACE_ENTRY();
2908
2909         atomic_dec(&cmd->sess->sess_cmd_count);
2910
2911         cmd->finished = 1;
2912         smp_mb(); /* to sync with scst_abort_cmd() */
2913
2914         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2915                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2916                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2917                         atomic_read(&scst_cmd_count));
2918
2919                 scst_finish_cmd_mgmt(cmd);
2920         }
2921
2922         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2923                 if ((cmd->tgt_dev != NULL) &&
2924                     scst_is_ua_sense(cmd->sense)) {
2925                         /* This UA delivery failed, so requeue it */
2926                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2927                                 "%p", cmd);
2928                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2929                                         SCST_SENSE_BUFFERSIZE, 1);
2930                 }
2931         }
2932
2933         __scst_cmd_put(cmd);
2934
2935         res = SCST_CMD_STATE_RES_CONT_NEXT;
2936
2937         TRACE_EXIT_HRES(res);
2938         return res;
2939 }
2940
2941 /*
2942  * No locks, but it must be externally serialized (see comment for
2943  * scst_cmd_init_done() in scst.h)
2944  */
2945 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2946 {
2947         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2948         unsigned long flags;
2949
2950         TRACE_ENTRY();
2951
2952         if (scst_is_implicit_hq(cmd)) {
2953                 TRACE_SN("Implicit HQ cmd %p", cmd);
2954                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2955         }
2956
2957         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2958
2959         /* Optimized for lockless fast path */
2960
2961         scst_check_debug_sn(cmd);
2962
2963         if (cmd->dev->queue_alg ==
2964                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2965                 /*
2966                  * Not the best way, but well enough until there will be a
2967                  * possibility to specify queue type during pass-through
2968                  * commands submission.
2969                  */
2970                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2971         }
2972
2973         switch (cmd->queue_type) {
2974         case SCST_CMD_QUEUE_SIMPLE:
2975         case SCST_CMD_QUEUE_UNTAGGED:
2976 #if 0 /* left for future performance investigations */
2977                 if (scst_cmd_is_expected_set(cmd)) {
2978                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2979                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2980                                 goto ordered;
2981                 } else
2982                         goto ordered;
2983 #endif
2984                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2985                         /*
2986                          * atomic_inc_return() implies memory barrier to sync
2987                          * with scst_inc_expected_sn()
2988                          */
2989                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2990                                 tgt_dev->curr_sn++;
2991                                 TRACE_SN("Incremented curr_sn %ld",
2992                                         tgt_dev->curr_sn);
2993                         }
2994                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2995                         cmd->sn = tgt_dev->curr_sn;
2996
2997                         tgt_dev->prev_cmd_ordered = 0;
2998                 } else {
2999                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
3000                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
3001                         goto ordered;
3002                 }
3003                 break;
3004
3005         case SCST_CMD_QUEUE_ORDERED:
3006                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3007 ordered:
3008                 if (!tgt_dev->prev_cmd_ordered) {
3009                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3010                         if (tgt_dev->num_free_sn_slots >= 0) {
3011                                 tgt_dev->num_free_sn_slots--;
3012                                 if (tgt_dev->num_free_sn_slots >= 0) {
3013                                         int i = 0;
3014                                         /* Commands can finish in any order, so
3015                                          * we don't know which slot is empty.
3016                                          */
3017                                         while (1) {
3018                                                 tgt_dev->cur_sn_slot++;
3019                                                 if (tgt_dev->cur_sn_slot ==
3020                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3021                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3022
3023                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3024                                                         break;
3025
3026                                                 i++;
3027                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3028                                         }
3029                                         TRACE_SN("New cur SN slot %zd",
3030                                                 tgt_dev->cur_sn_slot -
3031                                                 tgt_dev->sn_slots);
3032                                 }
3033                         }
3034                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3035                 }
3036                 tgt_dev->prev_cmd_ordered = 1;
3037                 tgt_dev->curr_sn++;
3038                 cmd->sn = tgt_dev->curr_sn;
3039                 break;
3040
3041         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3042                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3043                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3044                 tgt_dev->hq_cmd_count++;
3045                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3046                 cmd->hq_cmd_inced = 1;
3047                 goto out;
3048
3049         default:
3050                 sBUG();
3051         }
3052
3053         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3054                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3055                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3056                 atomic_read(tgt_dev->cur_sn_slot),
3057                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3058                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3059
3060         cmd->sn_set = 1;
3061
3062 out:
3063         TRACE_EXIT();
3064         return;
3065 }
3066
3067 /*
3068  * Returns 0 on success, > 0 when we need to wait for unblock,
3069  * < 0 if there is no device (lun) or device type handler.
3070  *
3071  * No locks, but might be on IRQ, protection is done by the
3072  * suspended activity.
3073  */
3074 static int scst_translate_lun(struct scst_cmd *cmd)
3075 {
3076         struct scst_tgt_dev *tgt_dev = NULL;
3077         int res;
3078
3079         TRACE_ENTRY();
3080
3081         /* See comment about smp_mb() in scst_suspend_activity() */
3082         __scst_get(1);
3083
3084         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3085                 struct list_head *sess_tgt_dev_list_head =
3086                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3087                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3088                         (long long unsigned int)cmd->lun);
3089                 res = -1;
3090                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3091                                 sess_tgt_dev_list_entry) {
3092                         if (tgt_dev->lun == cmd->lun) {
3093                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3094
3095                                 if (unlikely(tgt_dev->dev->handler ==
3096                                                 &scst_null_devtype)) {
3097                                         PRINT_INFO("Dev handler for device "
3098                                           "%lld is NULL, the device will not "
3099                                           "be visible remotely",
3100                                            (long long unsigned int)cmd->lun);
3101                                         break;
3102                                 }
3103
3104                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3105                                 cmd->tgt_dev = tgt_dev;
3106                                 cmd->dev = tgt_dev->dev;
3107
3108                                 res = 0;
3109                                 break;
3110                         }
3111                 }
3112                 if (res != 0) {
3113                         TRACE(TRACE_MINOR,
3114                                 "tgt_dev for lun %lld not found, command to "
3115                                 "unexisting LU?",
3116                                 (long long unsigned int)cmd->lun);
3117                         __scst_put();
3118                 }
3119         } else {
3120                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3121                 __scst_put();
3122                 res = 1;
3123         }
3124
3125         TRACE_EXIT_RES(res);
3126         return res;
3127 }
3128
3129 /*
3130  * No locks, but might be on IRQ
3131  *
3132  * Returns 0 on success, > 0 when we need to wait for unblock,
3133  * < 0 if there is no device (lun) or device type handler.
3134  */
3135 static int __scst_init_cmd(struct scst_cmd *cmd)
3136 {
3137         int res = 0;
3138
3139         TRACE_ENTRY();
3140
3141         res = scst_translate_lun(cmd);
3142         if (likely(res == 0)) {
3143                 int cnt;
3144                 bool failure = false;
3145
3146                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3147
3148                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3149                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3150                         TRACE(TRACE_MGMT_MINOR,
3151                                 "Too many pending commands (%d) in "
3152                                 "session, returning BUSY to initiator \"%s\"",
3153                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3154                                   "Anonymous" : cmd->sess->initiator_name);
3155                         failure = true;
3156                 }
3157
3158                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3159                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3160                         if (!failure) {
3161                                 TRACE(TRACE_MGMT_MINOR,
3162                                         "Too many pending device "
3163                                         "commands (%d), returning BUSY to "
3164                                         "initiator \"%s\"", cnt,
3165                                         (cmd->sess->initiator_name[0] == '\0') ?
3166                                                 "Anonymous" :
3167                                                 cmd->sess->initiator_name);
3168                                 failure = true;
3169                         }
3170                 }
3171
3172                 /* If expected values not set, expected direction is UNKNOWN */
3173                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3174                         atomic_inc(&cmd->dev->write_cmd_count);
3175
3176                 if (unlikely(failure))
3177                         goto out_busy;
3178
3179                 if (!cmd->set_sn_on_restart_cmd)
3180                         scst_cmd_set_sn(cmd);
3181         } else if (res < 0) {
3182                 TRACE_DBG("Finishing cmd %p", cmd);
3183                 scst_set_cmd_error(cmd,
3184                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3185                 scst_set_cmd_abnormal_done_state(cmd);
3186         } else
3187                 goto out;
3188
3189 out:
3190         TRACE_EXIT_RES(res);
3191         return res;
3192
3193 out_busy:
3194         scst_set_busy(cmd);
3195         scst_set_cmd_abnormal_done_state(cmd);
3196         goto out;
3197 }
3198
3199 /* Called under scst_init_lock and IRQs disabled */
3200 static void scst_do_job_init(void)
3201         __releases(&scst_init_lock)
3202         __acquires(&scst_init_lock)
3203 {
3204         struct scst_cmd *cmd;
3205         int susp;
3206
3207         TRACE_ENTRY();
3208
3209 restart:
3210         /*
3211          * There is no need for read barrier here, because we don't care where
3212          * this check will be done.
3213          */
3214         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3215         if (scst_init_poll_cnt > 0)
3216                 scst_init_poll_cnt--;
3217
3218         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3219                 int rc;
3220                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3221                         continue;
3222                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3223                         spin_unlock_irq(&scst_init_lock);
3224                         rc = __scst_init_cmd(cmd);
3225                         spin_lock_irq(&scst_init_lock);
3226                         if (rc > 0) {
3227                                 TRACE_MGMT_DBG("%s",
3228                                         "FLAG SUSPENDED set, restarting");
3229                                 goto restart;
3230                         }
3231                 } else {
3232                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3233                                        cmd, (long long unsigned int)cmd->tag);
3234                         scst_set_cmd_abnormal_done_state(cmd);
3235                 }
3236
3237                 /*
3238                  * Deleting cmd from init cmd list after __scst_init_cmd()
3239                  * is necessary to keep the check in scst_init_cmd() correct
3240                  * to preserve the commands order.
3241                  *
3242                  * We don't care about the race, when init cmd list is empty
3243                  * and one command detected that it just was not empty, so
3244                  * it's inserting to it, but another command at the same time
3245                  * seeing init cmd list empty and goes directly, because it
3246                  * could affect only commands from the same initiator to the
3247                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3248                  * the order in case of simultaneous such calls anyway.
3249                  */
3250                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3251                 smp_wmb(); /* enforce the required order */
3252                 list_del(&cmd->cmd_list_entry);
3253                 spin_unlock(&scst_init_lock);
3254
3255                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3256                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3257                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3258                         list_add(&cmd->cmd_list_entry,
3259                                 &cmd->cmd_lists->active_cmd_list);
3260                 else
3261                         list_add_tail(&cmd->cmd_list_entry,
3262                                 &cmd->cmd_lists->active_cmd_list);
3263                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3264                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3265
3266                 spin_lock(&scst_init_lock);
3267                 goto restart;
3268         }
3269
3270         /* It isn't really needed, but let's keep it */
3271         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3272                 goto restart;
3273
3274         TRACE_EXIT();
3275         return;
3276 }
3277
3278 static inline int test_init_cmd_list(void)
3279 {
3280         int res = (!list_empty(&scst_init_cmd_list) &&
3281                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3282                   unlikely(kthread_should_stop()) ||
3283                   (scst_init_poll_cnt > 0);
3284         return res;
3285 }
3286
3287 int scst_init_cmd_thread(void *arg)
3288 {
3289         TRACE_ENTRY();
3290
3291         PRINT_INFO("Init thread started, PID %d", current->pid);
3292
3293         current->flags |= PF_NOFREEZE;
3294
3295         set_user_nice(current, -10);
3296
3297         spin_lock_irq(&scst_init_lock);
3298         while (!kthread_should_stop()) {
3299                 wait_queue_t wait;
3300                 init_waitqueue_entry(&wait, current);
3301
3302                 if (!test_init_cmd_list()) {
3303                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3304                                                  &wait);
3305                         for (;;) {
3306                                 set_current_state(TASK_INTERRUPTIBLE);
3307                                 if (test_init_cmd_list())
3308                                         break;
3309                                 spin_unlock_irq(&scst_init_lock);
3310                                 schedule();
3311                                 spin_lock_irq(&scst_init_lock);
3312                         }
3313                         set_current_state(TASK_RUNNING);
3314                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3315                 }
3316                 scst_do_job_init();
3317         }
3318         spin_unlock_irq(&scst_init_lock);
3319
3320         /*
3321          * If kthread_should_stop() is true, we are guaranteed to be
3322          * on the module unload, so scst_init_cmd_list must be empty.
3323          */
3324         sBUG_ON(!list_empty(&scst_init_cmd_list));
3325
3326         PRINT_INFO("Init thread PID %d finished", current->pid);
3327
3328         TRACE_EXIT();
3329         return 0;
3330 }
3331
3332 /* Called with no locks held */
3333 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3334 {
3335         int res;
3336
3337         TRACE_ENTRY();
3338
3339         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3340
3341         cmd->atomic = atomic;
3342
3343         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3344
3345         do {
3346                 switch (cmd->state) {
3347                 case SCST_CMD_STATE_PRE_PARSE:
3348                         res = scst_pre_parse(cmd);
3349                         EXTRACHECKS_BUG_ON(res ==
3350                                 SCST_CMD_STATE_RES_NEED_THREAD);
3351                         break;
3352
3353                 case SCST_CMD_STATE_DEV_PARSE:
3354                         res = scst_parse_cmd(cmd);
3355                         break;
3356
3357                 case SCST_CMD_STATE_PREPARE_SPACE:
3358                         res = scst_prepare_space(cmd);
3359                         break;
3360
3361                 case SCST_CMD_STATE_RDY_TO_XFER:
3362                         res = scst_rdy_to_xfer(cmd);
3363                         break;
3364
3365                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3366                         res = scst_tgt_pre_exec(cmd);
3367                         break;
3368
3369                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3370                         if (tm_dbg_check_cmd(cmd) != 0) {
3371                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3372                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3373                                         "because of TM DBG delay", cmd,
3374                                         (long long unsigned int)cmd->tag);
3375                                 break;
3376                         }
3377                         res = scst_send_for_exec(&cmd);
3378                         /*
3379                          * !! At this point cmd, sess & tgt_dev can already be
3380                          * freed !!
3381                          */
3382                         break;
3383
3384                 case SCST_CMD_STATE_LOCAL_EXEC:
3385                         res = scst_local_exec(cmd);
3386                         /*
3387                          * !! At this point cmd, sess & tgt_dev can already be
3388                          * freed !!
3389                          */
3390                         break;
3391
3392                 case SCST_CMD_STATE_REAL_EXEC:
3393                         res = scst_real_exec(cmd);
3394                         /*
3395                          * !! At this point cmd, sess & tgt_dev can already be
3396                          * freed !!
3397                          */
3398                         break;
3399
3400                 case SCST_CMD_STATE_PRE_DEV_DONE:
3401                         res = scst_pre_dev_done(cmd);
3402                         EXTRACHECKS_BUG_ON(res ==
3403                                 SCST_CMD_STATE_RES_NEED_THREAD);
3404                         break;
3405
3406                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3407                         res = scst_mode_select_checks(cmd);
3408                         break;
3409
3410                 case SCST_CMD_STATE_DEV_DONE:
3411                         res = scst_dev_done(cmd);
3412                         break;
3413
3414                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3415                         res = scst_pre_xmit_response(cmd);
3416                         EXTRACHECKS_BUG_ON(res ==
3417                                 SCST_CMD_STATE_RES_NEED_THREAD);
3418                         break;
3419
3420                 case SCST_CMD_STATE_XMIT_RESP:
3421                         res = scst_xmit_response(cmd);
3422                         break;
3423
3424                 case SCST_CMD_STATE_FINISHED:
3425                         res = scst_finish_cmd(cmd);
3426                         EXTRACHECKS_BUG_ON(res ==
3427                                 SCST_CMD_STATE_RES_NEED_THREAD);
3428                         break;
3429
3430                 case SCST_CMD_STATE_FINISHED_INTERNAL:
3431                         res = scst_finish_internal_cmd(cmd);
3432                         EXTRACHECKS_BUG_ON(res ==
3433                                 SCST_CMD_STATE_RES_NEED_THREAD);
3434                         break;
3435
3436                 default:
3437                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3438                                 "be", cmd, cmd->state);
3439                         sBUG();
3440                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3441                         break;
3442                 }
3443         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3444
3445         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3446                 /* None */
3447         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3448                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3449                 switch (cmd->state) {
3450                 case SCST_CMD_STATE_PRE_PARSE:
3451                 case SCST_CMD_STATE_DEV_PARSE:
3452                 case SCST_CMD_STATE_PREPARE_SPACE:
3453                 case SCST_CMD_STATE_RDY_TO_XFER:
3454                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3455                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3456                 case SCST_CMD_STATE_LOCAL_EXEC:
3457                 case SCST_CMD_STATE_REAL_EXEC:
3458                 case SCST_CMD_STATE_PRE_DEV_DONE:
3459                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3460                 case SCST_CMD_STATE_DEV_DONE:
3461                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3462                 case SCST_CMD_STATE_XMIT_RESP:
3463                 case SCST_CMD_STATE_FINISHED:
3464                 case SCST_CMD_STATE_FINISHED_INTERNAL:
3465                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3466                                   cmd);
3467                         list_add(&cmd->cmd_list_entry,
3468                                 &cmd->cmd_lists->active_cmd_list);
3469                         break;
3470 #ifdef CONFIG_SCST_EXTRACHECKS
3471                 /* not very valid commands */
3472                 case SCST_CMD_STATE_DEFAULT:
3473                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3474                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3475                                 cmd->state);
3476                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3477                         sBUG();
3478                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3479                         break;
3480 #endif
3481                 default:
3482                         break;
3483                 }
3484                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3485                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3486         } else
3487                 sBUG();
3488
3489         TRACE_EXIT();
3490         return;
3491 }
3492 EXPORT_SYMBOL(scst_process_active_cmd);
3493
3494 /* Called under cmd_list_lock and IRQs disabled */
3495 static void scst_do_job_active(struct list_head *cmd_list,
3496         spinlock_t *cmd_list_lock, bool atomic)
3497         __releases(cmd_list_lock)
3498         __acquires(cmd_list_lock)
3499 {
3500         TRACE_ENTRY();