- ISCSI sending response timeout increased to 30 seconds
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23
24 #define LOG_PREFIX              DEV_USER_NAME
25
26 #include "scst.h"
27 #include "scst_user.h"
28 #include "scst_dev_handler.h"
29
30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
31 #warning "HIGHMEM kernel configurations are not supported by this module, \
32         because nowadays it doesn't worth the effort. Consider change \
33         VMSPLIT option or use 64-bit configuration instead. See README file \
34         for details."
35 #endif
36
37 #define DEV_USER_MAJOR                  237
38
39 #define DEV_USER_CMD_HASH_ORDER         6
40
41 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
42
43 struct scst_user_dev {
44         struct rw_semaphore dev_rwsem;
45
46         struct scst_cmd_lists cmd_lists;
47
48         /* Protected by cmd_lists.cmd_list_lock */
49         struct list_head ready_cmd_list;
50
51         /* Protected by dev_rwsem or don't need any protection */
52         unsigned int blocking:1;
53         unsigned int cleanup_done:1;
54         unsigned int tst:3;
55         unsigned int queue_alg:4;
56         unsigned int tas:1;
57         unsigned int swp:1;
58         unsigned int has_own_order_mgmt:1;
59
60         int (*generic_parse)(struct scst_cmd *cmd,
61                 int (*get_block)(struct scst_cmd *cmd));
62
63         int block;
64         int def_block;
65
66         struct scst_mem_lim udev_mem_lim;
67         struct sgv_pool *pool;
68
69         uint8_t parse_type;
70         uint8_t on_free_cmd_type;
71         uint8_t memory_reuse_type;
72         uint8_t partial_transfers_type;
73         uint32_t partial_len;
74
75         struct scst_dev_type devtype;
76
77         /* Both protected by cmd_lists.cmd_list_lock */
78         unsigned int handle_counter;
79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
80
81         struct scst_device *sdev;
82
83         int virt_id;
84         struct list_head dev_list_entry;
85         char name[SCST_MAX_NAME];
86
87         struct list_head cleanup_list_entry;
88         /* ToDo: make it on-stack */
89         struct completion cleanup_cmpl;
90 };
91
92 /* Most fields are unprotected, since only one thread at time can access them */
93 struct scst_user_cmd {
94         struct scst_cmd *cmd;
95         struct scst_user_dev *dev;
96
97         atomic_t ucmd_ref;
98
99         unsigned int buff_cached:1;
100         unsigned int buf_dirty:1;
101         unsigned int background_exec:1;
102         unsigned int aborted:1;
103
104         struct scst_user_cmd *buf_ucmd;
105
106         int cur_data_page;
107         int num_data_pages;
108         int first_page_offset;
109         unsigned long ubuff;
110         struct page **data_pages;
111         struct sgv_pool_obj *sgv;
112
113         /*
114          * Special flags, which can be accessed asynchronously (hence "long").
115          * Protected by cmd_lists.cmd_list_lock.
116          */
117         unsigned long sent_to_user:1;
118         unsigned long jammed:1;
119         unsigned long this_state_unjammed:1;
120         unsigned long seen_by_user:1; /* here only as a small optimization */
121
122         unsigned int state;
123
124         struct list_head ready_cmd_list_entry;
125
126         unsigned int h;
127         struct list_head hash_list_entry;
128
129         struct scst_user_get_cmd user_cmd;
130
131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
132         union {
133                 struct completion *cmpl;
134                 struct scst_mgmt_cmd *mcmd;
135         };
136         int result;
137 };
138
139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
140         int gfp_mask);
141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
142
143 static int dev_user_parse(struct scst_cmd *cmd);
144 static int dev_user_exec(struct scst_cmd *cmd);
145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
147         struct scst_tgt_dev *tgt_dev);
148
149 static int dev_user_disk_done(struct scst_cmd *cmd);
150 static int dev_user_tape_done(struct scst_cmd *cmd);
151
152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
153         gfp_t gfp_mask, void *priv);
154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
155                                      void *priv);
156
157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
158
159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
160         unsigned long *flags);
161
162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
164         int status);
165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
166 static int dev_user_register_dev(struct file *file,
167         const struct scst_user_dev_desc *dev_desc);
168 static int __dev_user_set_opt(struct scst_user_dev *dev,
169         const struct scst_user_opt *opt);
170 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
171 static int dev_user_get_opt(struct file *file, void *arg);
172
173 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
174 static long dev_user_ioctl(struct file *file, unsigned int cmd,
175         unsigned long arg);
176 static int dev_user_release(struct inode *inode, struct file *file);
177
178 /** Data **/
179
180 static struct kmem_cache *user_cmd_cachep;
181
182 static DEFINE_MUTEX(dev_priv_mutex);
183
184 static struct file_operations dev_user_fops = {
185         .poll           = dev_user_poll,
186         .unlocked_ioctl = dev_user_ioctl,
187 #ifdef CONFIG_COMPAT
188         .compat_ioctl   = dev_user_ioctl,
189 #endif
190         .release        = dev_user_release,
191 };
192
193 static struct class *dev_user_sysfs_class;
194
195 static DEFINE_SPINLOCK(dev_list_lock);
196 static LIST_HEAD(dev_list);
197
198 static DEFINE_SPINLOCK(cleanup_lock);
199 static LIST_HEAD(cleanup_list);
200 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
201 static struct task_struct *cleanup_thread;
202
203 /*
204  * Skip this command if result is not 0. Must be called under
205  * cmd_lists.cmd_list_lock and IRQ off.
206  */
207 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
208 {
209         int r = atomic_inc_return(&ucmd->ucmd_ref);
210         int res;
211         if (unlikely(r == 1)) {
212                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
213                 atomic_dec(&ucmd->ucmd_ref);
214                 res = true;
215                 /*
216                  * Necessary code is serialized by cmd_list_lock in
217                  * cmd_remove_hash()
218                  */
219         } else {
220                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
221                         atomic_read(&ucmd->ucmd_ref));
222                 res = false;
223         }
224         return res;
225 }
226
227 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
228 {
229         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
230         atomic_inc(&ucmd->ucmd_ref);
231         if (barrier)
232                 smp_mb__after_atomic_inc();
233 }
234
235 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
236 {
237         __ucmd_get(ucmd, true);
238 }
239
240 static inline void ucmd_get(struct scst_user_cmd *ucmd)
241 {
242         __ucmd_get(ucmd, false);
243 }
244
245 /* Must not be called under cmd_list_lock!! */
246 static inline void ucmd_put(struct scst_user_cmd *ucmd)
247 {
248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
249
250         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
251
252         if (atomic_dec_and_test(&ucmd->ucmd_ref))
253                 dev_user_free_ucmd(ucmd);
254 }
255
256 static inline int calc_num_pg(unsigned long buf, int len)
257 {
258         len += buf & ~PAGE_MASK;
259         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
260 }
261
262 static inline int is_need_offs_page(unsigned long buf, int len)
263 {
264         return ((buf & ~PAGE_MASK) != 0) &&
265                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
266 }
267
268 static void __dev_user_not_reg(void)
269 {
270         PRINT_ERROR("%s", "Device not registered");
271         return;
272 }
273
274 static inline int dev_user_check_reg(struct scst_user_dev *dev)
275 {
276         if (dev == NULL) {
277                 __dev_user_not_reg();
278                 return -EINVAL;
279         }
280         return 0;
281 }
282
283 static inline int scst_user_cmd_hashfn(int h)
284 {
285         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
286 }
287
288 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
289         unsigned int h)
290 {
291         struct list_head *head;
292         struct scst_user_cmd *ucmd;
293
294         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
295         list_for_each_entry(ucmd, head, hash_list_entry) {
296                 if (ucmd->h == h) {
297                         TRACE_DBG("Found ucmd %p", ucmd);
298                         return ucmd;
299                 }
300         }
301         return NULL;
302 }
303
304 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
305 {
306         struct list_head *head;
307         struct scst_user_dev *dev = ucmd->dev;
308         struct scst_user_cmd *u;
309         unsigned long flags;
310
311         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
312         do {
313                 ucmd->h = dev->handle_counter++;
314                 u = __ucmd_find_hash(dev, ucmd->h);
315         } while (u != NULL);
316         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
317         list_add_tail(&ucmd->hash_list_entry, head);
318         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
319
320         TRACE_DBG("Inserted ucmd %p, h=%d", ucmd, ucmd->h);
321         return;
322 }
323
324 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
325 {
326         unsigned long flags;
327
328         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
329         list_del(&ucmd->hash_list_entry);
330         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
331
332         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
333         return;
334 }
335
336 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
337 {
338         TRACE_ENTRY();
339
340         TRACE_MEM("Freeing ucmd %p", ucmd);
341
342         cmd_remove_hash(ucmd);
343         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
344
345         kmem_cache_free(user_cmd_cachep, ucmd);
346
347         TRACE_EXIT();
348         return;
349 }
350
351 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
352         gfp_t gfp_mask, void *priv)
353 {
354         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
355         int offset = 0;
356
357         TRACE_ENTRY();
358
359         /* *sg supposed to be zeroed */
360
361         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
362                 ucmd->ubuff, ucmd->cur_data_page);
363
364         if (ucmd->cur_data_page == 0) {
365                 TRACE_MEM("ucmd->first_page_offset %d",
366                         ucmd->first_page_offset);
367                 offset = ucmd->first_page_offset;
368                 ucmd_get(ucmd);
369         }
370
371         if (ucmd->cur_data_page >= ucmd->num_data_pages)
372                 goto out;
373
374         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
375                 PAGE_SIZE - offset, offset);
376         ucmd->cur_data_page++;
377
378         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
379                 sg->offset);
380         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
381
382 out:
383         TRACE_EXIT();
384         return sg_page(sg);
385 }
386
387 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
388 {
389         TRACE_ENTRY();
390
391         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
392                 ucmd, ucmd->h, ucmd->ubuff);
393
394         ucmd->user_cmd.cmd_h = ucmd->h;
395         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
396         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
397
398         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
399
400         dev_user_add_to_ready(ucmd);
401
402         TRACE_EXIT();
403         return;
404 }
405
406 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
407 {
408         int i;
409
410         TRACE_ENTRY();
411
412         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
413                 ucmd->ubuff, ucmd->num_data_pages);
414
415         for (i = 0; i < ucmd->num_data_pages; i++) {
416                 struct page *page = ucmd->data_pages[i];
417
418                 if (ucmd->buf_dirty)
419                         SetPageDirty(page);
420
421                 page_cache_release(page);
422         }
423
424         kfree(ucmd->data_pages);
425         ucmd->data_pages = NULL;
426
427         TRACE_EXIT();
428         return;
429 }
430
431 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
432 {
433         TRACE_ENTRY();
434
435         sBUG_ON(ucmd->data_pages == NULL);
436
437         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
438                 ucmd, ucmd->ubuff, ucmd->buff_cached);
439
440         dev_user_unmap_buf(ucmd);
441
442         if (ucmd->buff_cached)
443                 dev_user_on_cached_mem_free(ucmd);
444         else
445                 ucmd_put(ucmd);
446
447         TRACE_EXIT();
448         return;
449 }
450
451 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
452         void *priv)
453 {
454         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
455
456         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
457                 sg_count, ucmd);
458
459         __dev_user_free_sg_entries(ucmd);
460
461         return;
462 }
463
464 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
465 {
466         int mem_reuse_type = ucmd->dev->memory_reuse_type;
467
468         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
469             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
470              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
471             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
472              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
473                 return 1;
474         else
475                 return 0;
476 }
477
478 /*
479  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
480  * Unmaps the buffer, if needed in case of error
481  */
482 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
483 {
484         int res = 0;
485         struct scst_cmd *cmd = ucmd->cmd;
486         struct scst_user_dev *dev = ucmd->dev;
487         int gfp_mask, flags = 0;
488         int bufflen = cmd->bufflen;
489         int last_len = 0;
490
491         TRACE_ENTRY();
492
493         gfp_mask = __GFP_NOWARN;
494         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
495
496         if (cached_buff) {
497                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
498                 if (ucmd->ubuff == 0)
499                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
500         } else {
501                 TRACE_MEM("%s", "Not cached buff");
502                 flags |= SCST_POOL_ALLOC_NO_CACHED;
503                 if (ucmd->ubuff == 0) {
504                         res = 1;
505                         goto out;
506                 }
507                 bufflen += ucmd->first_page_offset;
508                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
509                         last_len = bufflen & ~PAGE_MASK;
510                 else
511                         last_len = cmd->bufflen & ~PAGE_MASK;
512                 if (last_len == 0)
513                         last_len = PAGE_SIZE;
514         }
515         ucmd->buff_cached = cached_buff;
516
517         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
518                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
519         if (cmd->sg != NULL) {
520                 struct scst_user_cmd *buf_ucmd =
521                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
522
523                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
524
525                 ucmd->ubuff = buf_ucmd->ubuff;
526                 ucmd->buf_ucmd = buf_ucmd;
527
528                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
529                                    (ucmd != buf_ucmd));
530
531                 if (last_len != 0) {
532                         /* We don't use clustering, so the assignment is safe */
533                         cmd->sg[cmd->sg_cnt-1].length = last_len;
534                 }
535
536                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
537                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
538                         last_len, cmd->sg[cmd->sg_cnt-1].length);
539
540                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
541                         static int ll;
542                         if (ll < 10) {
543                                 PRINT_INFO("Unable to complete command due to "
544                                         "SG IO count limitation (requested %d, "
545                                         "available %d, tgt lim %d)", cmd->sg_cnt,
546                                         cmd->tgt_dev->max_sg_cnt,
547                                         cmd->tgt->sg_tablesize);
548                                 ll++;
549                         }
550                         cmd->sg = NULL;
551                         /* sgv will be freed in dev_user_free_sgv() */
552                         res = -1;
553                 }
554         } else {
555                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
556                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
557                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
558                 if (unlikely(cmd->sg_cnt == 0)) {
559                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
560                         sBUG_ON(ucmd->sgv != NULL);
561                         res = -1;
562                 } else {
563                         switch (ucmd->state) {
564                         case UCMD_STATE_BUF_ALLOCING:
565                                 res = 1;
566                                 break;
567                         case UCMD_STATE_EXECING:
568                                 res = -1;
569                                 break;
570                         default:
571                                 sBUG();
572                                 break;
573                         }
574                 }
575         }
576
577 out:
578         TRACE_EXIT_RES(res);
579         return res;
580 }
581
582 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
583 {
584         int rc, res = SCST_CMD_STATE_DEFAULT;
585         struct scst_cmd *cmd = ucmd->cmd;
586
587         TRACE_ENTRY();
588
589         if (unlikely(ucmd->cmd->data_buf_tgt_alloc)) {
590                 PRINT_ERROR("Target driver %s requested own memory "
591                         "allocation", ucmd->cmd->tgtt->name);
592                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
593                 res = scst_get_cmd_abnormal_done_state(cmd);
594                 goto out;
595         }
596
597         ucmd->state = UCMD_STATE_BUF_ALLOCING;
598         cmd->data_buf_alloced = 1;
599
600         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
601         if (rc == 0)
602                 goto out;
603         else if (rc < 0) {
604                 scst_set_busy(cmd);
605                 res = scst_get_cmd_abnormal_done_state(cmd);
606                 goto out;
607         }
608
609         if ((cmd->data_direction != SCST_DATA_WRITE) &&
610             !scst_is_cmd_local(cmd)) {
611                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
612                 goto out;
613         }
614
615         ucmd->user_cmd.cmd_h = ucmd->h;
616         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
617         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
618         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
619                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
620         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
621         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
622                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
623         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
624         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
625         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
626
627         dev_user_add_to_ready(ucmd);
628
629         res = SCST_CMD_STATE_STOP;
630
631 out:
632         TRACE_EXIT_RES(res);
633         return res;
634 }
635
636 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
637         int gfp_mask)
638 {
639         struct scst_user_cmd *ucmd = NULL;
640
641         TRACE_ENTRY();
642
643 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
644         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
645         if (ucmd != NULL)
646                 memset(ucmd, 0, sizeof(*ucmd));
647 #else
648         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
649 #endif
650         if (unlikely(ucmd == NULL)) {
651                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
652                         "user cmd (gfp_mask %x)", gfp_mask);
653                 goto out;
654         }
655         ucmd->dev = dev;
656         atomic_set(&ucmd->ucmd_ref, 1);
657
658         cmd_insert_hash(ucmd);
659
660         TRACE_MEM("ucmd %p allocated", ucmd);
661
662 out:
663         TRACE_EXIT_HRES((unsigned long)ucmd);
664         return ucmd;
665 }
666
667 static int dev_user_get_block(struct scst_cmd *cmd)
668 {
669         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
670         /*
671          * No need for locks here, since *_detach() can not be
672          * called, when there are existing commands.
673          */
674         TRACE_EXIT_RES(dev->block);
675         return dev->block;
676 }
677
678 static int dev_user_parse(struct scst_cmd *cmd)
679 {
680         int rc, res = SCST_CMD_STATE_DEFAULT;
681         struct scst_user_cmd *ucmd;
682         int atomic = scst_cmd_atomic(cmd);
683         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
684         int gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
685
686         TRACE_ENTRY();
687
688         if (cmd->dh_priv == NULL) {
689                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
690                 if (unlikely(ucmd == NULL)) {
691                         if (atomic) {
692                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
693                                 goto out;
694                         } else {
695                                 scst_set_busy(cmd);
696                                 goto out_error;
697                         }
698                 }
699                 ucmd->cmd = cmd;
700                 cmd->dh_priv = ucmd;
701         } else {
702                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
703                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
704         }
705
706         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
707
708         if (ucmd->state != UCMD_STATE_NEW)
709                 goto alloc;
710
711         switch (dev->parse_type) {
712         case SCST_USER_PARSE_STANDARD:
713                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
714                 rc = dev->generic_parse(cmd, dev_user_get_block);
715                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
716                         goto out_invalid;
717                 break;
718
719         case SCST_USER_PARSE_EXCEPTION:
720                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
721                 rc = dev->generic_parse(cmd, dev_user_get_block);
722                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
723                         break;
724                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
725                         TRACE_MEM("Restarting PARSE to thread context "
726                                 "(ucmd %p)", ucmd);
727                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
728                         goto out;
729                 }
730                 /* else go through */
731
732         case SCST_USER_PARSE_CALL:
733                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
734                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
735                 ucmd->user_cmd.cmd_h = ucmd->h;
736                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
737                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
738                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
739                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
740                             sizeof(cmd->cdb)));
741                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
742                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
743                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
744                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
745                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
746                 ucmd->user_cmd.parse_cmd.expected_values_set =
747                                         cmd->expected_values_set;
748                 ucmd->user_cmd.parse_cmd.expected_data_direction =
749                                         cmd->expected_data_direction;
750                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
751                                         cmd->expected_transfer_len;
752                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
753                 ucmd->state = UCMD_STATE_PARSING;
754                 dev_user_add_to_ready(ucmd);
755                 res = SCST_CMD_STATE_STOP;
756                 goto out;
757
758         default:
759                 sBUG();
760                 goto out;
761         }
762
763 alloc:
764         if (cmd->data_direction != SCST_DATA_NONE)
765                 res = dev_user_alloc_space(ucmd);
766
767 out:
768         TRACE_EXIT_RES(res);
769         return res;
770
771 out_invalid:
772         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
773                 cmd->op_flags & SCST_INFO_INVALID);
774         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
775
776 out_error:
777         res = scst_get_cmd_abnormal_done_state(cmd);
778         goto out;
779 }
780
781 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
782 {
783         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
784         unsigned long start = buf_ucmd->ubuff;
785         int i;
786
787         TRACE_ENTRY();
788
789         if (start == 0)
790                 goto out;
791
792         for (i = 0; i < buf_ucmd->num_data_pages; i++) {
793                 struct page *page;
794                 page = buf_ucmd->data_pages[i];
795 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
796                 struct vm_area_struct *vma = find_vma(current->mm, start);
797                 if (vma != NULL)
798                         flush_anon_page(vma, page, start);
799 #endif
800                 flush_dcache_page(page);
801                 start += PAGE_SIZE;
802         }
803
804 out:
805         TRACE_EXIT();
806         return;
807 }
808
809 static int dev_user_exec(struct scst_cmd *cmd)
810 {
811         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
812         int res = SCST_EXEC_COMPLETED;
813
814         TRACE_ENTRY();
815
816 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
817        * context to complete the necessary actions, but all we are going to
818        * do in this function is, in fact, atomic, so let's skip this check.
819        */
820         if (scst_cmd_atomic(cmd)) {
821                 TRACE_DBG("%s", "User exec() can not be called in atomic "
822                         "context, rescheduling to the thread");
823                 res = SCST_EXEC_NEED_THREAD;
824                 goto out;
825         }
826 #endif
827
828         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
829                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
830                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
831
832         if (cmd->data_direction == SCST_DATA_WRITE)
833                 dev_user_flush_dcache(ucmd);
834
835         ucmd->user_cmd.cmd_h = ucmd->h;
836         ucmd->user_cmd.subcode = SCST_USER_EXEC;
837         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
838         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
839                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
840                     sizeof(cmd->cdb)));
841         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
842         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
843         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
844         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
845         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
846                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
847                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
848         }
849         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
850         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
851         ucmd->user_cmd.exec_cmd.partial = 0;
852         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
853         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
854
855         ucmd->state = UCMD_STATE_EXECING;
856
857         dev_user_add_to_ready(ucmd);
858
859         TRACE_EXIT_RES(res);
860         return res;
861 }
862
863 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
864 {
865         if (ucmd->sgv != NULL) {
866                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
867                 ucmd->sgv = NULL;
868         } else if (ucmd->data_pages != NULL) {
869                 /* We mapped pages, but for some reason didn't allocate them */
870                 ucmd_get(ucmd);
871                 __dev_user_free_sg_entries(ucmd);
872         }
873         return;
874 }
875
876 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
877 {
878         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
879
880         TRACE_ENTRY();
881
882         if (unlikely(ucmd == NULL))
883                 goto out;
884
885         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
886                 ucmd->buff_cached, ucmd->ubuff);
887
888         ucmd->cmd = NULL;
889         if ((cmd->data_direction == SCST_DATA_WRITE) && (ucmd->buf_ucmd != NULL))
890                 ucmd->buf_ucmd->buf_dirty = 1;
891
892         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
893                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
894                 /* The state assignment must be before freeing sgv! */
895                 goto out_reply;
896         }
897
898         if (unlikely(!ucmd->seen_by_user)) {
899                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
900                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
901                 goto out_reply;
902         }
903
904         ucmd->user_cmd.cmd_h = ucmd->h;
905         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
906
907         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
908         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
909         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
910         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
911         ucmd->user_cmd.on_free_cmd.status = cmd->status;
912         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
913
914         ucmd->state = UCMD_STATE_ON_FREEING;
915
916         dev_user_add_to_ready(ucmd);
917
918 out:
919         TRACE_EXIT();
920         return;
921
922 out_reply:
923         dev_user_process_reply_on_free(ucmd);
924         goto out;
925 }
926
927 static void dev_user_set_block(struct scst_cmd *cmd, int block)
928 {
929         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
930         /*
931          * No need for locks here, since *_detach() can not be
932          * called, when there are existing commands.
933          */
934         TRACE_DBG("dev %p, new block %d", dev, block);
935         if (block != 0)
936                 dev->block = block;
937         else
938                 dev->block = dev->def_block;
939         return;
940 }
941
942 static int dev_user_disk_done(struct scst_cmd *cmd)
943 {
944         int res = SCST_CMD_STATE_DEFAULT;
945
946         TRACE_ENTRY();
947
948         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
949
950         TRACE_EXIT_RES(res);
951         return res;
952 }
953
954 static int dev_user_tape_done(struct scst_cmd *cmd)
955 {
956         int res = SCST_CMD_STATE_DEFAULT;
957
958         TRACE_ENTRY();
959
960         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
961
962         TRACE_EXIT_RES(res);
963         return res;
964 }
965
966 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
967 {
968         struct scst_user_dev *dev = ucmd->dev;
969         unsigned long flags;
970         int do_wake;
971
972         TRACE_ENTRY();
973
974         do_wake = (in_interrupt() ||
975                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
976         if (ucmd->cmd)
977                 do_wake |= ucmd->cmd->preprocessing_only;
978
979         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
980
981         ucmd->this_state_unjammed = 0;
982
983         if ((ucmd->state == UCMD_STATE_PARSING) ||
984             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
985                 /*
986                  * If we don't put such commands in the queue head, then under
987                  * high load we might delay threads, waiting for memory
988                  * allocations, for too long and start loosing NOPs, which
989                  * would lead to consider us by remote initiators as
990                  * unresponsive and stuck => broken connections, etc. If none
991                  * of our commands completed in NOP timeout to allow the head
992                  * commands to go, then we are really overloaded and/or stuck.
993                  */
994                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
995                         "cmd list", ucmd, ucmd->state);
996                 list_add(&ucmd->ready_cmd_list_entry,
997                         &dev->ready_cmd_list);
998                 do_wake = 1;
999         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1000                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1001                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1002                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1003                         "ready cmd list", ucmd, ucmd->state);
1004                 list_add(&ucmd->ready_cmd_list_entry,
1005                         &dev->ready_cmd_list);
1006                 do_wake = 1;
1007         } else if ((ucmd->cmd != NULL) &&
1008                    unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1009                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1010                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1011         } else {
1012                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1013                 list_add_tail(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1014         }
1015
1016         if (do_wake) {
1017                 TRACE_DBG("Waking up dev %p", dev);
1018                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1019         }
1020
1021         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1022
1023         TRACE_EXIT();
1024         return;
1025 }
1026
1027 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1028         int num_pg)
1029 {
1030         int res = 0, rc;
1031         int i;
1032
1033         TRACE_ENTRY();
1034
1035         if (unlikely(ubuff == 0))
1036                 goto out_nomem;
1037
1038         sBUG_ON(ucmd->data_pages != NULL);
1039
1040         ucmd->num_data_pages = num_pg;
1041
1042         ucmd->data_pages = kzalloc(sizeof(*ucmd->data_pages)*ucmd->num_data_pages,
1043                 GFP_KERNEL);
1044         if (ucmd->data_pages == NULL) {
1045                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1046                         "(num_data_pages=%d)", ucmd->num_data_pages);
1047                 res = -ENOMEM;
1048                 goto out_nomem;
1049         }
1050
1051         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d, "
1052                 "first_page_offset %d, len %d)", ucmd, ubuff,
1053                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1054                 ucmd->cmd->bufflen);
1055
1056         down_read(&current->mm->mmap_sem);
1057         rc = get_user_pages(current, current->mm, ubuff, ucmd->num_data_pages,
1058                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1059         up_read(&current->mm->mmap_sem);
1060
1061         /* get_user_pages() flushes dcache */
1062
1063         if (rc < ucmd->num_data_pages)
1064                 goto out_unmap;
1065
1066         ucmd->ubuff = ubuff;
1067         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1068
1069 out:
1070         TRACE_EXIT_RES(res);
1071         return res;
1072
1073 out_nomem:
1074         scst_set_busy(ucmd->cmd);
1075         /* go through */
1076
1077 out_err:
1078         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1079         goto out;
1080
1081 out_unmap:
1082         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1083                 ucmd->num_data_pages, rc);
1084         if (rc > 0) {
1085                 for (i = 0; i < rc; i++)
1086                         page_cache_release(ucmd->data_pages[i]);
1087         }
1088         kfree(ucmd->data_pages);
1089         ucmd->data_pages = NULL;
1090         res = -EFAULT;
1091         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1092         goto out_err;
1093 }
1094
1095 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1096         struct scst_user_reply_cmd *reply)
1097 {
1098         int res = 0;
1099         struct scst_cmd *cmd = ucmd->cmd;
1100
1101         TRACE_ENTRY();
1102
1103         TRACE_DBG("ucmd %p, pbuf %Lx", ucmd, reply->alloc_reply.pbuf);
1104
1105         if (likely(reply->alloc_reply.pbuf != 0)) {
1106                 int pages;
1107                 if (ucmd->buff_cached) {
1108                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1109                                 PRINT_ERROR("Supplied pbuf %Lx isn't "
1110                                         "page aligned", reply->alloc_reply.pbuf);
1111                                 goto out_hwerr;
1112                         }
1113                         pages = cmd->sg_cnt;
1114                 } else
1115                         pages = calc_num_pg(reply->alloc_reply.pbuf, cmd->bufflen);
1116                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1117         } else {
1118                 scst_set_busy(ucmd->cmd);
1119                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1120         }
1121
1122 out_process:
1123         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1124
1125         TRACE_EXIT_RES(res);
1126         return res;
1127
1128 out_hwerr:
1129         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1130         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1131         res = -EINVAL;
1132         goto out_process;
1133 }
1134
1135 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1136         struct scst_user_reply_cmd *reply)
1137 {
1138         int res = 0;
1139         struct scst_user_scsi_cmd_reply_parse *preply =
1140                 &reply->parse_reply;
1141         struct scst_cmd *cmd = ucmd->cmd;
1142
1143         TRACE_ENTRY();
1144
1145         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1146                 goto out_inval;
1147
1148         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1149                      (preply->data_direction != SCST_DATA_READ) &&
1150                      (preply->data_direction != SCST_DATA_NONE)))
1151                 goto out_inval;
1152
1153         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1154                      (preply->bufflen == 0)))
1155                 goto out_inval;
1156
1157         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1158                 goto out_inval;
1159
1160         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1161                 "data_len %d, pbuf %Lx", ucmd, preply->queue_type,
1162                 preply->data_direction, preply->bufflen, preply->data_len,
1163                 reply->alloc_reply.pbuf);
1164
1165         cmd->queue_type = preply->queue_type;
1166         cmd->data_direction = preply->data_direction;
1167         cmd->bufflen = preply->bufflen;
1168         cmd->data_len = preply->data_len;
1169
1170 out_process:
1171         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1172
1173         TRACE_EXIT_RES(res);
1174         return res;
1175
1176 out_inval:
1177         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1178                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1179         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1180         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1181         scst_set_cmd_abnormal_done_state(cmd);
1182         res = -EINVAL;
1183         goto out_process;
1184 }
1185
1186 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1187 {
1188         int res = 0;
1189
1190         TRACE_ENTRY();
1191
1192         TRACE_DBG("ON FREE ucmd %p", ucmd);
1193
1194         dev_user_free_sgv(ucmd);
1195         ucmd_put(ucmd);
1196
1197         TRACE_EXIT_RES(res);
1198         return res;
1199 }
1200
1201 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1202 {
1203         int res = 0;
1204
1205         TRACE_ENTRY();
1206
1207         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1208
1209         ucmd_put(ucmd);
1210
1211         TRACE_EXIT_RES(res);
1212         return res;
1213 }
1214
1215 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1216         struct scst_user_reply_cmd *reply)
1217 {
1218         int res = 0;
1219         struct scst_user_scsi_cmd_reply_exec *ereply =
1220                 &reply->exec_reply;
1221         struct scst_cmd *cmd = ucmd->cmd;
1222
1223         TRACE_ENTRY();
1224
1225         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1226                 if (ucmd->background_exec) {
1227                         TRACE_DBG("Background ucmd %p finished", ucmd);
1228                         ucmd_put(ucmd);
1229                         goto out;
1230                 }
1231                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1232                         goto out_inval;
1233                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1234                              (ereply->resp_data_len != 0)))
1235                         goto out_inval;
1236         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1237                 if (unlikely(ucmd->background_exec))
1238                         goto out_inval;
1239                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1240                              (cmd->resp_data_len != 0)))
1241                         goto out_inval;
1242                 ucmd_get_ordered(ucmd);
1243                 ucmd->background_exec = 1;
1244                 TRACE_DBG("Background ucmd %p", ucmd);
1245                 goto out_compl;
1246         } else
1247                 goto out_inval;
1248
1249         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1250                 ereply->status, ereply->resp_data_len);
1251
1252          if (ereply->resp_data_len != 0) {
1253                 if (ucmd->ubuff == 0) {
1254                         int pages, rc;
1255                         if (unlikely(ereply->pbuf == 0))
1256                                 goto out_busy;
1257                         if (ucmd->buff_cached) {
1258                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1259                                         PRINT_ERROR("Supplied pbuf %Lx isn't "
1260                                                 "page aligned", ereply->pbuf);
1261                                         goto out_hwerr;
1262                                 }
1263                                 pages = cmd->sg_cnt;
1264                         } else
1265                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1266                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1267                         if ((rc != 0) || (ucmd->ubuff == 0))
1268                                 goto out_compl;
1269
1270                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1271                         if (unlikely(rc != 0))
1272                                 goto out_busy;
1273                 } else
1274                         dev_user_flush_dcache(ucmd);
1275                 cmd->may_need_dma_sync = 1;
1276                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1277         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1278                 if (ucmd->ubuff == 0)
1279                         cmd->resp_data_len = ereply->resp_data_len;
1280                 else
1281                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1282         }
1283
1284         cmd->status = ereply->status;
1285         if (ereply->sense_len != 0) {
1286                 res = scst_alloc_sense(cmd, 0);
1287                 if (res != 0)
1288                         goto out_compl;
1289                 res = copy_from_user(cmd->sense,
1290                         (void *)(unsigned long)ereply->psense_buffer,
1291                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1292                                 (unsigned int)ereply->sense_len));
1293                 if (res < 0) {
1294                         PRINT_ERROR("%s", "Unable to get sense data");
1295                         goto out_hwerr_res_set;
1296                 }
1297         }
1298
1299 out_compl:
1300         cmd->completed = 1;
1301         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1302         /* !! At this point cmd can be already freed !! */
1303
1304 out:
1305         TRACE_EXIT_RES(res);
1306         return res;
1307
1308 out_inval:
1309         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1310                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1311         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1312
1313 out_hwerr:
1314         res = -EINVAL;
1315
1316 out_hwerr_res_set:
1317         if (ucmd->background_exec) {
1318                 ucmd_put(ucmd);
1319                 goto out;
1320         } else {
1321                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1322                 goto out_compl;
1323         }
1324
1325 out_busy:
1326         scst_set_busy(cmd);
1327         goto out_compl;
1328 }
1329
1330 static int dev_user_process_reply(struct scst_user_dev *dev,
1331         struct scst_user_reply_cmd *reply)
1332 {
1333         int res = 0;
1334         struct scst_user_cmd *ucmd;
1335         int state;
1336
1337         TRACE_ENTRY();
1338
1339         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1340
1341         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1342         if (unlikely(ucmd == NULL)) {
1343                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1344                 res = -ESRCH;
1345                 goto out_unlock;
1346         }
1347
1348         if (unlikely(ucmd_get_check(ucmd))) {
1349                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1350                 res = -ESRCH;
1351                 goto out_unlock;
1352         }
1353
1354         if (ucmd->background_exec) {
1355                 state = UCMD_STATE_EXECING;
1356                 goto unlock_process;
1357         }
1358
1359         if (unlikely(ucmd->this_state_unjammed)) {
1360                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1361                         ucmd);
1362                 goto out_unlock_put;
1363         }
1364
1365         if (unlikely(!ucmd->sent_to_user)) {
1366                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1367                         "state %x", ucmd, ucmd->state);
1368                 res = -EINVAL;
1369                 goto out_unlock_put;
1370         }
1371
1372         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1373                 goto out_wrong_state;
1374
1375         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1376                 goto out_wrong_state;
1377
1378         state = ucmd->state;
1379         ucmd->sent_to_user = 0;
1380
1381 unlock_process:
1382         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1383
1384         switch (state) {
1385         case UCMD_STATE_PARSING:
1386                 res = dev_user_process_reply_parse(ucmd, reply);
1387                 break;
1388
1389         case UCMD_STATE_BUF_ALLOCING:
1390                 res = dev_user_process_reply_alloc(ucmd, reply);
1391                 break;
1392
1393         case UCMD_STATE_EXECING:
1394                 res = dev_user_process_reply_exec(ucmd, reply);
1395                 break;
1396
1397         case UCMD_STATE_ON_FREEING:
1398                 res = dev_user_process_reply_on_free(ucmd);
1399                 break;
1400
1401         case UCMD_STATE_ON_CACHE_FREEING:
1402                 res = dev_user_process_reply_on_cache_free(ucmd);
1403                 break;
1404
1405         case UCMD_STATE_TM_EXECING:
1406                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1407                 break;
1408
1409         case UCMD_STATE_ATTACH_SESS:
1410         case UCMD_STATE_DETACH_SESS:
1411                 res = dev_user_process_reply_sess(ucmd, reply->result);
1412                 break;
1413
1414         default:
1415                 sBUG();
1416                 break;
1417         }
1418
1419 out_put:
1420         ucmd_put(ucmd);
1421
1422 out:
1423         TRACE_EXIT_RES(res);
1424         return res;
1425
1426 out_wrong_state:
1427         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1428                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1429                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1430                 reply->subcode, ucmd->user_cmd.subcode);
1431         res = -EINVAL;
1432         dev_user_unjam_cmd(ucmd, 0, NULL);
1433
1434 out_unlock_put:
1435         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1436         goto out_put;
1437
1438 out_unlock:
1439         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1440         goto out;
1441 }
1442
1443 static int dev_user_reply_cmd(struct file *file, unsigned long arg)
1444 {
1445         int res = 0;
1446         struct scst_user_dev *dev;
1447         struct scst_user_reply_cmd *reply;
1448
1449         TRACE_ENTRY();
1450
1451         mutex_lock(&dev_priv_mutex);
1452         dev = (struct scst_user_dev *)file->private_data;
1453         res = dev_user_check_reg(dev);
1454         if (res != 0) {
1455                 mutex_unlock(&dev_priv_mutex);
1456                 goto out;
1457         }
1458         down_read(&dev->dev_rwsem);
1459         mutex_unlock(&dev_priv_mutex);
1460
1461         reply = kzalloc(sizeof(*reply), GFP_KERNEL);
1462         if (reply == NULL) {
1463                 res = -ENOMEM;
1464                 goto out_up;
1465         }
1466
1467         res = copy_from_user(reply, (void *)arg, sizeof(*reply));
1468         if (res < 0)
1469                 goto out_free;
1470
1471         TRACE_BUFFER("Reply", reply, sizeof(*reply));
1472
1473         res = dev_user_process_reply(dev, reply);
1474         if (res < 0)
1475                 goto out_free;
1476
1477 out_free:
1478         kfree(reply);
1479
1480 out_up:
1481         up_read(&dev->dev_rwsem);
1482
1483 out:
1484         TRACE_EXIT_RES(res);
1485         return res;
1486 }
1487
1488 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1489 {
1490         int res = 0;
1491
1492         TRACE_ENTRY();
1493
1494         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1495                 struct scst_cmd *cmd = list_entry(
1496                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1497                         cmd_list_entry);
1498                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1499                 list_del(&cmd->cmd_list_entry);
1500                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1501                 scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT |
1502                                                  SCST_CONTEXT_PROCESSABLE);
1503                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1504                 res++;
1505         }
1506
1507         TRACE_EXIT_RES(res);
1508         return res;
1509 }
1510
1511 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1512 struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1513 {
1514         struct scst_user_cmd *u;
1515
1516 again:
1517         u = NULL;
1518         if (!list_empty(cmd_list)) {
1519                 u = list_entry(cmd_list->next, typeof(*u), ready_cmd_list_entry);
1520
1521                 TRACE_DBG("Found ready ucmd %p", u);
1522                 list_del(&u->ready_cmd_list_entry);
1523
1524                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1525
1526                 if (u->cmd != NULL) {
1527                         if (u->state == UCMD_STATE_EXECING) {
1528                                 struct scst_user_dev *dev = u->dev;
1529                                 int rc;
1530
1531                                 EXTRACHECKS_BUG_ON(u->jammed);
1532
1533                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1534
1535                                 rc = scst_check_local_events(u->cmd);
1536                                 if (unlikely(rc != 0)) {
1537                                         u->cmd->scst_cmd_done(u->cmd,
1538                                                 SCST_CMD_STATE_DEFAULT);
1539                                         /*
1540                                          * !! At this point cmd & u can be !!
1541                                          * !! already freed                !!
1542                                          */
1543                                         spin_lock_irq(
1544                                                 &dev->cmd_lists.cmd_list_lock);
1545                                         goto again;
1546                                 }
1547
1548                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1549                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1550                                         &u->cmd->cmd_flags))) {
1551                                 switch (u->state) {
1552                                 case UCMD_STATE_PARSING:
1553                                 case UCMD_STATE_BUF_ALLOCING:
1554                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1555                                         dev_user_unjam_cmd(u, 0, NULL);
1556                                         goto again;
1557                                 case UCMD_STATE_EXECING:
1558                                         EXTRACHECKS_BUG_ON(1);
1559                                 }
1560                         }
1561                 }
1562                 u->sent_to_user = 1;
1563                 u->seen_by_user = 1;
1564         }
1565         return u;
1566 }
1567
1568 static inline int test_cmd_lists(struct scst_user_dev *dev)
1569 {
1570         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1571                   !list_empty(&dev->ready_cmd_list) ||
1572                   !dev->blocking || dev->cleanup_done ||
1573                   signal_pending(current);
1574         return res;
1575 }
1576
1577 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1578 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1579         struct scst_user_cmd **ucmd)
1580 {
1581         int res = 0;
1582         wait_queue_t wait;
1583
1584         TRACE_ENTRY();
1585
1586         init_waitqueue_entry(&wait, current);
1587
1588         while (1) {
1589                 if (!test_cmd_lists(dev)) {
1590                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1591                                 &wait);
1592                         for (;;) {
1593                                 set_current_state(TASK_INTERRUPTIBLE);
1594                                 if (test_cmd_lists(dev))
1595                                         break;
1596                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1597                                 schedule();
1598                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1599                         }
1600                         set_current_state(TASK_RUNNING);
1601                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1602                                 &wait);
1603                 }
1604
1605                 dev_user_process_scst_commands(dev);
1606
1607                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1608                 if (*ucmd != NULL)
1609                         break;
1610
1611                 if (!dev->blocking || dev->cleanup_done) {
1612                         res = -EAGAIN;
1613                         TRACE_DBG("No ready commands, returning %d", res);
1614                         break;
1615                 }
1616
1617                 if (signal_pending(current)) {
1618                         res = -EINTR;
1619                         TRACE_DBG("Signal pending, returning %d", res);
1620                         break;
1621                 }
1622         }
1623
1624         TRACE_EXIT_RES(res);
1625         return res;
1626 }
1627
1628 static int dev_user_reply_get_cmd(struct file *file, unsigned long arg)
1629 {
1630         int res = 0;
1631         struct scst_user_dev *dev;
1632         struct scst_user_get_cmd *cmd;
1633         struct scst_user_reply_cmd *reply;
1634         struct scst_user_cmd *ucmd;
1635         uint64_t ureply;
1636
1637         TRACE_ENTRY();
1638
1639         mutex_lock(&dev_priv_mutex);
1640         dev = (struct scst_user_dev *)file->private_data;
1641         res = dev_user_check_reg(dev);
1642         if (res != 0) {
1643                 mutex_unlock(&dev_priv_mutex);
1644                 goto out;
1645         }
1646         down_read(&dev->dev_rwsem);
1647         mutex_unlock(&dev_priv_mutex);
1648
1649         res = copy_from_user(&ureply, (void *)arg, sizeof(ureply));
1650         if (res < 0)
1651                 goto out_up;
1652
1653         TRACE_DBG("ureply %Ld", (long long unsigned int)ureply);
1654
1655         cmd = kzalloc(max(sizeof(*cmd), sizeof(*reply)), GFP_KERNEL);
1656         if (cmd == NULL) {
1657                 res = -ENOMEM;
1658                 goto out_up;
1659         }
1660
1661         if (ureply != 0) {
1662                 unsigned long u = (unsigned long)ureply;
1663                 reply = (struct scst_user_reply_cmd *)cmd;
1664                 res = copy_from_user(reply, (void *)u, sizeof(*reply));
1665                 if (res < 0)
1666                         goto out_free;
1667
1668                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1669
1670                 res = dev_user_process_reply(dev, reply);
1671                 if (res < 0)
1672                         goto out_free;
1673         }
1674
1675         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1676         res = dev_user_get_next_cmd(dev, &ucmd);
1677         if (res == 0) {
1678                 *cmd = ucmd->user_cmd;
1679                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1680                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1681                 res = copy_to_user((void *)arg, cmd, sizeof(*cmd));
1682         } else
1683                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1684
1685 out_free:
1686         kfree(cmd);
1687
1688 out_up:
1689         up_read(&dev->dev_rwsem);
1690
1691 out:
1692         TRACE_EXIT_RES(res);
1693         return res;
1694 }
1695
1696 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1697         unsigned long arg)
1698 {
1699         long res;
1700
1701         TRACE_ENTRY();
1702
1703         switch (cmd) {
1704         case SCST_USER_REPLY_AND_GET_CMD:
1705                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1706                 res = dev_user_reply_get_cmd(file, arg);
1707                 break;
1708
1709         case SCST_USER_REPLY_CMD:
1710                 TRACE_DBG("%s", "REPLY_CMD");
1711                 res = dev_user_reply_cmd(file, arg);
1712                 break;
1713
1714         case SCST_USER_REGISTER_DEVICE:
1715         {
1716                 struct scst_user_dev_desc *dev_desc;
1717                 TRACE_DBG("%s", "REGISTER_DEVICE");
1718                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1719                 if (dev_desc == NULL) {
1720                         res = -ENOMEM;
1721                         goto out;
1722                 }
1723                 res = copy_from_user(dev_desc, (void *)arg, sizeof(*dev_desc));
1724                 if (res < 0) {
1725                         kfree(dev_desc);
1726                         goto out;
1727                 }
1728                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1729                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1730                 res = dev_user_register_dev(file, dev_desc);
1731                 kfree(dev_desc);
1732                 break;
1733         }
1734
1735         case SCST_USER_SET_OPTIONS:
1736         {
1737                 struct scst_user_opt opt;
1738                 TRACE_DBG("%s", "SET_OPTIONS");
1739                 res = copy_from_user(&opt, (void *)arg, sizeof(opt));
1740                 if (res < 0)
1741                         goto out;
1742                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1743                 res = dev_user_set_opt(file, &opt);
1744                 break;
1745         }
1746
1747         case SCST_USER_GET_OPTIONS:
1748                 TRACE_DBG("%s", "GET_OPTIONS");
1749                 res = dev_user_get_opt(file, (void *)arg);
1750                 break;
1751
1752         default:
1753                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1754                 res = -EINVAL;
1755                 goto out;
1756         }
1757
1758 out:
1759         TRACE_EXIT_RES(res);
1760         return res;
1761 }
1762
1763 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1764 {
1765         int res = 0;
1766         struct scst_user_dev *dev;
1767
1768         TRACE_ENTRY();
1769
1770         mutex_lock(&dev_priv_mutex);
1771         dev = (struct scst_user_dev *)file->private_data;
1772         res = dev_user_check_reg(dev);
1773         if (res != 0) {
1774                 mutex_unlock(&dev_priv_mutex);
1775                 goto out;
1776         }
1777         down_read(&dev->dev_rwsem);
1778         mutex_unlock(&dev_priv_mutex);
1779
1780         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1781
1782         if (!list_empty(&dev->ready_cmd_list) ||
1783             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1784                 res |= POLLIN | POLLRDNORM;
1785                 goto out_unlock;
1786         }
1787
1788         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1789
1790         TRACE_DBG("Before poll_wait() (dev %p)", dev);
1791         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1792         TRACE_DBG("After poll_wait() (dev %p)", dev);
1793
1794         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1795
1796         if (!list_empty(&dev->ready_cmd_list) ||
1797             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1798                 res |= POLLIN | POLLRDNORM;
1799                 goto out_unlock;
1800         }
1801
1802 out_unlock:
1803         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1804
1805         up_read(&dev->dev_rwsem);
1806
1807 out:
1808         TRACE_EXIT_HRES(res);
1809         return res;
1810 }
1811
1812 /*
1813  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reaquire.
1814  */
1815 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1816         unsigned long *flags)
1817 {
1818         int state = ucmd->state;
1819         struct scst_user_dev *dev = ucmd->dev;
1820
1821         TRACE_ENTRY();
1822
1823         if (ucmd->this_state_unjammed)
1824                 goto out;
1825
1826         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1827                 state);
1828
1829         ucmd->jammed = 1;
1830         ucmd->this_state_unjammed = 1;
1831         ucmd->sent_to_user = 0;
1832
1833         switch (state) {
1834         case UCMD_STATE_PARSING:
1835         case UCMD_STATE_BUF_ALLOCING:
1836                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1837                         ucmd->aborted = 1;
1838                 else {
1839                         if (busy)
1840                                 scst_set_busy(ucmd->cmd);
1841                         else
1842                                 scst_set_cmd_error(ucmd->cmd,
1843                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1844                 }
1845                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1846
1847                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1848                 list_add(&ucmd->cmd->cmd_list_entry,
1849                         &ucmd->cmd->cmd_lists->active_cmd_list);
1850                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1851                 break;
1852
1853         case UCMD_STATE_EXECING:
1854                 if (flags != NULL)
1855                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1856                 else
1857                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1858
1859                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1860
1861                 if (test_bit(SCST_CMD_ABORTED,  &ucmd->cmd->cmd_flags))
1862                         ucmd->aborted = 1;
1863                 else {
1864                         if (busy)
1865                                 scst_set_busy(ucmd->cmd);
1866                         else
1867                                 scst_set_cmd_error(ucmd->cmd,
1868                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1869                 }
1870
1871                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT);
1872                 /* !! At this point cmd and ucmd can be already freed !! */
1873
1874                 if (flags != NULL)
1875                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1876                 else
1877                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1878                 break;
1879
1880         case UCMD_STATE_ON_FREEING:
1881         case UCMD_STATE_ON_CACHE_FREEING:
1882         case UCMD_STATE_TM_EXECING:
1883         case UCMD_STATE_ATTACH_SESS:
1884         case UCMD_STATE_DETACH_SESS:
1885         {
1886                 if (flags != NULL)
1887                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1888                 else
1889                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1890
1891                 switch (state) {
1892                 case UCMD_STATE_ON_FREEING:
1893                         dev_user_process_reply_on_free(ucmd);
1894                         break;
1895
1896                 case UCMD_STATE_ON_CACHE_FREEING:
1897                         dev_user_process_reply_on_cache_free(ucmd);
1898                         break;
1899
1900                 case UCMD_STATE_TM_EXECING:
1901                         dev_user_process_reply_tm_exec(ucmd, SCST_MGMT_STATUS_FAILED);
1902                         break;
1903
1904                 case UCMD_STATE_ATTACH_SESS:
1905                 case UCMD_STATE_DETACH_SESS:
1906                         dev_user_process_reply_sess(ucmd, -EFAULT);
1907                         break;
1908                 }
1909
1910                 if (flags != NULL)
1911                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1912                 else
1913                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1914                 break;
1915         }
1916
1917         default:
1918                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1919                 sBUG();
1920                 break;
1921         }
1922
1923 out:
1924         TRACE_EXIT();
1925         return;
1926 }
1927
1928 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1929 {
1930         int i;
1931         struct scst_user_cmd *ucmd;
1932
1933         TRACE_ENTRY();
1934
1935         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1936
1937         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1938
1939 repeat:
1940         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1941                 struct list_head *head = &dev->ucmd_hash[i];
1942
1943                 list_for_each_entry(ucmd, head, hash_list_entry) {
1944                         if (!ucmd->sent_to_user)
1945                                 continue;
1946
1947                         if (ucmd_get_check(ucmd))
1948                                 continue;
1949
1950                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1951                                 ucmd->state, ucmd->cmd);
1952
1953                         dev_user_unjam_cmd(ucmd, 0, NULL);
1954
1955                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1956                         ucmd_put(ucmd);
1957                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1958
1959                         goto repeat;
1960                 }
1961         }
1962
1963         if (dev_user_process_scst_commands(dev) != 0)
1964                 goto repeat;
1965
1966         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1967
1968         TRACE_EXIT();
1969         return;
1970 }
1971
1972 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
1973         int status)
1974 {
1975         int res = 0;
1976
1977         TRACE_ENTRY();
1978
1979         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
1980                 ucmd->user_cmd.tm_cmd.fn, status);
1981
1982         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
1983                 /*
1984                  * It is possible that user space seen TM cmd before cmd
1985                  * to abort or will never see it at all, because it was
1986                  * aborted on the way there. So, it is safe to return
1987                  * success instead, because, if there is the TM cmd at this
1988                  * point, then the cmd to abort apparrently does exist.
1989                  */
1990                 status = SCST_MGMT_STATUS_SUCCESS;
1991         }
1992
1993         scst_async_mcmd_completed(ucmd->mcmd, status);
1994
1995         ucmd_put(ucmd);
1996
1997         TRACE_EXIT_RES(res);
1998         return res;
1999 }
2000
2001 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2002 {
2003         struct scst_user_cmd *ucmd;
2004         unsigned long flags;
2005
2006         TRACE_ENTRY();
2007
2008         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2009 again:
2010         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2011                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2012                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2013                         switch (ucmd->state) {
2014                         case UCMD_STATE_PARSING:
2015                         case UCMD_STATE_BUF_ALLOCING:
2016                         case UCMD_STATE_EXECING:
2017                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2018                                 list_del(&ucmd->ready_cmd_list_entry);
2019                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2020                                 goto again;
2021                         }
2022                 }
2023         }
2024
2025         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2026
2027         TRACE_EXIT();
2028         return;
2029 }
2030
2031 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2032         struct scst_tgt_dev *tgt_dev)
2033 {
2034         struct scst_user_cmd *ucmd;
2035         struct scst_user_dev *dev = (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2036         struct scst_user_cmd *ucmd_to_abort = NULL;
2037
2038         TRACE_ENTRY();
2039
2040         /*
2041          * In the used approach we don't do anything with hung devices, which
2042          * stopped responding and/or have stuck commands. We forcedly abort such
2043          * commands only if they not yet sent to the user space or if the device
2044          * is getting unloaded, e.g. if its handler program gets killed. This is
2045          * because it's pretty hard to distinguish between stuck and temporary
2046          * overloaded states of the device. There are several reasons for that:
2047          *
2048          * 1. Some commands need a lot of time to complete (several hours),
2049          *    so for an impatient user such command(s) will always look as
2050          *    stuck.
2051          *
2052          * 2. If we forcedly abort, i.e. abort before it's actually completed
2053          *    in the user space, just one command, we will have to put the whole
2054          *    device offline until we are sure that no more previously aborted
2055          *    commands will get executed. Otherwise, we might have a possibility
2056          *    for data corruption, when aborted and reported as completed
2057          *    command actually gets executed *after* new commands sent
2058          *    after the force abort was done. Many journaling file systems and
2059          *    databases use "provide required commands order via queue draining"
2060          *    approach and not putting the whole device offline after the forced
2061          *    abort will break it. This makes our decision, if a command stuck
2062          *    or not, cost a lot.
2063          *
2064          * So, we leave policy definition if a device stuck or not to
2065          * the user space and simply let all commands live until they are
2066          * completed or their devices get closed/killed. This approach is very
2067          * much OK, but can affect management commands, which need activity
2068          * suspending via scst_suspend_activity() function such as devices or
2069          * targets registration/removal. But during normal life such commands
2070          * should be rare. Plus, when possible, scst_suspend_activity() will
2071          * return after timeout EBUSY status to allow caller to not stuck
2072          * forever as well.
2073          *
2074          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2075          * stuck commands would affect only related devices.
2076          */
2077
2078         dev_user_abort_ready_commands(dev);
2079
2080         /* We can't afford missing TM command due to memory shortage */
2081         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2082
2083         ucmd->user_cmd.cmd_h = ucmd->h;
2084         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2085         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2086         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2087         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2088         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2089
2090         if (mcmd->cmd_to_abort != NULL) {
2091                 ucmd_to_abort = (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2092                 if (ucmd_to_abort != NULL)
2093                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2094         }
2095
2096         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2097                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2098                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2099                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2100
2101         ucmd->mcmd = mcmd;
2102         ucmd->state = UCMD_STATE_TM_EXECING;
2103
2104         scst_prepare_async_mcmd(mcmd);
2105
2106         dev_user_add_to_ready(ucmd);
2107
2108         TRACE_EXIT();
2109         return SCST_DEV_TM_NOT_COMPLETED;
2110 }
2111
2112 static int dev_user_attach(struct scst_device *sdev)
2113 {
2114         int res = 0;
2115         struct scst_user_dev *dev = NULL, *d;
2116
2117         TRACE_ENTRY();
2118
2119         spin_lock(&dev_list_lock);
2120         list_for_each_entry(d, &dev_list, dev_list_entry) {
2121                 if (strcmp(d->name, sdev->virt_name) == 0) {
2122                         dev = d;
2123                         break;
2124                 }
2125         }
2126         spin_unlock(&dev_list_lock);
2127         if (dev == NULL) {
2128                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2129                 res = -EINVAL;
2130                 goto out;
2131         }
2132
2133         sdev->p_cmd_lists = &dev->cmd_lists;
2134         sdev->dh_priv = dev;
2135         sdev->tst = dev->tst;
2136         sdev->queue_alg = dev->queue_alg;
2137         sdev->swp = dev->swp;
2138         sdev->tas = dev->tas;
2139         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2140
2141         dev->sdev = sdev;
2142
2143         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2144                 dev->name);
2145
2146 out:
2147         TRACE_EXIT();
2148         return res;
2149 }
2150
2151 static void dev_user_detach(struct scst_device *sdev)
2152 {
2153         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2154
2155         TRACE_ENTRY();
2156
2157         TRACE_DBG("virt_id %d", sdev->virt_id);
2158
2159         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2160                 dev->name);
2161
2162         /* dev will be freed by the caller */
2163         sdev->dh_priv = NULL;
2164         dev->sdev = NULL;
2165
2166         TRACE_EXIT();
2167         return;
2168 }
2169
2170 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2171 {
2172         int res = 0;
2173         unsigned long flags;
2174
2175         TRACE_ENTRY();
2176
2177         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2178
2179         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2180
2181         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2182                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2183                 ucmd->result = status;
2184         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2185                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2186         } else
2187                 sBUG();
2188
2189         if (ucmd->cmpl != NULL)
2190                 complete_all(ucmd->cmpl);
2191
2192         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2193
2194         ucmd_put(ucmd);
2195
2196         TRACE_EXIT_RES(res);
2197         return res;
2198 }
2199
2200 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2201 {
2202         struct scst_user_dev *dev =
2203                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2204         int res = 0, rc;
2205         struct scst_user_cmd *ucmd;
2206
2207         TRACE_ENTRY();
2208
2209         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2210         if (ucmd == NULL)
2211                 goto out_nomem;
2212
2213         ucmd->cmpl = kmalloc(sizeof(*ucmd->cmpl), GFP_KERNEL);
2214         if (ucmd->cmpl == NULL)
2215                 goto out_put_nomem;
2216
2217         init_completion(ucmd->cmpl);
2218
2219         ucmd->user_cmd.cmd_h = ucmd->h;
2220         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2221         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2222         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2223         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2224         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2225         strncpy(ucmd->user_cmd.sess.initiator_name,
2226                 tgt_dev->sess->initiator_name,
2227                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2228         ucmd->user_cmd.sess.initiator_name[
2229                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2230
2231         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %Lx, LUN %Lx, "
2232                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2233                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2234                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2235                 ucmd->user_cmd.sess.initiator_name);
2236
2237         ucmd->state = UCMD_STATE_ATTACH_SESS;
2238
2239         ucmd_get(ucmd);
2240
2241         dev_user_add_to_ready(ucmd);
2242
2243         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2244         if (rc > 0)
2245                 res = ucmd->result;
2246         else {
2247                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2248                 res = -EFAULT;
2249         }
2250
2251         sBUG_ON(irqs_disabled());
2252
2253         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2254         kfree(ucmd->cmpl);
2255         ucmd->cmpl = NULL;
2256         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2257
2258         ucmd_put(ucmd);
2259
2260 out:
2261         TRACE_EXIT_RES(res);
2262         return res;
2263
2264 out_put_nomem:
2265         ucmd_put(ucmd);
2266
2267 out_nomem:
2268         res = -ENOMEM;
2269         goto out;
2270 }
2271
2272 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2273 {
2274         struct scst_user_dev *dev =
2275                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2276         struct scst_user_cmd *ucmd;
2277
2278         TRACE_ENTRY();
2279
2280         /*
2281          * We can't miss TM command due to memory shortage, because it might
2282          * lead to a memory leak in the user space handler.
2283          */
2284         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2285         if (ucmd == NULL)
2286                 goto out;
2287
2288         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %Lx)", ucmd,
2289                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2290
2291         ucmd->user_cmd.cmd_h = ucmd->h;
2292         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2293         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2294
2295         ucmd->state = UCMD_STATE_DETACH_SESS;
2296
2297         dev_user_add_to_ready(ucmd);
2298
2299 out:
2300         TRACE_EXIT();
2301         return;
2302 }
2303
2304 /* No locks are needed, but the activity must be suspended */
2305 static void dev_user_setup_functions(struct scst_user_dev *dev)
2306 {
2307         TRACE_ENTRY();
2308
2309         dev->devtype.parse = dev_user_parse;
2310         dev->devtype.dev_done = NULL;
2311
2312         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2313                 switch (dev->devtype.type) {
2314                 case TYPE_DISK:
2315                         dev->generic_parse = scst_sbc_generic_parse;
2316                         dev->devtype.dev_done = dev_user_disk_done;
2317                         break;
2318
2319                 case TYPE_TAPE:
2320                         dev->generic_parse = scst_tape_generic_parse;
2321                         dev->devtype.dev_done = dev_user_tape_done;
2322                         break;
2323
2324                 case TYPE_MOD:
2325                         dev->generic_parse = scst_modisk_generic_parse;
2326                         dev->devtype.dev_done = dev_user_disk_done;
2327                         break;
2328
2329                 case TYPE_ROM:
2330                         dev->generic_parse = scst_cdrom_generic_parse;
2331                         dev->devtype.dev_done = dev_user_disk_done;
2332                         break;
2333
2334                 case TYPE_MEDIUM_CHANGER:
2335                         dev->generic_parse = scst_changer_generic_parse;
2336                         break;
2337
2338                 case TYPE_PROCESSOR:
2339                         dev->generic_parse = scst_processor_generic_parse;
2340                         break;
2341
2342                 case TYPE_RAID:
2343                         dev->generic_parse = scst_raid_generic_parse;
2344                         break;
2345
2346                 default:
2347                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2348                                 "for it", dev->devtype.type);
2349                         dev->parse_type = SCST_USER_PARSE_CALL;
2350                         break;
2351                 }
2352         } else {
2353                 dev->generic_parse = NULL;
2354                 dev->devtype.dev_done = NULL;
2355         }
2356
2357         TRACE_EXIT();
2358         return;
2359 }
2360
2361 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2362 {
2363         char ver[sizeof(DEV_USER_VERSION)+1];
2364         int res;
2365
2366         res = copy_from_user(ver, (void *)(unsigned long)dev_desc->version_str,
2367                                 sizeof(ver));
2368         if (res < 0) {
2369                 PRINT_ERROR("%s", "Unable to get version string");
2370                 goto out;
2371         }
2372         ver[sizeof(ver)-1] = '\0';
2373
2374         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2375                 /* ->name already 0-terminated in dev_user_ioctl() */
2376                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2377                         dev_desc->name, ver);
2378                 res = -EINVAL;
2379                 goto out;
2380         }
2381
2382 out:
2383         return res;
2384 }
2385
2386 static int dev_user_register_dev(struct file *file,
2387         const struct scst_user_dev_desc *dev_desc)
2388 {
2389         int res = -ENOMEM, i;
2390         struct scst_user_dev *dev, *d;
2391         int block;
2392
2393         TRACE_ENTRY();
2394
2395         res = dev_user_check_version(dev_desc);
2396         if (res != 0)
2397                 goto out;
2398
2399         switch (dev_desc->type) {
2400         case TYPE_DISK:
2401         case TYPE_ROM:
2402         case TYPE_MOD:
2403                 if (dev_desc->block_size == 0) {
2404                         PRINT_ERROR("Wrong block size %d", dev_desc->block_size);
2405                         res = -EINVAL;
2406                         goto out;
2407                 }
2408                 block = scst_calc_block_shift(dev_desc->block_size);
2409                 if (block == -1) {
2410                         res = -EINVAL;
2411                         goto out;
2412                 }
2413                 break;
2414         default:
2415                 block = dev_desc->block_size;
2416                 break;
2417         }
2418
2419         if (!try_module_get(THIS_MODULE)) {
2420                 PRINT_ERROR("%s", "Fail to get module");
2421                 goto out;
2422         }
2423
2424         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2425         if (dev == NULL)
2426                 goto out_put;
2427
2428         init_rwsem(&dev->dev_rwsem);
2429         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2430         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2431         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2432         INIT_LIST_HEAD(&dev->ready_cmd_list);
2433         if (file->f_flags & O_NONBLOCK) {
2434                 TRACE_DBG("%s", "Non-blocking operations");
2435                 dev->blocking = 0;
2436         } else
2437                 dev->blocking = 1;
2438         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2439                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2440
2441         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2442         dev->name[sizeof(dev->name)-1] = '\0';
2443
2444         scst_init_mem_lim(&dev->udev_mem_lim);
2445
2446         /*
2447          * We don't use clustered pool, since it implies pages reordering,
2448          * which isn't possible with user space supplied buffers. Although
2449          * it's still possible to cluster pages by the tail of each other,
2450          * seems it doesn't worth the effort.
2451          */
2452         dev->pool = sgv_pool_create(dev->name, 0);
2453         if (dev->pool == NULL)
2454                 goto out_put;
2455         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2456                 dev_user_free_sg_entries);
2457
2458         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2459                 dev->name);
2460         dev->devtype.type = dev_desc->type;
2461         dev->devtype.threads_num = -1;
2462         dev->devtype.parse_atomic = 1;
2463         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2464         dev->devtype.dev_done_atomic = 1;
2465         dev->devtype.no_proc = 1;
2466         dev->devtype.attach = dev_user_attach;
2467         dev->devtype.detach = dev_user_detach;
2468         dev->devtype.attach_tgt = dev_user_attach_tgt;
2469         dev->devtype.detach_tgt = dev_user_detach_tgt;
2470         dev->devtype.exec = dev_user_exec;
2471         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2472         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2473
2474         init_completion(&dev->cleanup_cmpl);
2475         dev->block = block;
2476         dev->def_block = block;
2477
2478         res = __dev_user_set_opt(dev, &dev_desc->opt);
2479
2480         TRACE_MEM("dev %p, name %s", dev, dev->name);
2481
2482         spin_lock(&dev_list_lock);
2483
2484         list_for_each_entry(d, &dev_list, dev_list_entry) {
2485                 if (strcmp(d->name, dev->name) == 0) {
2486                         PRINT_ERROR("Device %s already exist",
2487                                 dev->name);
2488                         res = -EEXIST;
2489                         spin_unlock(&dev_list_lock);
2490                         goto out_free;
2491                 }
2492         }
2493
2494         list_add_tail(&dev->dev_list_entry, &dev_list);
2495
2496         spin_unlock(&dev_list_lock);
2497
2498         if (res != 0)
2499                 goto out_del_free;
2500
2501         res = scst_register_virtual_dev_driver(&dev->devtype);
2502         if (res < 0)
2503                 goto out_del_free;
2504
2505         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2506         if (dev->virt_id < 0) {
2507                 res = dev->virt_id;
2508                 goto out_unreg_handler;
2509         }
2510
2511         mutex_lock(&dev_priv_mutex);
2512         if (file->private_data != NULL) {
2513                 mutex_unlock(&dev_priv_mutex);
2514                 PRINT_ERROR("%s", "Device already registered");
2515                 res = -EINVAL;
2516                 goto out_unreg_drv;
2517         }
2518         file->private_data = dev;
2519         mutex_unlock(&dev_priv_mutex);
2520
2521 out:
2522         TRACE_EXIT_RES(res);
2523         return res;
2524
2525 out_unreg_drv:
2526         scst_unregister_virtual_device(dev->virt_id);
2527
2528 out_unreg_handler:
2529         scst_unregister_virtual_dev_driver(&dev->devtype);
2530
2531 out_del_free:
2532         spin_lock(&dev_list_lock);
2533         list_del(&dev->dev_list_entry);
2534         spin_unlock(&dev_list_lock);
2535
2536 out_free:
2537         sgv_pool_destroy(dev->pool);
2538         kfree(dev);
2539         goto out_put;
2540
2541 out_put:
2542         module_put(THIS_MODULE);
2543         goto out;
2544 }
2545
2546 static int __dev_user_set_opt(struct scst_user_dev *dev,
2547         const struct scst_user_opt *opt)
2548 {
2549         int res = 0;
2550
2551         TRACE_ENTRY();
2552
2553         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2554                 "partial_transfers_type %x, partial_len %d", opt->parse_type,
2555                 opt->on_free_cmd_type, opt->memory_reuse_type,
2556                 opt->partial_transfers_type, opt->partial_len);
2557
2558         if ((opt->parse_type > SCST_USER_MAX_PARSE_OPT) ||
2559             (opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT) ||
2560             (opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT) ||
2561             (opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT)) {
2562                 PRINT_ERROR("%s", "Invalid option");
2563                 res = -EINVAL;
2564                 goto out;
2565         }
2566
2567         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2568              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2569             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2570              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2571             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2572                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x, "
2573                         "tas %x, has_own_order_mgmt %x)", opt->tst,
2574                         opt->queue_alg, opt->swp, opt->tas, opt->has_own_order_mgmt);
2575                 res = -EINVAL;
2576                 goto out;
2577         }
2578
2579         dev->parse_type = opt->parse_type;
2580         dev->on_free_cmd_type = opt->on_free_cmd_type;
2581         dev->memory_reuse_type = opt->memory_reuse_type;
2582         dev->partial_transfers_type = opt->partial_transfers_type;
2583         dev->partial_len = opt->partial_len;
2584
2585         dev->tst = opt->tst;
2586         dev->queue_alg = opt->queue_alg;
2587         dev->swp = opt->swp;
2588         dev->tas = opt->tas;
2589         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2590         if (dev->sdev != NULL) {
2591                 dev->sdev->tst = opt->tst;
2592                 dev->sdev->queue_alg = opt->queue_alg;
2593                 dev->sdev->swp = opt->swp;
2594                 dev->sdev->tas = opt->tas;
2595                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2596         }
2597
2598         dev_user_setup_functions(dev);
2599
2600 out:
2601         TRACE_EXIT_RES(res);
2602         return res;
2603 }
2604
2605 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2606 {
2607         int res = 0;
2608         struct scst_user_dev *dev;
2609
2610         TRACE_ENTRY();
2611
2612         mutex_lock(&dev_priv_mutex);
2613         dev = (struct scst_user_dev *)file->private_data;
2614         res = dev_user_check_reg(dev);
2615         if (res != 0) {
2616                 mutex_unlock(&dev_priv_mutex);
2617                 goto out;
2618         }
2619         down_write(&dev->dev_rwsem);
2620         mutex_unlock(&dev_priv_mutex);
2621
2622         res = scst_suspend_activity(true);
2623         if (res != 0)
2624                 goto out;
2625
2626         res = __dev_user_set_opt(dev, opt);
2627
2628         scst_resume_activity();
2629
2630         up_write(&dev->dev_rwsem);
2631
2632 out:
2633         TRACE_EXIT_RES(res);
2634         return res;
2635 }
2636
2637 static int dev_user_get_opt(struct file *file, void *arg)
2638 {
2639         int res = 0;
2640         struct scst_user_dev *dev;
2641         struct scst_user_opt opt;
2642
2643         TRACE_ENTRY();
2644
2645         mutex_lock(&dev_priv_mutex);
2646         dev = (struct scst_user_dev *)file->private_data;
2647         res = dev_user_check_reg(dev);
2648         if (res != 0) {
2649                 mutex_unlock(&dev_priv_mutex);
2650                 goto out;
2651         }
2652         down_read(&dev->dev_rwsem);
2653         mutex_unlock(&dev_priv_mutex);
2654
2655         opt.parse_type = dev->parse_type;
2656         opt.on_free_cmd_type = dev->on_free_cmd_type;
2657         opt.memory_reuse_type = dev->memory_reuse_type;
2658         opt.partial_transfers_type = dev->partial_transfers_type;
2659         opt.partial_len = dev->partial_len;
2660         opt.tst = dev->tst;
2661         opt.queue_alg = dev->queue_alg;
2662         opt.tas = dev->tas;
2663         opt.swp = dev->swp;
2664         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2665
2666         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2667                 "partial_transfers_type %x, partial_len %d", opt.parse_type,
2668                 opt.on_free_cmd_type, opt.memory_reuse_type,
2669                 opt.partial_transfers_type, opt.partial_len);
2670
2671         res = copy_to_user(arg, &opt, sizeof(opt));
2672
2673         up_read(&dev->dev_rwsem);
2674 out:
2675         TRACE_EXIT_RES(res);
2676         return res;
2677 }
2678
2679 static int dev_usr_parse(struct scst_cmd *cmd)
2680 {
2681         sBUG();
2682         return SCST_CMD_STATE_DEFAULT;
2683 }
2684
2685 /* Needed only for /proc support */
2686 #define USR_TYPE {                      \
2687         .name =         DEV_USER_NAME,  \
2688         .type =         -1,             \
2689         .parse =        dev_usr_parse,  \
2690 }
2691
2692 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2693
2694 static int dev_user_release(struct inode *inode, struct file *file)
2695 {
2696         int res = 0;
2697         struct scst_user_dev *dev;
2698
2699         TRACE_ENTRY();
2700
2701         mutex_lock(&dev_priv_mutex);
2702         dev = (struct scst_user_dev *)file->private_data;
2703         if (dev == NULL) {
2704                 mutex_unlock(&dev_priv_mutex);
2705                 goto out;
2706         }
2707         file->private_data = NULL;
2708
2709         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2710
2711         spin_lock(&dev_list_lock);
2712         list_del(&dev->dev_list_entry);
2713         spin_unlock(&dev_list_lock);
2714
2715         mutex_unlock(&dev_priv_mutex);
2716
2717         down_write(&dev->dev_rwsem);
2718
2719         spin_lock_irq(&cleanup_lock);
2720         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2721         spin_unlock_irq(&cleanup_lock);
2722
2723         wake_up(&cleanup_list_waitQ);
2724         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2725
2726         scst_unregister_virtual_device(dev->virt_id);
2727         scst_unregister_virtual_dev_driver(&dev->devtype);
2728
2729         sgv_pool_destroy(dev->pool);
2730
2731         TRACE_DBG("Unregistering finished (dev %p)", dev);
2732
2733         dev->cleanup_done = 1;
2734
2735         wake_up(&cleanup_list_waitQ);
2736         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2737
2738         wait_for_completion(&dev->cleanup_cmpl);
2739
2740         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2741
2742         TRACE_DBG("Releasing completed (dev %p)", dev);
2743
2744         kfree(dev);
2745
2746         module_put(THIS_MODULE);
2747
2748 out:
2749         TRACE_EXIT_RES(res);
2750         return res;
2751 }
2752
2753 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2754 {
2755         struct scst_user_cmd *ucmd;
2756         int rc, res = 1;
2757
2758         TRACE_ENTRY();
2759
2760         dev->blocking = 0;
2761
2762         while (1) {
2763                 TRACE_DBG("Cleanuping dev %p", dev);
2764
2765                 dev_user_unjam_dev(dev);
2766
2767                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2768
2769                 rc = dev_user_get_next_cmd(dev, &ucmd);
2770                 if (rc == 0)
2771                         dev_user_unjam_cmd(ucmd, 1, NULL);
2772
2773                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2774
2775                 if (rc == -EAGAIN) {
2776                         if (dev->cleanup_done)
2777                                 break;
2778                         else {
2779                                 TRACE_DBG("No more commands (dev %p)", dev);
2780                                 goto out;
2781                         }
2782                 }
2783         }
2784
2785 #ifdef EXTRACHECKS
2786 {
2787         int i;
2788         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2789                 struct list_head *head = &dev->ucmd_hash[i];
2790                 struct scst_user_cmd *ucmd;
2791 again:
2792                 list_for_each_entry(ucmd, head, hash_list_entry) {
2793                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd,
2794                                 ucmd->state, atomic_read(&ucmd->ucmd_ref));
2795                         ucmd_put(ucmd);
2796                         goto again;
2797                 }
2798         }
2799 }
2800 #endif
2801
2802         TRACE_DBG("Cleanuping done (dev %p)", dev);
2803         complete_all(&dev->cleanup_cmpl);
2804         res = 0;
2805
2806 out:
2807         TRACE_EXIT_RES(res);
2808         return res;
2809 }
2810
2811 static inline int test_cleanup_list(void)
2812 {
2813         int res = !list_empty(&cleanup_list) ||
2814                   unlikely(kthread_should_stop());
2815         return res;
2816 }
2817
2818 static int dev_user_cleanup_thread(void *arg)
2819 {
2820         TRACE_ENTRY();
2821
2822         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2823
2824         current->flags |= PF_NOFREEZE;
2825
2826         spin_lock_irq(&cleanup_lock);
2827         while (!kthread_should_stop()) {
2828                 wait_queue_t wait;
2829                 init_waitqueue_entry(&wait, current);
2830
2831                 if (!test_cleanup_list()) {
2832                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2833                         for (;;) {
2834                                 set_current_state(TASK_INTERRUPTIBLE);
2835                                 if (test_cleanup_list())
2836                                         break;
2837                                 spin_unlock_irq(&cleanup_lock);
2838                                 schedule();
2839                                 spin_lock_irq(&cleanup_lock);
2840                         }
2841                         set_current_state(TASK_RUNNING);
2842                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2843                 }
2844
2845                 /*
2846                  * We have to poll devices, because commands can go from SCST
2847                  * core on cmd_list_waitQ and we have no practical way to
2848                  * detect them.
2849                  */
2850
2851                 while (1) {
2852                         struct scst_user_dev *dev;
2853                         LIST_HEAD(cl_devs);
2854
2855                         while (!list_empty(&cleanup_list)) {
2856                                 int rc;
2857
2858                                 dev = list_entry(cleanup_list.next,
2859                                         typeof(*dev), cleanup_list_entry);
2860                                 list_del(&dev->cleanup_list_entry);
2861
2862                                 spin_unlock_irq(&cleanup_lock);
2863                                 rc = dev_user_process_cleanup(dev);
2864                                 spin_lock_irq(&cleanup_lock);
2865
2866                                 if (rc != 0)
2867                                         list_add_tail(&dev->cleanup_list_entry,
2868                                                 &cl_devs);
2869                         }
2870
2871                         if (list_empty(&cl_devs))
2872                                 break;
2873
2874                         spin_unlock_irq(&cleanup_lock);
2875                         msleep(100);
2876                         spin_lock_irq(&cleanup_lock);
2877
2878                         while (!list_empty(&cl_devs)) {
2879                                 dev = list_entry(cl_devs.next, typeof(*dev),
2880                                         cleanup_list_entry);
2881                                 list_move_tail(&dev->cleanup_list_entry,
2882                                         &cleanup_list);
2883                         }
2884                 }
2885         }
2886         spin_unlock_irq(&cleanup_lock);
2887
2888         /*
2889          * If kthread_should_stop() is true, we are guaranteed to be
2890          * on the module unload, so cleanup_list must be empty.
2891          */
2892         sBUG_ON(!list_empty(&cleanup_list));
2893
2894         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2895
2896         TRACE_EXIT();
2897         return 0;
2898 }
2899
2900 static int __init init_scst_user(void)
2901 {
2902         int res = 0;
2903         struct class_device *class_member;
2904
2905         TRACE_ENTRY();
2906
2907 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2908         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2909                 "Consider change VMSPLIT option or use 64-bit "
2910                 "configuration instead. See README file for details.");
2911         res = -EINVAL;
2912         goto out;
2913 #endif
2914
2915         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2916         if (user_cmd_cachep == NULL) {
2917                 res = -ENOMEM;
2918                 goto out;
2919         }
2920
2921         dev_user_devtype.module = THIS_MODULE;
2922
2923         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2924         if (res < 0)
2925                 goto out_cache;
2926
2927         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2928         if (res != 0)
2929                 goto out_unreg;
2930
2931         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2932         if (IS_ERR(dev_user_sysfs_class)) {
2933                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2934                         "space handler");
2935                 res = PTR_ERR(dev_user_sysfs_class);
2936                 goto out_proc;
2937         }
2938
2939         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2940         if (res) {
2941                 PRINT_ERROR("Unable to get major %d for SCSI tapes", DEV_USER_MAJOR);
2942                 goto out_class;
2943         }
2944
2945         class_member = class_device_create(dev_user_sysfs_class, NULL,
2946                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
2947         if (IS_ERR(class_member)) {
2948                 res = PTR_ERR(class_member);
2949                 goto out_chrdev;
2950         }
2951
2952         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
2953                 "scst_usr_cleanupd");
2954         if (IS_ERR(cleanup_thread)) {
2955                 res = PTR_ERR(cleanup_thread);
2956                 PRINT_ERROR("kthread_create() failed: %d", res);
2957                 goto out_dev;
2958         }
2959
2960 out:
2961         TRACE_EXIT_RES(res);
2962         return res;
2963
2964 out_dev:
2965         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2966
2967 out_chrdev:
2968         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2969
2970 out_class:
2971         class_destroy(dev_user_sysfs_class);
2972
2973 out_proc:
2974         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
2975
2976 out_unreg:
2977         scst_unregister_dev_driver(&dev_user_devtype);
2978
2979 out_cache:
2980         kmem_cache_destroy(user_cmd_cachep);
2981         goto out;
2982 }
2983
2984 static void __exit exit_scst_user(void)
2985 {
2986         int rc;
2987
2988         TRACE_ENTRY();
2989
2990         rc = kthread_stop(cleanup_thread);
2991         if (rc < 0)
2992                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2993
2994         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2995         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2996         class_destroy(dev_user_sysfs_class);
2997
2998         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
2999         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3000
3001         kmem_cache_destroy(user_cmd_cachep);
3002
3003         TRACE_EXIT();
3004         return;
3005 }
3006
3007 module_init(init_scst_user);
3008 module_exit(exit_scst_user);
3009
3010 MODULE_AUTHOR("Vladislav Bolkhovitin");
3011 MODULE_LICENSE("GPL");
3012 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3013 MODULE_VERSION(SCST_VERSION_STRING);
3014 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);