995778be6d7a107c2e7ee1958b10ad4ecf6dfd6e
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2009 ID7 Ltd.
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23 #include <linux/stddef.h>
24
25 #define LOG_PREFIX              DEV_USER_NAME
26
27 #include "scst.h"
28 #include "scst_user.h"
29 #include "scst_dev_handler.h"
30
31 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
32 #warning "HIGHMEM kernel configurations are not supported by this module,\
33  because nowadays it doesn't worth the effort. Consider changing\
34  VMSPLIT option or use a 64-bit configuration instead. See README file\
35  for details."
36 #endif
37
38 #define DEV_USER_CMD_HASH_ORDER         6
39 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
40
41 struct scst_user_dev {
42         struct rw_semaphore dev_rwsem;
43
44         struct scst_cmd_lists cmd_lists;
45
46         /* Protected by cmd_lists.cmd_list_lock */
47         struct list_head ready_cmd_list;
48
49         /* Protected by dev_rwsem or don't need any protection */
50         unsigned int blocking:1;
51         unsigned int cleanup_done:1;
52         unsigned int tst:3;
53         unsigned int queue_alg:4;
54         unsigned int tas:1;
55         unsigned int swp:1;
56         unsigned int d_sense:1;
57         unsigned int has_own_order_mgmt:1;
58
59         int (*generic_parse)(struct scst_cmd *cmd,
60                 int (*get_block)(struct scst_cmd *cmd));
61
62         int block;
63         int def_block;
64
65         struct scst_mem_lim udev_mem_lim;
66         struct sgv_pool *pool;
67         struct sgv_pool *pool_clust;
68
69         uint8_t parse_type;
70         uint8_t on_free_cmd_type;
71         uint8_t memory_reuse_type;
72         uint8_t partial_transfers_type;
73         uint32_t partial_len;
74
75         struct scst_dev_type devtype;
76
77         /* Both protected by cmd_lists.cmd_list_lock */
78         unsigned int handle_counter;
79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
80
81         struct scst_device *sdev;
82
83         int virt_id;
84         struct list_head dev_list_entry;
85         char name[SCST_MAX_NAME];
86
87         struct list_head cleanup_list_entry;
88         struct completion cleanup_cmpl;
89 };
90
91 /* Most fields are unprotected, since only one thread at time can access them */
92 struct scst_user_cmd {
93         struct scst_cmd *cmd;
94         struct scst_user_dev *dev;
95
96         atomic_t ucmd_ref;
97
98         unsigned int buff_cached:1;
99         unsigned int buf_dirty:1;
100         unsigned int background_exec:1;
101         unsigned int aborted:1;
102
103         struct scst_user_cmd *buf_ucmd;
104
105         int cur_data_page;
106         int num_data_pages;
107         int first_page_offset;
108         unsigned long ubuff;
109         struct page **data_pages;
110         struct sgv_pool_obj *sgv;
111
112         /*
113          * Special flags, which can be accessed asynchronously (hence "long").
114          * Protected by cmd_lists.cmd_list_lock.
115          */
116         unsigned long sent_to_user:1;
117         unsigned long jammed:1;
118         unsigned long this_state_unjammed:1;
119         unsigned long seen_by_user:1; /* here only as a small optimization */
120
121         unsigned int state;
122
123         struct list_head ready_cmd_list_entry;
124
125         unsigned int h;
126         struct list_head hash_list_entry;
127
128         int user_cmd_payload_len;
129         struct scst_user_get_cmd user_cmd;
130
131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
132         union {
133                 struct completion *cmpl;
134                 struct scst_mgmt_cmd *mcmd;
135         };
136         int result;
137 };
138
139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
140         gfp_t gfp_mask);
141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
142
143 static int dev_user_parse(struct scst_cmd *cmd);
144 static int dev_user_exec(struct scst_cmd *cmd);
145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
147         struct scst_tgt_dev *tgt_dev);
148
149 static int dev_user_disk_done(struct scst_cmd *cmd);
150 static int dev_user_tape_done(struct scst_cmd *cmd);
151
152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
153         gfp_t gfp_mask, void *priv);
154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
155                                      void *priv);
156
157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
158
159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
160         unsigned long *flags);
161
162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
164         int status);
165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
166 static int dev_user_register_dev(struct file *file,
167         const struct scst_user_dev_desc *dev_desc);
168 static int dev_user_unregister_dev(struct file *file);
169 static int dev_user_flush_cache(struct file *file);
170 static int dev_user_capacity_changed(struct file *file);
171 static int dev_user_prealloc_buffer(struct file *file, void __user *arg);
172 static int __dev_user_set_opt(struct scst_user_dev *dev,
173         const struct scst_user_opt *opt);
174 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
175 static int dev_user_get_opt(struct file *file, void __user *arg);
176
177 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
178 static long dev_user_ioctl(struct file *file, unsigned int cmd,
179         unsigned long arg);
180 static int dev_user_release(struct inode *inode, struct file *file);
181 static int dev_user_exit_dev(struct scst_user_dev *dev);
182
183 #ifdef CONFIG_SCST_PROC
184
185 static int dev_user_read_proc(struct seq_file *seq,
186         struct scst_dev_type *dev_type);
187
188 #else /* CONFIG_SCST_PROC */
189
190 static ssize_t dev_user_sysfs_commands_show(struct kobject *kobj,
191         struct kobj_attribute *attr, char *buf);
192
193 static struct kobj_attribute dev_user_commands_attr =
194         __ATTR(commands, S_IRUGO, dev_user_sysfs_commands_show, NULL);
195
196 static const struct attribute *dev_user_dev_attrs[] = {
197         &dev_user_commands_attr.attr,
198         NULL,
199 };
200
201 #endif /* CONFIG_SCST_PROC */
202
203 static int dev_usr_parse(struct scst_cmd *cmd);
204
205 /** Data **/
206
207 static struct kmem_cache *user_cmd_cachep;
208 static struct kmem_cache *user_get_cmd_cachep;
209
210 static DEFINE_MUTEX(dev_priv_mutex);
211
212 static const struct file_operations dev_user_fops = {
213         .poll           = dev_user_poll,
214         .unlocked_ioctl = dev_user_ioctl,
215 #ifdef CONFIG_COMPAT
216         .compat_ioctl   = dev_user_ioctl,
217 #endif
218         .release        = dev_user_release,
219 };
220
221 static struct scst_dev_type dev_user_devtype = {
222         .name =         DEV_USER_NAME,
223         .type =         -1,
224         .parse =        dev_usr_parse,
225 #ifdef CONFIG_SCST_PROC
226         .read_proc =    dev_user_read_proc,
227 #endif
228 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
229         .default_trace_flags = SCST_DEFAULT_DEV_LOG_FLAGS,
230         .trace_flags = &trace_flag,
231 #endif
232 };
233
234 static int dev_user_major;
235
236 static struct class *dev_user_sysfs_class;
237
238 static DEFINE_SPINLOCK(dev_list_lock);
239 static LIST_HEAD(dev_list);
240
241 static DEFINE_SPINLOCK(cleanup_lock);
242 static LIST_HEAD(cleanup_list);
243 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
244 static struct task_struct *cleanup_thread;
245
246 /*
247  * Skip this command if result is not 0. Must be called under
248  * cmd_lists.cmd_list_lock and IRQ off.
249  */
250 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
251 {
252         int r = atomic_inc_return(&ucmd->ucmd_ref);
253         int res;
254         if (unlikely(r == 1)) {
255                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
256                 atomic_dec(&ucmd->ucmd_ref);
257                 res = true;
258                 /*
259                  * Necessary code is serialized by cmd_list_lock in
260                  * cmd_remove_hash()
261                  */
262         } else {
263                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
264                         atomic_read(&ucmd->ucmd_ref));
265                 res = false;
266         }
267         return res;
268 }
269
270 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
271 {
272         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
273         atomic_inc(&ucmd->ucmd_ref);
274         if (barrier)
275                 smp_mb__after_atomic_inc();
276 }
277
278 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
279 {
280         __ucmd_get(ucmd, true);
281 }
282
283 static inline void ucmd_get(struct scst_user_cmd *ucmd)
284 {
285         __ucmd_get(ucmd, false);
286 }
287
288 /* Must not be called under cmd_list_lock!! */
289 static inline void ucmd_put(struct scst_user_cmd *ucmd)
290 {
291         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
292
293         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
294
295         if (atomic_dec_and_test(&ucmd->ucmd_ref))
296                 dev_user_free_ucmd(ucmd);
297 }
298
299 static inline int calc_num_pg(unsigned long buf, int len)
300 {
301         len += buf & ~PAGE_MASK;
302         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
303 }
304
305 static void __dev_user_not_reg(void)
306 {
307         TRACE_MGMT_DBG("%s", "Device not registered");
308         return;
309 }
310
311 static inline int dev_user_check_reg(struct scst_user_dev *dev)
312 {
313         if (dev == NULL) {
314                 __dev_user_not_reg();
315                 return -ENODEV;
316         }
317         return 0;
318 }
319
320 static inline int scst_user_cmd_hashfn(int h)
321 {
322         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
323 }
324
325 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
326         unsigned int h)
327 {
328         struct list_head *head;
329         struct scst_user_cmd *ucmd;
330
331         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
332         list_for_each_entry(ucmd, head, hash_list_entry) {
333                 if (ucmd->h == h) {
334                         TRACE_DBG("Found ucmd %p", ucmd);
335                         return ucmd;
336                 }
337         }
338         return NULL;
339 }
340
341 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
342 {
343         struct list_head *head;
344         struct scst_user_dev *dev = ucmd->dev;
345         struct scst_user_cmd *u;
346         unsigned long flags;
347
348         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
349         do {
350                 ucmd->h = dev->handle_counter++;
351                 u = __ucmd_find_hash(dev, ucmd->h);
352         } while (u != NULL);
353         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
354         list_add_tail(&ucmd->hash_list_entry, head);
355         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
356
357         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
358         return;
359 }
360
361 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
362 {
363         unsigned long flags;
364
365         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
366         list_del(&ucmd->hash_list_entry);
367         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
368
369         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
370         return;
371 }
372
373 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
374 {
375         TRACE_ENTRY();
376
377         TRACE_MEM("Freeing ucmd %p", ucmd);
378
379         cmd_remove_hash(ucmd);
380         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
381
382         kmem_cache_free(user_cmd_cachep, ucmd);
383
384         TRACE_EXIT();
385         return;
386 }
387
388 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
389         gfp_t gfp_mask, void *priv)
390 {
391         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
392         int offset = 0;
393
394         TRACE_ENTRY();
395
396         /* *sg supposed to be zeroed */
397
398         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
399                 ucmd->ubuff, ucmd->cur_data_page);
400
401         if (ucmd->cur_data_page == 0) {
402                 TRACE_MEM("ucmd->first_page_offset %d",
403                         ucmd->first_page_offset);
404                 offset = ucmd->first_page_offset;
405                 ucmd_get(ucmd);
406         }
407
408         if (ucmd->cur_data_page >= ucmd->num_data_pages)
409                 goto out;
410
411         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
412                 PAGE_SIZE - offset, offset);
413         ucmd->cur_data_page++;
414
415         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
416                 sg->offset);
417         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
418
419 out:
420         TRACE_EXIT();
421         return sg_page(sg);
422 }
423
424 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
425 {
426         TRACE_ENTRY();
427
428         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
429                 ucmd, ucmd->h, ucmd->ubuff);
430
431         ucmd->user_cmd_payload_len =
432                 offsetof(struct scst_user_get_cmd, on_cached_mem_free) +
433                 sizeof(ucmd->user_cmd.on_cached_mem_free);
434         ucmd->user_cmd.cmd_h = ucmd->h;
435         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
436         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
437
438         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
439
440         dev_user_add_to_ready(ucmd);
441
442         TRACE_EXIT();
443         return;
444 }
445
446 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
447 {
448         int i;
449
450         TRACE_ENTRY();
451
452         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
453                 ucmd->ubuff, ucmd->num_data_pages);
454
455         for (i = 0; i < ucmd->num_data_pages; i++) {
456                 struct page *page = ucmd->data_pages[i];
457
458                 if (ucmd->buf_dirty)
459                         SetPageDirty(page);
460
461                 page_cache_release(page);
462         }
463
464         kfree(ucmd->data_pages);
465         ucmd->data_pages = NULL;
466
467         TRACE_EXIT();
468         return;
469 }
470
471 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
472 {
473         TRACE_ENTRY();
474
475         sBUG_ON(ucmd->data_pages == NULL);
476
477         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
478                 ucmd, ucmd->ubuff, ucmd->buff_cached);
479
480         dev_user_unmap_buf(ucmd);
481
482         if (ucmd->buff_cached)
483                 dev_user_on_cached_mem_free(ucmd);
484         else
485                 ucmd_put(ucmd);
486
487         TRACE_EXIT();
488         return;
489 }
490
491 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
492         void *priv)
493 {
494         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
495
496         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
497                 sg_count, ucmd);
498
499         __dev_user_free_sg_entries(ucmd);
500
501         return;
502 }
503
504 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
505 {
506         int mem_reuse_type = ucmd->dev->memory_reuse_type;
507
508         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
509             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
510              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
511             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
512              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
513                 return 1;
514         else
515                 return 0;
516 }
517
518 static inline int is_need_offs_page(unsigned long buf, int len)
519 {
520         return ((buf & ~PAGE_MASK) != 0) &&
521                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
522 }
523
524 /*
525  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
526  * Unmaps the buffer, if needed in case of error
527  */
528 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
529 {
530         int res = 0;
531         struct scst_cmd *cmd = ucmd->cmd;
532         struct scst_user_dev *dev = ucmd->dev;
533         struct sgv_pool *pool;
534         gfp_t gfp_mask;
535         int flags = 0;
536         int bufflen, orig_bufflen;
537         int last_len = 0;
538         int out_sg_pages = 0;
539
540         TRACE_ENTRY();
541
542         gfp_mask = __GFP_NOWARN;
543         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
544
545         if (cmd->data_direction != SCST_DATA_BIDI) {
546                 orig_bufflen = cmd->bufflen;
547                 pool = (struct sgv_pool *)cmd->tgt_dev->dh_priv;
548         } else {
549                 /* Make in_sg->offset 0 */
550                 int len = cmd->bufflen + ucmd->first_page_offset;
551                 out_sg_pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
552                 orig_bufflen = (out_sg_pages << PAGE_SHIFT) + cmd->in_bufflen;
553                 pool = dev->pool;
554         }
555         bufflen = orig_bufflen;
556
557         EXTRACHECKS_BUG_ON(bufflen == 0);
558
559         if (cached_buff) {
560                 flags |= SGV_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
561                 if (ucmd->ubuff == 0)
562                         flags |= SGV_POOL_NO_ALLOC_ON_CACHE_MISS;
563         } else {
564                 TRACE_MEM("%s", "Not cached buff");
565                 flags |= SGV_POOL_ALLOC_NO_CACHED;
566                 if (ucmd->ubuff == 0) {
567                         res = 1;
568                         goto out;
569                 }
570                 bufflen += ucmd->first_page_offset;
571                 if (is_need_offs_page(ucmd->ubuff, orig_bufflen))
572                         last_len = bufflen & ~PAGE_MASK;
573                 else
574                         last_len = orig_bufflen & ~PAGE_MASK;
575                 if (last_len == 0)
576                         last_len = PAGE_SIZE;
577         }
578         ucmd->buff_cached = cached_buff;
579
580         cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt,
581                         &ucmd->sgv, &dev->udev_mem_lim, ucmd);
582         if (cmd->sg != NULL) {
583                 struct scst_user_cmd *buf_ucmd =
584                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
585
586                 TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
587                         "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
588                         cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
589
590                 ucmd->ubuff = buf_ucmd->ubuff;
591                 ucmd->buf_ucmd = buf_ucmd;
592
593                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
594                                    (ucmd != buf_ucmd));
595
596                 if (last_len != 0) {
597                         cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
598                         cmd->sg[cmd->sg_cnt-1].length += last_len;
599                 }
600
601                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
602                         "last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
603                         cmd->sg[cmd->sg_cnt-1].length);
604
605                 if (cmd->data_direction == SCST_DATA_BIDI) {
606                         cmd->in_sg = &cmd->sg[out_sg_pages];
607                         cmd->in_sg_cnt = cmd->sg_cnt - out_sg_pages;
608                         cmd->sg_cnt = out_sg_pages;
609                         TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d",
610                                 cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt);
611                 }
612
613                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
614                         static int ll;
615                         if (ll < 10) {
616                                 PRINT_INFO("Unable to complete command due to "
617                                         "SG IO count limitation (requested %d, "
618                                         "available %d, tgt lim %d)",
619                                         cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt,
620                                         cmd->tgt->sg_tablesize);
621                                 ll++;
622                         }
623                         cmd->sg = NULL;
624                         /* sgv will be freed in dev_user_free_sgv() */
625                         res = -1;
626                 }
627         } else {
628                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
629                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
630                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
631                 if (unlikely(cmd->sg_cnt == 0)) {
632                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
633                         sBUG_ON(ucmd->sgv != NULL);
634                         res = -1;
635                 } else {
636                         switch (ucmd->state) {
637                         case UCMD_STATE_BUF_ALLOCING:
638                                 res = 1;
639                                 break;
640                         case UCMD_STATE_EXECING:
641                                 res = -1;
642                                 break;
643                         default:
644                                 sBUG();
645                                 break;
646                         }
647                 }
648         }
649
650 out:
651         TRACE_EXIT_RES(res);
652         return res;
653 }
654
655 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
656 {
657         int rc, res = SCST_CMD_STATE_DEFAULT;
658         struct scst_cmd *cmd = ucmd->cmd;
659
660         TRACE_ENTRY();
661
662         ucmd->state = UCMD_STATE_BUF_ALLOCING;
663         cmd->dh_data_buf_alloced = 1;
664
665         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
666         if (rc == 0)
667                 goto out;
668         else if (rc < 0) {
669                 scst_set_busy(cmd);
670                 res = scst_get_cmd_abnormal_done_state(cmd);
671                 goto out;
672         }
673
674         if (!(cmd->data_direction & SCST_DATA_WRITE) &&
675             !scst_is_cmd_local(cmd)) {
676                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
677                 goto out;
678         }
679
680         ucmd->user_cmd_payload_len =
681                 offsetof(struct scst_user_get_cmd, alloc_cmd) +
682                 sizeof(ucmd->user_cmd.alloc_cmd);
683         ucmd->user_cmd.cmd_h = ucmd->h;
684         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
685         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
686         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb, cmd->cdb_len);
687         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
688         ucmd->user_cmd.alloc_cmd.ext_cdb_len = cmd->ext_cdb_len;
689         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
690                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
691         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
692         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
693         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
694
695         dev_user_add_to_ready(ucmd);
696
697         res = SCST_CMD_STATE_STOP;
698
699 out:
700         TRACE_EXIT_RES(res);
701         return res;
702 }
703
704 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
705         gfp_t gfp_mask)
706 {
707         struct scst_user_cmd *ucmd = NULL;
708
709         TRACE_ENTRY();
710
711 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
712         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
713         if (ucmd != NULL)
714                 memset(ucmd, 0, sizeof(*ucmd));
715 #else
716         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
717 #endif
718         if (unlikely(ucmd == NULL)) {
719                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
720                         "user cmd (gfp_mask %x)", gfp_mask);
721                 goto out;
722         }
723         ucmd->dev = dev;
724         atomic_set(&ucmd->ucmd_ref, 1);
725
726         cmd_insert_hash(ucmd);
727
728         TRACE_MEM("ucmd %p allocated", ucmd);
729
730 out:
731         TRACE_EXIT_HRES((unsigned long)ucmd);
732         return ucmd;
733 }
734
735 static int dev_user_get_block(struct scst_cmd *cmd)
736 {
737         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
738         /*
739          * No need for locks here, since *_detach() can not be
740          * called, when there are existing commands.
741          */
742         TRACE_EXIT_RES(dev->block);
743         return dev->block;
744 }
745
746 static int dev_user_parse(struct scst_cmd *cmd)
747 {
748         int rc, res = SCST_CMD_STATE_DEFAULT;
749         struct scst_user_cmd *ucmd;
750         int atomic = scst_cmd_atomic(cmd);
751         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
752         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
753
754         TRACE_ENTRY();
755
756         if (cmd->dh_priv == NULL) {
757                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
758                 if (unlikely(ucmd == NULL)) {
759                         if (atomic) {
760                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
761                                 goto out;
762                         } else {
763                                 scst_set_busy(cmd);
764                                 goto out_error;
765                         }
766                 }
767                 ucmd->cmd = cmd;
768                 cmd->dh_priv = ucmd;
769         } else {
770                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
771                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
772         }
773
774         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
775
776         if (ucmd->state != UCMD_STATE_NEW)
777                 goto alloc;
778
779         switch (dev->parse_type) {
780         case SCST_USER_PARSE_STANDARD:
781                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
782                 rc = dev->generic_parse(cmd, dev_user_get_block);
783                 if ((rc != 0) || !(cmd->op_flags & SCST_INFO_VALID))
784                         goto out_invalid;
785                 break;
786
787         case SCST_USER_PARSE_EXCEPTION:
788                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
789                 rc = dev->generic_parse(cmd, dev_user_get_block);
790                 if ((rc == 0) && (cmd->op_flags & SCST_INFO_VALID))
791                         break;
792                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
793                         TRACE_MEM("Restarting PARSE to thread context "
794                                 "(ucmd %p)", ucmd);
795                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
796                         goto out;
797                 }
798                 /* else go through */
799
800         case SCST_USER_PARSE_CALL:
801                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
802                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
803                 ucmd->user_cmd_payload_len =
804                         offsetof(struct scst_user_get_cmd, parse_cmd) +
805                         sizeof(ucmd->user_cmd.parse_cmd);
806                 ucmd->user_cmd.cmd_h = ucmd->h;
807                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
808                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
809                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb, cmd->cdb_len);
810                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
811                 ucmd->user_cmd.parse_cmd.ext_cdb_len = cmd->ext_cdb_len;
812                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
813                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
814                 ucmd->user_cmd.parse_cmd.in_bufflen = cmd->in_bufflen;
815                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
816                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
817                 ucmd->user_cmd.parse_cmd.expected_values_set =
818                                         cmd->expected_values_set;
819                 ucmd->user_cmd.parse_cmd.expected_data_direction =
820                                         cmd->expected_data_direction;
821                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
822                                         cmd->expected_transfer_len;
823                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
824                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
825                 ucmd->user_cmd.parse_cmd.op_flags = cmd->op_flags;
826                 ucmd->state = UCMD_STATE_PARSING;
827                 dev_user_add_to_ready(ucmd);
828                 res = SCST_CMD_STATE_STOP;
829                 goto out;
830
831         default:
832                 sBUG();
833                 goto out;
834         }
835
836 alloc:
837         if (cmd->bufflen == 0) {
838                 /*
839                  * According to SPC bufflen 0 for data transfer commands isn't
840                  * an error, so we need to fix the transfer direction.
841                  */
842                 cmd->data_direction = SCST_DATA_NONE;
843         }
844
845         if (cmd->data_direction != SCST_DATA_NONE)
846                 res = dev_user_alloc_space(ucmd);
847
848 out:
849         TRACE_EXIT_RES(res);
850         return res;
851
852 out_invalid:
853         PRINT_ERROR("PARSE failed (ucmd %p, rc %d)", ucmd, rc);
854         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
855
856 out_error:
857         res = scst_get_cmd_abnormal_done_state(cmd);
858         goto out;
859 }
860
861 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
862 {
863         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
864         unsigned long start = buf_ucmd->ubuff;
865         int i, bufflen = ucmd->cmd->bufflen;
866
867         TRACE_ENTRY();
868
869         if (start == 0)
870                 goto out;
871
872         /*
873          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
874          * faster, since it should be cache hot, while ucmd->buf_ucmd and
875          * buf_ucmd->data_pages are cache cold. But, from other side,
876          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
877          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
878          * data_pages array can lead to less cache misses. So, real numbers are
879          * needed. ToDo.
880          */
881
882         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
883                 struct page *page;
884                 page = buf_ucmd->data_pages[i];
885 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
886                 struct vm_area_struct *vma = find_vma(current->mm, start);
887                 if (vma != NULL)
888                         flush_anon_page(vma, page, start);
889 #endif
890                 flush_dcache_page(page);
891                 start += PAGE_SIZE;
892                 bufflen -= PAGE_SIZE;
893         }
894
895 out:
896         TRACE_EXIT();
897         return;
898 }
899
900 static int dev_user_exec(struct scst_cmd *cmd)
901 {
902         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
903         int res = SCST_EXEC_COMPLETED;
904
905         TRACE_ENTRY();
906
907 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
908        * context to complete the necessary actions, but all we are going to
909        * do in this function is, in fact, atomic, so let's skip this check.
910        */
911         if (scst_cmd_atomic(cmd)) {
912                 TRACE_DBG("%s", "User exec() can not be called in atomic "
913                         "context, rescheduling to the thread");
914                 res = SCST_EXEC_NEED_THREAD;
915                 goto out;
916         }
917 #endif
918
919         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
920                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
921                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
922
923         if (cmd->data_direction & SCST_DATA_WRITE)
924                 dev_user_flush_dcache(ucmd);
925
926         BUILD_BUG_ON(sizeof(ucmd->user_cmd.exec_cmd.cdb) != sizeof(cmd->cdb));
927
928         ucmd->user_cmd_payload_len =
929                 offsetof(struct scst_user_get_cmd, exec_cmd) +
930                 sizeof(ucmd->user_cmd.exec_cmd);
931         ucmd->user_cmd.cmd_h = ucmd->h;
932         ucmd->user_cmd.subcode = SCST_USER_EXEC;
933         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
934         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb, cmd->cdb_len);
935         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
936         ucmd->user_cmd.exec_cmd.ext_cdb_len = cmd->ext_cdb_len;
937         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
938         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
939         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
940         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
941                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
942                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
943         }
944         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
945         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
946         ucmd->user_cmd.exec_cmd.partial = 0;
947         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
948         ucmd->user_cmd.exec_cmd.p_in_buf = ucmd->ubuff +
949                                                 (cmd->sg_cnt << PAGE_SHIFT);
950         ucmd->user_cmd.exec_cmd.in_bufflen = cmd->in_bufflen;
951         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
952
953         ucmd->state = UCMD_STATE_EXECING;
954
955         dev_user_add_to_ready(ucmd);
956
957         TRACE_EXIT_RES(res);
958         return res;
959 }
960
961 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
962 {
963         if (ucmd->sgv != NULL) {
964                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
965                 ucmd->sgv = NULL;
966         } else if (ucmd->data_pages != NULL) {
967                 /* We mapped pages, but for some reason didn't allocate them */
968                 ucmd_get(ucmd);
969                 __dev_user_free_sg_entries(ucmd);
970         }
971         return;
972 }
973
974 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
975 {
976         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
977
978         TRACE_ENTRY();
979
980         if (unlikely(ucmd == NULL))
981                 goto out;
982
983         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
984                 ucmd->buff_cached, ucmd->ubuff);
985
986         ucmd->cmd = NULL;
987         if ((cmd->data_direction & SCST_DATA_WRITE) && ucmd->buf_ucmd != NULL)
988                 ucmd->buf_ucmd->buf_dirty = 1;
989
990         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
991                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
992                 /* The state assignment must be before freeing sgv! */
993                 goto out_reply;
994         }
995
996         if (unlikely(!ucmd->seen_by_user)) {
997                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
998                 goto out_reply;
999         }
1000
1001         ucmd->user_cmd_payload_len =
1002                 offsetof(struct scst_user_get_cmd, on_free_cmd) +
1003                 sizeof(ucmd->user_cmd.on_free_cmd);
1004         ucmd->user_cmd.cmd_h = ucmd->h;
1005         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
1006         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
1007         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
1008         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
1009         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
1010         ucmd->user_cmd.on_free_cmd.status = cmd->status;
1011         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
1012
1013         ucmd->state = UCMD_STATE_ON_FREEING;
1014
1015         dev_user_add_to_ready(ucmd);
1016
1017 out:
1018         TRACE_EXIT();
1019         return;
1020
1021 out_reply:
1022         dev_user_process_reply_on_free(ucmd);
1023         goto out;
1024 }
1025
1026 static void dev_user_set_block(struct scst_cmd *cmd, int block)
1027 {
1028         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
1029         /*
1030          * No need for locks here, since *_detach() can not be
1031          * called, when there are existing commands.
1032          */
1033         TRACE_DBG("dev %p, new block %d", dev, block);
1034         if (block != 0)
1035                 dev->block = block;
1036         else
1037                 dev->block = dev->def_block;
1038         return;
1039 }
1040
1041 static int dev_user_disk_done(struct scst_cmd *cmd)
1042 {
1043         int res = SCST_CMD_STATE_DEFAULT;
1044
1045         TRACE_ENTRY();
1046
1047         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
1048
1049         TRACE_EXIT_RES(res);
1050         return res;
1051 }
1052
1053 static int dev_user_tape_done(struct scst_cmd *cmd)
1054 {
1055         int res = SCST_CMD_STATE_DEFAULT;
1056
1057         TRACE_ENTRY();
1058
1059         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
1060
1061         TRACE_EXIT_RES(res);
1062         return res;
1063 }
1064
1065 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
1066 {
1067         struct scst_user_dev *dev = ucmd->dev;
1068         unsigned long flags;
1069         int do_wake = in_interrupt();
1070
1071         TRACE_ENTRY();
1072
1073         if (ucmd->cmd)
1074                 do_wake |= ucmd->cmd->preprocessing_only;
1075
1076         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1077
1078         ucmd->this_state_unjammed = 0;
1079
1080         if ((ucmd->state == UCMD_STATE_PARSING) ||
1081             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1082                 /*
1083                  * If we don't put such commands in the queue head, then under
1084                  * high load we might delay threads, waiting for memory
1085                  * allocations, for too long and start loosing NOPs, which
1086                  * would lead to consider us by remote initiators as
1087                  * unresponsive and stuck => broken connections, etc. If none
1088                  * of our commands completed in NOP timeout to allow the head
1089                  * commands to go, then we are really overloaded and/or stuck.
1090                  */
1091                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1092                         "cmd list", ucmd, ucmd->state);
1093                 list_add(&ucmd->ready_cmd_list_entry,
1094                         &dev->ready_cmd_list);
1095         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1096                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1097                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1098                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1099                         "ready cmd list", ucmd, ucmd->state);
1100                 list_add(&ucmd->ready_cmd_list_entry,
1101                         &dev->ready_cmd_list);
1102                 do_wake = 1;
1103         } else {
1104                 if ((ucmd->cmd != NULL) &&
1105                     unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1106                         TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list",
1107                                 ucmd);
1108                         list_add(&ucmd->ready_cmd_list_entry,
1109                                 &dev->ready_cmd_list);
1110                 } else {
1111                         TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1112                         list_add_tail(&ucmd->ready_cmd_list_entry,
1113                                       &dev->ready_cmd_list);
1114                 }
1115                 do_wake |= ((ucmd->state == UCMD_STATE_ON_CACHE_FREEING) ||
1116                             (ucmd->state == UCMD_STATE_ON_FREEING));
1117         }
1118
1119         if (do_wake) {
1120                 TRACE_DBG("Waking up dev %p", dev);
1121                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1122         }
1123
1124         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1125
1126         TRACE_EXIT();
1127         return;
1128 }
1129
1130 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1131         int num_pg)
1132 {
1133         int res = 0, rc;
1134         int i;
1135         struct task_struct *tsk = current;
1136
1137         TRACE_ENTRY();
1138
1139         if (unlikely(ubuff == 0))
1140                 goto out_nomem;
1141
1142         sBUG_ON(ucmd->data_pages != NULL);
1143
1144         ucmd->num_data_pages = num_pg;
1145
1146         ucmd->data_pages =
1147                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1148                           GFP_KERNEL);
1149         if (ucmd->data_pages == NULL) {
1150                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1151                         "(num_data_pages=%d)", ucmd->num_data_pages);
1152                 res = -ENOMEM;
1153                 goto out_nomem;
1154         }
1155
1156         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1157                 " first_page_offset %d, len %d)", ucmd, ubuff,
1158                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1159                 (ucmd->cmd != NULL) ? ucmd->cmd->bufflen : -1);
1160
1161         down_read(&tsk->mm->mmap_sem);
1162         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1163                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1164         up_read(&tsk->mm->mmap_sem);
1165
1166         /* get_user_pages() flushes dcache */
1167
1168         if (rc < ucmd->num_data_pages)
1169                 goto out_unmap;
1170
1171         ucmd->ubuff = ubuff;
1172         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1173
1174 out:
1175         TRACE_EXIT_RES(res);
1176         return res;
1177
1178 out_nomem:
1179         if (ucmd->cmd != NULL)
1180                 scst_set_busy(ucmd->cmd);
1181         /* go through */
1182
1183 out_err:
1184         if (ucmd->cmd != NULL)
1185                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1186         goto out;
1187
1188 out_unmap:
1189         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1190                 ucmd->num_data_pages, rc);
1191         if (rc > 0) {
1192                 for (i = 0; i < rc; i++)
1193                         page_cache_release(ucmd->data_pages[i]);
1194         }
1195         kfree(ucmd->data_pages);
1196         ucmd->data_pages = NULL;
1197         res = -EFAULT;
1198         if (ucmd->cmd != NULL)
1199                 scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1200         goto out_err;
1201 }
1202
1203 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1204         struct scst_user_reply_cmd *reply)
1205 {
1206         int res = 0;
1207         struct scst_cmd *cmd = ucmd->cmd;
1208
1209         TRACE_ENTRY();
1210
1211         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1212
1213         if (likely(reply->alloc_reply.pbuf != 0)) {
1214                 int pages;
1215                 if (ucmd->buff_cached) {
1216                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1217                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1218                                         "page aligned",
1219                                         reply->alloc_reply.pbuf);
1220                                 goto out_hwerr;
1221                         }
1222                         pages = cmd->sg_cnt;
1223                 } else
1224                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1225                                             cmd->bufflen);
1226                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1227         } else {
1228                 scst_set_busy(ucmd->cmd);
1229                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1230         }
1231
1232 out_process:
1233         scst_post_parse_process_active_cmd(cmd, false);
1234
1235         TRACE_EXIT_RES(res);
1236         return res;
1237
1238 out_hwerr:
1239         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1240         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1241         res = -EINVAL;
1242         goto out_process;
1243 }
1244
1245 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1246         struct scst_user_reply_cmd *reply)
1247 {
1248         int res = 0;
1249         struct scst_user_scsi_cmd_reply_parse *preply =
1250                 &reply->parse_reply;
1251         struct scst_cmd *cmd = ucmd->cmd;
1252
1253         TRACE_ENTRY();
1254
1255         if (preply->status != 0)
1256                 goto out_status;
1257
1258         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1259                 goto out_inval;
1260
1261         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1262                      (preply->data_direction != SCST_DATA_READ) &&
1263                      (preply->data_direction != SCST_DATA_BIDI) &&
1264                      (preply->data_direction != SCST_DATA_NONE)))
1265                 goto out_inval;
1266
1267         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1268                      (preply->bufflen == 0)))
1269                 goto out_inval;
1270
1271         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1272                 goto out_inval;
1273
1274         if (unlikely(preply->cdb_len > SCST_MAX_CDB_SIZE))
1275                 goto out_inval;
1276
1277         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1278                 "data_len %d, pbuf %llx, cdb_len %d, op_flags %x", ucmd,
1279                 preply->queue_type, preply->data_direction, preply->bufflen,
1280                 preply->data_len, reply->alloc_reply.pbuf, preply->cdb_len,
1281                 preply->op_flags);
1282
1283         cmd->queue_type = preply->queue_type;
1284         cmd->data_direction = preply->data_direction;
1285         cmd->bufflen = preply->bufflen;
1286         cmd->data_len = preply->data_len;
1287         if (preply->cdb_len > 0)
1288                 cmd->cdb_len = preply->cdb_len;
1289         if (preply->op_flags & SCST_INFO_VALID)
1290                 cmd->op_flags = preply->op_flags;
1291
1292 out_process:
1293         scst_post_parse_process_active_cmd(cmd, false);
1294
1295         TRACE_EXIT_RES(res);
1296         return res;
1297
1298 out_inval:
1299         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1300                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1301         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1302         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1303         res = -EINVAL;
1304         goto out_abnormal;
1305
1306 out_hwerr_res_set:
1307         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1308
1309 out_abnormal:
1310         scst_set_cmd_abnormal_done_state(cmd);
1311         goto out_process;
1312
1313 out_status:
1314         TRACE_DBG("ucmd %p returned with error from user status %x",
1315                 ucmd, preply->status);
1316
1317         if (preply->sense_len != 0) {
1318                 int sense_len;
1319
1320                 res = scst_alloc_sense(cmd, 0);
1321                 if (res != 0)
1322                         goto out_hwerr_res_set;
1323
1324                 sense_len = min_t(int, cmd->sense_buflen, preply->sense_len);
1325
1326                 res = copy_from_user(cmd->sense,
1327                         (void __user *)(unsigned long)preply->psense_buffer,
1328                         sense_len);
1329                 if (res < 0) {
1330                         PRINT_ERROR("%s", "Unable to get sense data");
1331                         goto out_hwerr_res_set;
1332                 }
1333                 cmd->sense_valid_len = sense_len;
1334         }
1335         scst_set_cmd_error_status(cmd, preply->status);
1336         goto out_abnormal;
1337 }
1338
1339 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1340 {
1341         int res = 0;
1342
1343         TRACE_ENTRY();
1344
1345         TRACE_DBG("ON FREE ucmd %p", ucmd);
1346
1347         dev_user_free_sgv(ucmd);
1348         ucmd_put(ucmd);
1349
1350         TRACE_EXIT_RES(res);
1351         return res;
1352 }
1353
1354 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1355 {
1356         int res = 0;
1357
1358         TRACE_ENTRY();
1359
1360         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1361
1362         ucmd_put(ucmd);
1363
1364         TRACE_EXIT_RES(res);
1365         return res;
1366 }
1367
1368 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1369         struct scst_user_reply_cmd *reply)
1370 {
1371         int res = 0;
1372         struct scst_user_scsi_cmd_reply_exec *ereply =
1373                 &reply->exec_reply;
1374         struct scst_cmd *cmd = ucmd->cmd;
1375
1376         TRACE_ENTRY();
1377
1378         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1379                 if (ucmd->background_exec) {
1380                         TRACE_DBG("Background ucmd %p finished", ucmd);
1381                         ucmd_put(ucmd);
1382                         goto out;
1383                 }
1384                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1385                         goto out_inval;
1386                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1387                              (ereply->resp_data_len != 0)))
1388                         goto out_inval;
1389         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1390                 if (unlikely(ucmd->background_exec))
1391                         goto out_inval;
1392                 if (unlikely((cmd->data_direction & SCST_DATA_READ) ||
1393                              (cmd->resp_data_len != 0)))
1394                         goto out_inval;
1395                 /*
1396                  * background_exec assignment must be after ucmd get.
1397                  * Otherwise, due to reorder, in dev_user_process_reply()
1398                  * it is possible that ucmd is destroyed before it "got" here.
1399                  */
1400                 ucmd_get_ordered(ucmd);
1401                 ucmd->background_exec = 1;
1402                 TRACE_DBG("Background ucmd %p", ucmd);
1403                 goto out_compl;
1404         } else
1405                 goto out_inval;
1406
1407         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1408                 ereply->status, ereply->resp_data_len);
1409
1410         cmd->atomic = 0;
1411
1412          if (ereply->resp_data_len != 0) {
1413                 if (ucmd->ubuff == 0) {
1414                         int pages, rc;
1415                         if (unlikely(ereply->pbuf == 0))
1416                                 goto out_busy;
1417                         if (ucmd->buff_cached) {
1418                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1419                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1420                                                 "page aligned", ereply->pbuf);
1421                                         goto out_hwerr;
1422                                 }
1423                                 pages = cmd->sg_cnt;
1424                         } else
1425                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1426                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1427                         if ((rc != 0) || (ucmd->ubuff == 0))
1428                                 goto out_compl;
1429
1430                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1431                         if (unlikely(rc != 0))
1432                                 goto out_busy;
1433                 } else
1434                         dev_user_flush_dcache(ucmd);
1435                 cmd->may_need_dma_sync = 1;
1436                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1437         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1438                 if (ucmd->ubuff == 0)
1439                         cmd->resp_data_len = ereply->resp_data_len;
1440                 else
1441                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1442         }
1443
1444         cmd->status = ereply->status;
1445         if (ereply->sense_len != 0) {
1446                 int sense_len;
1447
1448                 res = scst_alloc_sense(cmd, 0);
1449                 if (res != 0)
1450                         goto out_compl;
1451
1452                 sense_len = min((int)cmd->sense_buflen, (int)ereply->sense_len);
1453
1454                 res = copy_from_user(cmd->sense,
1455                         (void __user *)(unsigned long)ereply->psense_buffer,
1456                         sense_len);
1457                 if (res < 0) {
1458                         PRINT_ERROR("%s", "Unable to get sense data");
1459                         goto out_hwerr_res_set;
1460                 }
1461                 cmd->sense_valid_len = sense_len;
1462         }
1463
1464 out_compl:
1465         cmd->completed = 1;
1466         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1467         /* !! At this point cmd can be already freed !! */
1468
1469 out:
1470         TRACE_EXIT_RES(res);
1471         return res;
1472
1473 out_inval:
1474         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1475                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1476         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1477
1478 out_hwerr:
1479         res = -EINVAL;
1480
1481 out_hwerr_res_set:
1482         if (ucmd->background_exec) {
1483                 ucmd_put(ucmd);
1484                 goto out;
1485         } else {
1486                 scst_set_cmd_error(cmd,
1487                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1488                 goto out_compl;
1489         }
1490
1491 out_busy:
1492         scst_set_busy(cmd);
1493         goto out_compl;
1494 }
1495
1496 static int dev_user_process_reply(struct scst_user_dev *dev,
1497         struct scst_user_reply_cmd *reply)
1498 {
1499         int res = 0;
1500         struct scst_user_cmd *ucmd;
1501         int state;
1502
1503         TRACE_ENTRY();
1504
1505         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1506
1507         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1508         if (unlikely(ucmd == NULL)) {
1509                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1510                 res = -ESRCH;
1511                 goto out_unlock;
1512         }
1513
1514         if (unlikely(ucmd_get_check(ucmd))) {
1515                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1516                 res = -ESRCH;
1517                 goto out_unlock;
1518         }
1519
1520         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1521         smp_mb();
1522         if (ucmd->background_exec) {
1523                 state = UCMD_STATE_EXECING;
1524                 goto unlock_process;
1525         }
1526
1527         if (unlikely(ucmd->this_state_unjammed)) {
1528                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1529                         ucmd);
1530                 goto out_unlock_put;
1531         }
1532
1533         if (unlikely(!ucmd->sent_to_user)) {
1534                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1535                         "state %x", ucmd, ucmd->state);
1536                 res = -EINVAL;
1537                 goto out_unlock_put;
1538         }
1539
1540         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1541                 goto out_wrong_state;
1542
1543         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1544                 goto out_wrong_state;
1545
1546         state = ucmd->state;
1547         ucmd->sent_to_user = 0;
1548
1549 unlock_process:
1550         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1551
1552         switch (state) {
1553         case UCMD_STATE_PARSING:
1554                 res = dev_user_process_reply_parse(ucmd, reply);
1555                 break;
1556
1557         case UCMD_STATE_BUF_ALLOCING:
1558                 res = dev_user_process_reply_alloc(ucmd, reply);
1559                 break;
1560
1561         case UCMD_STATE_EXECING:
1562                 res = dev_user_process_reply_exec(ucmd, reply);
1563                 break;
1564
1565         case UCMD_STATE_ON_FREEING:
1566                 res = dev_user_process_reply_on_free(ucmd);
1567                 break;
1568
1569         case UCMD_STATE_ON_CACHE_FREEING:
1570                 res = dev_user_process_reply_on_cache_free(ucmd);
1571                 break;
1572
1573         case UCMD_STATE_TM_EXECING:
1574                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1575                 break;
1576
1577         case UCMD_STATE_ATTACH_SESS:
1578         case UCMD_STATE_DETACH_SESS:
1579                 res = dev_user_process_reply_sess(ucmd, reply->result);
1580                 break;
1581
1582         default:
1583                 sBUG();
1584                 break;
1585         }
1586
1587 out_put:
1588         ucmd_put(ucmd);
1589
1590 out:
1591         TRACE_EXIT_RES(res);
1592         return res;
1593
1594 out_wrong_state:
1595         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1596                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1597                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1598                 reply->subcode, ucmd->user_cmd.subcode);
1599         res = -EINVAL;
1600         dev_user_unjam_cmd(ucmd, 0, NULL);
1601
1602 out_unlock_put:
1603         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1604         goto out_put;
1605
1606 out_unlock:
1607         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1608         goto out;
1609 }
1610
1611 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1612 {
1613         int res = 0;
1614         struct scst_user_dev *dev;
1615         struct scst_user_reply_cmd reply;
1616
1617         TRACE_ENTRY();
1618
1619         mutex_lock(&dev_priv_mutex);
1620         dev = (struct scst_user_dev *)file->private_data;
1621         res = dev_user_check_reg(dev);
1622         if (unlikely(res != 0)) {
1623                 mutex_unlock(&dev_priv_mutex);
1624                 goto out;
1625         }
1626         down_read(&dev->dev_rwsem);
1627         mutex_unlock(&dev_priv_mutex);
1628
1629         res = copy_from_user(&reply, arg, sizeof(reply));
1630         if (unlikely(res < 0))
1631                 goto out_up;
1632
1633         TRACE_DBG("Reply for dev %s", dev->name);
1634
1635         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1636
1637         res = dev_user_process_reply(dev, &reply);
1638         if (unlikely(res < 0))
1639                 goto out_up;
1640
1641 out_up:
1642         up_read(&dev->dev_rwsem);
1643
1644 out:
1645         TRACE_EXIT_RES(res);
1646         return res;
1647 }
1648
1649 static int dev_user_get_ext_cdb(struct file *file, void __user *arg)
1650 {
1651         int res = 0;
1652         struct scst_user_dev *dev;
1653         struct scst_user_cmd *ucmd;
1654         struct scst_cmd *cmd = NULL;
1655         struct scst_user_get_ext_cdb get;
1656
1657         TRACE_ENTRY();
1658
1659         mutex_lock(&dev_priv_mutex);
1660         dev = (struct scst_user_dev *)file->private_data;
1661         res = dev_user_check_reg(dev);
1662         if (unlikely(res != 0)) {
1663                 mutex_unlock(&dev_priv_mutex);
1664                 goto out;
1665         }
1666         down_read(&dev->dev_rwsem);
1667         mutex_unlock(&dev_priv_mutex);
1668
1669         res = copy_from_user(&get, arg, sizeof(get));
1670         if (unlikely(res < 0))
1671                 goto out_up;
1672
1673         TRACE_MGMT_DBG("Get ext cdb for dev %s", dev->name);
1674
1675         TRACE_BUFFER("Get ext cdb", &get, sizeof(get));
1676
1677         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1678
1679         ucmd = __ucmd_find_hash(dev, get.cmd_h);
1680         if (unlikely(ucmd == NULL)) {
1681                 TRACE_MGMT_DBG("cmd_h %d not found", get.cmd_h);
1682                 res = -ESRCH;
1683                 goto out_unlock;
1684         }
1685
1686         if (unlikely(ucmd_get_check(ucmd))) {
1687                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", get.cmd_h);
1688                 res = -ESRCH;
1689                 goto out_unlock;
1690         }
1691
1692         if ((ucmd->cmd != NULL) && (ucmd->state <= UCMD_STATE_EXECING) &&
1693             (ucmd->sent_to_user || ucmd->background_exec)) {
1694                 cmd = ucmd->cmd;
1695                 scst_cmd_get(cmd);
1696         } else {
1697                 TRACE_MGMT_DBG("Invalid ucmd state %d for cmd_h %d",
1698                         ucmd->state, get.cmd_h);
1699                 res = -EINVAL;
1700                 goto out_unlock;
1701         }
1702
1703         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1704
1705         if (cmd == NULL)
1706                 goto out_put;
1707
1708         if (cmd->ext_cdb == NULL)
1709                 goto out_cmd_put;
1710
1711         TRACE_BUFFER("EXT CDB", cmd->ext_cdb, cmd->ext_cdb_len);
1712         res = copy_to_user((void __user *)(unsigned long)get.ext_cdb_buffer,
1713                 cmd->ext_cdb, cmd->ext_cdb_len);
1714
1715 out_cmd_put:
1716         scst_cmd_put(cmd);
1717
1718 out_put:
1719         ucmd_put(ucmd);
1720
1721 out_up:
1722         up_read(&dev->dev_rwsem);
1723
1724 out:
1725         TRACE_EXIT_RES(res);
1726         return res;
1727
1728 out_unlock:
1729         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1730         goto out_up;
1731 }
1732
1733 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1734         __releases(&dev->cmd_lists.cmd_list_lock)
1735         __acquires(&dev->cmd_lists.cmd_list_lock)
1736 {
1737         int res = 0;
1738
1739         TRACE_ENTRY();
1740
1741         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1742                 struct scst_cmd *cmd = list_entry(
1743                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1744                         cmd_list_entry);
1745                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1746                 list_del(&cmd->cmd_list_entry);
1747                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1748                 scst_process_active_cmd(cmd, false);
1749                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1750                 res++;
1751         }
1752
1753         TRACE_EXIT_RES(res);
1754         return res;
1755 }
1756
1757 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1758 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1759         __releases(&dev->cmd_lists.cmd_list_lock)
1760         __acquires(&dev->cmd_lists.cmd_list_lock)
1761 {
1762         struct scst_user_cmd *u;
1763
1764 again:
1765         u = NULL;
1766         if (!list_empty(cmd_list)) {
1767                 u = list_entry(cmd_list->next, typeof(*u),
1768                                ready_cmd_list_entry);
1769
1770                 TRACE_DBG("Found ready ucmd %p", u);
1771                 list_del(&u->ready_cmd_list_entry);
1772
1773                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1774
1775                 if (u->cmd != NULL) {
1776                         if (u->state == UCMD_STATE_EXECING) {
1777                                 struct scst_user_dev *dev = u->dev;
1778                                 int rc;
1779
1780                                 EXTRACHECKS_BUG_ON(u->jammed);
1781
1782                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1783
1784                                 rc = scst_check_local_events(u->cmd);
1785                                 if (unlikely(rc != 0)) {
1786                                         u->cmd->scst_cmd_done(u->cmd,
1787                                                 SCST_CMD_STATE_DEFAULT,
1788                                                 SCST_CONTEXT_DIRECT);
1789                                         /*
1790                                          * !! At this point cmd & u can be !!
1791                                          * !! already freed                !!
1792                                          */
1793                                         spin_lock_irq(
1794                                                 &dev->cmd_lists.cmd_list_lock);
1795                                         goto again;
1796                                 }
1797
1798                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1799                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1800                                         &u->cmd->cmd_flags))) {
1801                                 switch (u->state) {
1802                                 case UCMD_STATE_PARSING:
1803                                 case UCMD_STATE_BUF_ALLOCING:
1804                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1805                                         dev_user_unjam_cmd(u, 0, NULL);
1806                                         goto again;
1807                                 case UCMD_STATE_EXECING:
1808                                         EXTRACHECKS_BUG_ON(1);
1809                                 }
1810                         }
1811                 }
1812                 u->sent_to_user = 1;
1813                 u->seen_by_user = 1;
1814         }
1815         return u;
1816 }
1817
1818 static inline int test_cmd_lists(struct scst_user_dev *dev)
1819 {
1820         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1821                   !list_empty(&dev->ready_cmd_list) ||
1822                   !dev->blocking || dev->cleanup_done ||
1823                   signal_pending(current);
1824         return res;
1825 }
1826
1827 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1828 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1829         struct scst_user_cmd **ucmd)
1830 {
1831         int res = 0;
1832         wait_queue_t wait;
1833
1834         TRACE_ENTRY();
1835
1836         init_waitqueue_entry(&wait, current);
1837
1838         while (1) {
1839                 if (!test_cmd_lists(dev)) {
1840                         add_wait_queue_exclusive_head(
1841                                 &dev->cmd_lists.cmd_list_waitQ,
1842                                 &wait);
1843                         for (;;) {
1844                                 set_current_state(TASK_INTERRUPTIBLE);
1845                                 if (test_cmd_lists(dev))
1846                                         break;
1847                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1848                                 schedule();
1849                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1850                         }
1851                         set_current_state(TASK_RUNNING);
1852                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1853                                 &wait);
1854                 }
1855
1856                 dev_user_process_scst_commands(dev);
1857
1858                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1859                 if (*ucmd != NULL)
1860                         break;
1861
1862                 if (!dev->blocking || dev->cleanup_done) {
1863                         res = -EAGAIN;
1864                         TRACE_DBG("No ready commands, returning %d", res);
1865                         break;
1866                 }
1867
1868                 if (signal_pending(current)) {
1869                         res = -EINTR;
1870                         TRACE_DBG("Signal pending, returning %d", res);
1871                         break;
1872                 }
1873         }
1874
1875         TRACE_EXIT_RES(res);
1876         return res;
1877 }
1878
1879 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1880 {
1881         int res = 0;
1882         struct scst_user_dev *dev;
1883         struct scst_user_get_cmd *cmd;
1884         struct scst_user_reply_cmd *reply;
1885         struct scst_user_cmd *ucmd;
1886         uint64_t ureply;
1887
1888         TRACE_ENTRY();
1889
1890         mutex_lock(&dev_priv_mutex);
1891         dev = (struct scst_user_dev *)file->private_data;
1892         res = dev_user_check_reg(dev);
1893         if (unlikely(res != 0)) {
1894                 mutex_unlock(&dev_priv_mutex);
1895                 goto out;
1896         }
1897         down_read(&dev->dev_rwsem);
1898         mutex_unlock(&dev_priv_mutex);
1899
1900         /* get_user() can't be used with 64-bit values on x86_32 */
1901         res = copy_from_user(&ureply, (uint64_t __user *)
1902                 &((struct scst_user_get_cmd __user *)arg)->preply,
1903                 sizeof(ureply));
1904         if (unlikely(res < 0))
1905                 goto out_up;
1906
1907         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1908                 dev->name);
1909
1910         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1911         if (unlikely(cmd == NULL)) {
1912                 res = -ENOMEM;
1913                 goto out_up;
1914         }
1915
1916         if (ureply != 0) {
1917                 unsigned long u = (unsigned long)ureply;
1918                 reply = (struct scst_user_reply_cmd *)cmd;
1919                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1920                 if (unlikely(res < 0))
1921                         goto out_free;
1922
1923                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1924
1925                 res = dev_user_process_reply(dev, reply);
1926                 if (unlikely(res < 0))
1927                         goto out_free;
1928         }
1929
1930         kmem_cache_free(user_get_cmd_cachep, cmd);
1931
1932         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1933 again:
1934         res = dev_user_get_next_cmd(dev, &ucmd);
1935         if (res == 0) {
1936                 int len;
1937                 /*
1938                  * A misbehaving user space handler can make ucmd to get dead
1939                  * immediately after we released the lock, which can lead to
1940                  * copy of dead data to the user space, which can lead to a
1941                  * leak of sensitive information.
1942                  */
1943                 if (unlikely(ucmd_get_check(ucmd))) {
1944                         /* Oops, this ucmd is already being destroyed. Retry. */
1945                         goto again;
1946                 }
1947                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1948
1949                 EXTRACHECKS_BUG_ON(ucmd->user_cmd_payload_len == 0);
1950
1951                 len = ucmd->user_cmd_payload_len;
1952                 TRACE_DBG("ucmd %p (user_cmd %p), payload_len %d (len %d)",
1953                         ucmd, &ucmd->user_cmd, ucmd->user_cmd_payload_len, len);
1954                 TRACE_BUFFER("UCMD", &ucmd->user_cmd, len);
1955                 res = copy_to_user(arg, &ucmd->user_cmd, len);
1956                 if (unlikely(res != 0)) {
1957                         /* Requeue ucmd back */
1958                         TRACE_DBG("Copy to user failed (%d), requeuing ucmd %p "
1959                                 "back to head of ready cmd list", res, ucmd);
1960                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1961                         list_add(&ucmd->ready_cmd_list_entry,
1962                                 &dev->ready_cmd_list);
1963                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1964                 }
1965 #ifdef CONFIG_SCST_EXTRACHECKS
1966                 else
1967                         ucmd->user_cmd_payload_len = 0;
1968 #endif
1969                 ucmd_put(ucmd);
1970         } else
1971                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1972
1973 out_up:
1974         up_read(&dev->dev_rwsem);
1975
1976 out:
1977         TRACE_EXIT_RES(res);
1978         return res;
1979
1980 out_free:
1981         kmem_cache_free(user_get_cmd_cachep, cmd);
1982         goto out_up;
1983 }
1984
1985 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1986         unsigned long arg)
1987 {
1988         long res;
1989
1990         TRACE_ENTRY();
1991
1992         switch (cmd) {
1993         case SCST_USER_REPLY_AND_GET_CMD:
1994                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1995                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1996                 break;
1997
1998         case SCST_USER_REPLY_CMD:
1999                 TRACE_DBG("%s", "REPLY_CMD");
2000                 res = dev_user_reply_cmd(file, (void __user *)arg);
2001                 break;
2002
2003         case SCST_USER_GET_EXTENDED_CDB:
2004                 TRACE_DBG("%s", "GET_EXTENDED_CDB");
2005                 res = dev_user_get_ext_cdb(file, (void __user *)arg);
2006                 break;
2007
2008         case SCST_USER_REGISTER_DEVICE:
2009         {
2010                 struct scst_user_dev_desc *dev_desc;
2011                 TRACE_DBG("%s", "REGISTER_DEVICE");
2012                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
2013                 if (dev_desc == NULL) {
2014                         res = -ENOMEM;
2015                         goto out;
2016                 }
2017                 res = copy_from_user(dev_desc, (void __user *)arg,
2018                                      sizeof(*dev_desc));
2019                 if (res < 0) {
2020                         kfree(dev_desc);
2021                         goto out;
2022                 }
2023                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
2024                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
2025                 dev_desc->sgv_name[sizeof(dev_desc->sgv_name)-1] = '\0';
2026                 res = dev_user_register_dev(file, dev_desc);
2027                 kfree(dev_desc);
2028                 break;
2029         }
2030
2031         case SCST_USER_UNREGISTER_DEVICE:
2032                 TRACE_DBG("%s", "UNREGISTER_DEVICE");
2033                 res = dev_user_unregister_dev(file);
2034                 break;
2035
2036         case SCST_USER_FLUSH_CACHE:
2037                 TRACE_DBG("%s", "FLUSH_CACHE");
2038                 res = dev_user_flush_cache(file);
2039                 break;
2040
2041         case SCST_USER_SET_OPTIONS:
2042         {
2043                 struct scst_user_opt opt;
2044                 TRACE_DBG("%s", "SET_OPTIONS");
2045                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
2046                 if (res < 0)
2047                         goto out;
2048                 TRACE_BUFFER("opt", &opt, sizeof(opt));
2049                 res = dev_user_set_opt(file, &opt);
2050                 break;
2051         }
2052
2053         case SCST_USER_GET_OPTIONS:
2054                 TRACE_DBG("%s", "GET_OPTIONS");
2055                 res = dev_user_get_opt(file, (void __user *)arg);
2056                 break;
2057
2058         case SCST_USER_DEVICE_CAPACITY_CHANGED:
2059                 TRACE_DBG("%s", "CAPACITY_CHANGED");
2060                 res = dev_user_capacity_changed(file);
2061                 break;
2062
2063         case SCST_USER_PREALLOC_BUFFER:
2064                 TRACE_DBG("%s", "PREALLOC_BUFFER");
2065                 res = dev_user_prealloc_buffer(file, (void __user *)arg);
2066                 break;
2067
2068         default:
2069                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
2070                 res = -EINVAL;
2071                 goto out;
2072         }
2073
2074 out:
2075         TRACE_EXIT_RES(res);
2076         return res;
2077 }
2078
2079 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
2080 {
2081         int res = 0;
2082         struct scst_user_dev *dev;
2083
2084         TRACE_ENTRY();
2085
2086         mutex_lock(&dev_priv_mutex);
2087         dev = (struct scst_user_dev *)file->private_data;
2088         res = dev_user_check_reg(dev);
2089         if (unlikely(res != 0)) {
2090                 mutex_unlock(&dev_priv_mutex);
2091                 goto out;
2092         }
2093         down_read(&dev->dev_rwsem);
2094         mutex_unlock(&dev_priv_mutex);
2095
2096         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2097
2098         if (!list_empty(&dev->ready_cmd_list) ||
2099             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2100                 res |= POLLIN | POLLRDNORM;
2101                 goto out_unlock;
2102         }
2103
2104         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2105
2106         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
2107         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
2108         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
2109
2110         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2111
2112         if (!list_empty(&dev->ready_cmd_list) ||
2113             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2114                 res |= POLLIN | POLLRDNORM;
2115                 goto out_unlock;
2116         }
2117
2118 out_unlock:
2119         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2120
2121         up_read(&dev->dev_rwsem);
2122
2123 out:
2124         TRACE_EXIT_HRES(res);
2125         return res;
2126 }
2127
2128 /*
2129  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
2130  */
2131 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
2132         unsigned long *flags)
2133         __releases(&dev->cmd_lists.cmd_list_lock)
2134         __acquires(&dev->cmd_lists.cmd_list_lock)
2135 {
2136         int state = ucmd->state;
2137         struct scst_user_dev *dev = ucmd->dev;
2138
2139         TRACE_ENTRY();
2140
2141         if (ucmd->this_state_unjammed)
2142                 goto out;
2143
2144         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
2145                 state);
2146
2147         ucmd->jammed = 1;
2148         ucmd->this_state_unjammed = 1;
2149         ucmd->sent_to_user = 0;
2150
2151         switch (state) {
2152         case UCMD_STATE_PARSING:
2153         case UCMD_STATE_BUF_ALLOCING:
2154                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2155                         ucmd->aborted = 1;
2156                 else {
2157                         if (busy)
2158                                 scst_set_busy(ucmd->cmd);
2159                         else
2160                                 scst_set_cmd_error(ucmd->cmd,
2161                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2162                 }
2163                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
2164
2165                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
2166                 list_add(&ucmd->cmd->cmd_list_entry,
2167                         &ucmd->cmd->cmd_lists->active_cmd_list);
2168                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
2169                 break;
2170
2171         case UCMD_STATE_EXECING:
2172                 if (flags != NULL)
2173                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2174                                                *flags);
2175                 else
2176                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2177
2178                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
2179
2180                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2181                         ucmd->aborted = 1;
2182                 else {
2183                         if (busy)
2184                                 scst_set_busy(ucmd->cmd);
2185                         else
2186                                 scst_set_cmd_error(ucmd->cmd,
2187                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2188                 }
2189
2190                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
2191                         SCST_CONTEXT_DIRECT);
2192                 /* !! At this point cmd and ucmd can be already freed !! */
2193
2194                 if (flags != NULL)
2195                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2196                                           *flags);
2197                 else
2198                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2199                 break;
2200
2201         case UCMD_STATE_ON_FREEING:
2202         case UCMD_STATE_ON_CACHE_FREEING:
2203         case UCMD_STATE_TM_EXECING:
2204         case UCMD_STATE_ATTACH_SESS:
2205         case UCMD_STATE_DETACH_SESS:
2206                 if (flags != NULL)
2207                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2208                                                *flags);
2209                 else
2210                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2211
2212                 switch (state) {
2213                 case UCMD_STATE_ON_FREEING:
2214                         dev_user_process_reply_on_free(ucmd);
2215                         break;
2216
2217                 case UCMD_STATE_ON_CACHE_FREEING:
2218                         dev_user_process_reply_on_cache_free(ucmd);
2219                         break;
2220
2221                 case UCMD_STATE_TM_EXECING:
2222                         dev_user_process_reply_tm_exec(ucmd,
2223                                                        SCST_MGMT_STATUS_FAILED);
2224                         break;
2225
2226                 case UCMD_STATE_ATTACH_SESS:
2227                 case UCMD_STATE_DETACH_SESS:
2228                         dev_user_process_reply_sess(ucmd, -EFAULT);
2229                         break;
2230                 }
2231
2232                 if (flags != NULL)
2233                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2234                                           *flags);
2235                 else
2236                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2237                 break;
2238
2239         default:
2240                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
2241                 sBUG();
2242                 break;
2243         }
2244
2245 out:
2246         TRACE_EXIT();
2247         return;
2248 }
2249
2250 static int dev_user_unjam_dev(struct scst_user_dev *dev)
2251         __releases(&dev->cmd_lists.cmd_list_lock)
2252         __acquires(&dev->cmd_lists.cmd_list_lock)
2253 {
2254         int i, res = 0;
2255         struct scst_user_cmd *ucmd;
2256
2257         TRACE_ENTRY();
2258
2259         TRACE_MGMT_DBG("Unjamming dev %p", dev);
2260
2261         sgv_pool_flush(dev->pool);
2262         sgv_pool_flush(dev->pool_clust);
2263
2264         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2265
2266 repeat:
2267         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2268                 struct list_head *head = &dev->ucmd_hash[i];
2269
2270                 list_for_each_entry(ucmd, head, hash_list_entry) {
2271                         res++;
2272
2273                         if (!ucmd->sent_to_user)
2274                                 continue;
2275
2276                         if (ucmd_get_check(ucmd))
2277                                 continue;
2278
2279                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2280                                 ucmd->state, ucmd->cmd);
2281
2282                         dev_user_unjam_cmd(ucmd, 0, NULL);
2283
2284                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2285                         ucmd_put(ucmd);
2286                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2287
2288                         goto repeat;
2289                 }
2290         }
2291
2292         if (dev_user_process_scst_commands(dev) != 0)
2293                 goto repeat;
2294
2295         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2296
2297         TRACE_EXIT_RES(res);
2298         return res;
2299 }
2300
2301 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2302         int status)
2303 {
2304         int res = 0;
2305
2306         TRACE_ENTRY();
2307
2308         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2309                 ucmd->user_cmd.tm_cmd.fn, status);
2310
2311         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2312                 /*
2313                  * It is possible that user space seen TM cmd before cmd
2314                  * to abort or will never see it at all, because it was
2315                  * aborted on the way there. So, it is safe to return
2316                  * success instead, because, if there is the TM cmd at this
2317                  * point, then the cmd to abort apparrently does exist.
2318                  */
2319                 status = SCST_MGMT_STATUS_SUCCESS;
2320         }
2321
2322         scst_async_mcmd_completed(ucmd->mcmd, status);
2323
2324         ucmd_put(ucmd);
2325
2326         TRACE_EXIT_RES(res);
2327         return res;
2328 }
2329
2330 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2331 {
2332         struct scst_user_cmd *ucmd;
2333         unsigned long flags;
2334
2335         TRACE_ENTRY();
2336
2337         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2338 again:
2339         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2340                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2341                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2342                         switch (ucmd->state) {
2343                         case UCMD_STATE_PARSING:
2344                         case UCMD_STATE_BUF_ALLOCING:
2345                         case UCMD_STATE_EXECING:
2346                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2347                                 list_del(&ucmd->ready_cmd_list_entry);
2348                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2349                                 goto again;
2350                         }
2351                 }
2352         }
2353
2354         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2355
2356         TRACE_EXIT();
2357         return;
2358 }
2359
2360 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2361         struct scst_tgt_dev *tgt_dev)
2362 {
2363         struct scst_user_cmd *ucmd;
2364         struct scst_user_dev *dev =
2365                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2366         struct scst_user_cmd *ucmd_to_abort = NULL;
2367
2368         TRACE_ENTRY();
2369
2370         /*
2371          * In the used approach we don't do anything with hung devices, which
2372          * stopped responding and/or have stuck commands. We forcedly abort such
2373          * commands only if they not yet sent to the user space or if the device
2374          * is getting unloaded, e.g. if its handler program gets killed. This is
2375          * because it's pretty hard to distinguish between stuck and temporary
2376          * overloaded states of the device. There are several reasons for that:
2377          *
2378          * 1. Some commands need a lot of time to complete (several hours),
2379          *    so for an impatient user such command(s) will always look as
2380          *    stuck.
2381          *
2382          * 2. If we forcedly abort, i.e. abort before it's actually completed
2383          *    in the user space, just one command, we will have to put the whole
2384          *    device offline until we are sure that no more previously aborted
2385          *    commands will get executed. Otherwise, we might have a possibility
2386          *    for data corruption, when aborted and reported as completed
2387          *    command actually gets executed *after* new commands sent
2388          *    after the force abort was done. Many journaling file systems and
2389          *    databases use "provide required commands order via queue draining"
2390          *    approach and not putting the whole device offline after the forced
2391          *    abort will break it. This makes our decision, if a command stuck
2392          *    or not, cost a lot.
2393          *
2394          * So, we leave policy definition if a device stuck or not to
2395          * the user space and simply let all commands live until they are
2396          * completed or their devices get closed/killed. This approach is very
2397          * much OK, but can affect management commands, which need activity
2398          * suspending via scst_suspend_activity() function such as devices or
2399          * targets registration/removal. But during normal life such commands
2400          * should be rare. Plus, when possible, scst_suspend_activity() will
2401          * return after timeout EBUSY status to allow caller to not stuck
2402          * forever as well.
2403          *
2404          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2405          * stuck commands would affect only related devices.
2406          */
2407
2408         dev_user_abort_ready_commands(dev);
2409
2410         /* We can't afford missing TM command due to memory shortage */
2411         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2412
2413         ucmd->user_cmd_payload_len =
2414                 offsetof(struct scst_user_get_cmd, tm_cmd) +
2415                 sizeof(ucmd->user_cmd.tm_cmd);
2416         ucmd->user_cmd.cmd_h = ucmd->h;
2417         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2418         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2419         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2420         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2421         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2422
2423         if (mcmd->cmd_to_abort != NULL) {
2424                 ucmd_to_abort =
2425                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2426                 if (ucmd_to_abort != NULL)
2427                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2428         }
2429
2430         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2431                 "ucmd_to_abort %p, cmd_h_to_abort %d, mcmd %p)", ucmd, ucmd->h,
2432                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2433                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort, mcmd);
2434
2435         ucmd->mcmd = mcmd;
2436         ucmd->state = UCMD_STATE_TM_EXECING;
2437
2438         scst_prepare_async_mcmd(mcmd);
2439
2440         dev_user_add_to_ready(ucmd);
2441
2442         TRACE_EXIT();
2443         return SCST_DEV_TM_NOT_COMPLETED;
2444 }
2445
2446 static int dev_user_attach(struct scst_device *sdev)
2447 {
2448         int res = 0;
2449         struct scst_user_dev *dev = NULL, *d;
2450
2451         TRACE_ENTRY();
2452
2453         spin_lock(&dev_list_lock);
2454         list_for_each_entry(d, &dev_list, dev_list_entry) {
2455                 if (strcmp(d->name, sdev->virt_name) == 0) {
2456                         dev = d;
2457                         break;
2458                 }
2459         }
2460         spin_unlock(&dev_list_lock);
2461         if (dev == NULL) {
2462                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2463                 res = -EINVAL;
2464                 goto out;
2465         }
2466
2467         sdev->p_cmd_lists = &dev->cmd_lists;
2468         sdev->dh_priv = dev;
2469         sdev->tst = dev->tst;
2470         sdev->queue_alg = dev->queue_alg;
2471         sdev->swp = dev->swp;
2472         sdev->tas = dev->tas;
2473         sdev->d_sense = dev->d_sense;
2474         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2475
2476         dev->sdev = sdev;
2477
2478         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2479                 dev->name);
2480
2481 out:
2482         TRACE_EXIT();
2483         return res;
2484 }
2485
2486 static void dev_user_detach(struct scst_device *sdev)
2487 {
2488         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2489
2490         TRACE_ENTRY();
2491
2492         TRACE_DBG("virt_id %d", sdev->virt_id);
2493
2494         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2495                 dev->name);
2496
2497         /* dev will be freed by the caller */
2498         sdev->dh_priv = NULL;
2499         dev->sdev = NULL;
2500
2501         TRACE_EXIT();
2502         return;
2503 }
2504
2505 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2506 {
2507         int res = 0;
2508         unsigned long flags;
2509
2510         TRACE_ENTRY();
2511
2512         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2513
2514         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2515
2516         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2517                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2518                 ucmd->result = status;
2519         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2520                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2521         } else
2522                 sBUG();
2523
2524         if (ucmd->cmpl != NULL)
2525                 complete_all(ucmd->cmpl);
2526
2527         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2528
2529         ucmd_put(ucmd);
2530
2531         TRACE_EXIT_RES(res);
2532         return res;
2533 }
2534
2535 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2536 {
2537         struct scst_user_dev *dev =
2538                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2539         int res = 0, rc;
2540         struct scst_user_cmd *ucmd;
2541         DECLARE_COMPLETION_ONSTACK(cmpl);
2542
2543         TRACE_ENTRY();
2544
2545         /*
2546          * We can't replace tgt_dev->pool, because it can be used to allocate
2547          * memory for SCST local commands, like REPORT LUNS, where there is no
2548          * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
2549          */
2550         if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
2551                 tgt_dev->dh_priv = dev->pool_clust;
2552         else
2553                 tgt_dev->dh_priv = dev->pool;
2554
2555         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2556         if (ucmd == NULL)
2557                 goto out_nomem;
2558
2559         ucmd->cmpl = &cmpl;
2560
2561         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2562                 sizeof(ucmd->user_cmd.sess);
2563         ucmd->user_cmd.cmd_h = ucmd->h;
2564         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2565         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2566         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2567         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2568         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only;
2569         strlcpy(ucmd->user_cmd.sess.initiator_name,
2570                 tgt_dev->sess->initiator_name,
2571                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2572         strlcpy(ucmd->user_cmd.sess.target_name,
2573                 tgt_dev->sess->tgt->tgt_name,
2574                 sizeof(ucmd->user_cmd.sess.target_name)-1);
2575
2576         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2577                 "threads_num %d, rd_only %d, initiator %s, target %s)",
2578                 ucmd, ucmd->h, ucmd->user_cmd.sess.sess_h,
2579                 ucmd->user_cmd.sess.lun, ucmd->user_cmd.sess.threads_num,
2580                 ucmd->user_cmd.sess.rd_only, ucmd->user_cmd.sess.initiator_name,
2581                 ucmd->user_cmd.sess.target_name);
2582
2583         ucmd->state = UCMD_STATE_ATTACH_SESS;
2584
2585         ucmd_get(ucmd);
2586
2587         dev_user_add_to_ready(ucmd);
2588
2589         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2590         if (rc > 0)
2591                 res = ucmd->result;
2592         else {
2593                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2594                 res = -EFAULT;
2595         }
2596
2597         sBUG_ON(irqs_disabled());
2598
2599         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2600         ucmd->cmpl = NULL;
2601         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2602
2603         ucmd_put(ucmd);
2604
2605 out:
2606         TRACE_EXIT_RES(res);
2607         return res;
2608
2609 out_nomem:
2610         res = -ENOMEM;
2611         goto out;
2612 }
2613
2614 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2615 {
2616         struct scst_user_dev *dev =
2617                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2618         struct scst_user_cmd *ucmd;
2619
2620         TRACE_ENTRY();
2621
2622         /*
2623          * We can't miss TM command due to memory shortage, because it might
2624          * lead to a memory leak in the user space handler.
2625          */
2626         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2627         if (ucmd == NULL)
2628                 goto out;
2629
2630         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2631                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2632
2633         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2634                 sizeof(ucmd->user_cmd.sess);
2635         ucmd->user_cmd.cmd_h = ucmd->h;
2636         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2637         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2638
2639         ucmd->state = UCMD_STATE_DETACH_SESS;
2640
2641         dev_user_add_to_ready(ucmd);
2642
2643 out:
2644         TRACE_EXIT();
2645         return;
2646 }
2647
2648 /* No locks are needed, but the activity must be suspended */
2649 static void dev_user_setup_functions(struct scst_user_dev *dev)
2650 {
2651         TRACE_ENTRY();
2652
2653         dev->devtype.parse = dev_user_parse;
2654         dev->devtype.dev_done = NULL;
2655
2656         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2657                 switch (dev->devtype.type) {
2658                 case TYPE_DISK:
2659                         dev->generic_parse = scst_sbc_generic_parse;
2660                         dev->devtype.dev_done = dev_user_disk_done;
2661                         break;
2662
2663                 case TYPE_TAPE:
2664                         dev->generic_parse = scst_tape_generic_parse;
2665                         dev->devtype.dev_done = dev_user_tape_done;
2666                         break;
2667
2668                 case TYPE_MOD:
2669                         dev->generic_parse = scst_modisk_generic_parse;
2670                         dev->devtype.dev_done = dev_user_disk_done;
2671                         break;
2672
2673                 case TYPE_ROM:
2674                         dev->generic_parse = scst_cdrom_generic_parse;
2675                         dev->devtype.dev_done = dev_user_disk_done;
2676                         break;
2677
2678                 case TYPE_MEDIUM_CHANGER:
2679                         dev->generic_parse = scst_changer_generic_parse;
2680                         break;
2681
2682                 case TYPE_PROCESSOR:
2683                         dev->generic_parse = scst_processor_generic_parse;
2684                         break;
2685
2686                 case TYPE_RAID:
2687                         dev->generic_parse = scst_raid_generic_parse;
2688                         break;
2689
2690                 default:
2691                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2692                                 "for it", dev->devtype.type);
2693                         dev->parse_type = SCST_USER_PARSE_CALL;
2694                         break;
2695                 }
2696         } else {
2697                 dev->generic_parse = NULL;
2698                 dev->devtype.dev_done = NULL;
2699         }
2700
2701         TRACE_EXIT();
2702         return;
2703 }
2704
2705 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2706 {
2707         char ver[sizeof(DEV_USER_VERSION)+1];
2708         int res;
2709
2710         res = copy_from_user(ver,
2711                         (void __user *)(unsigned long)dev_desc->version_str,
2712                         sizeof(ver));
2713         if (res < 0) {
2714                 PRINT_ERROR("%s", "Unable to get version string");
2715                 goto out;
2716         }
2717         ver[sizeof(ver)-1] = '\0';
2718
2719         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2720                 /* ->name already 0-terminated in dev_user_ioctl() */
2721                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2722                         dev_desc->name, ver);
2723                 res = -EINVAL;
2724                 goto out;
2725         }
2726
2727 out:
2728         return res;
2729 }
2730
2731 static int dev_user_register_dev(struct file *file,
2732         const struct scst_user_dev_desc *dev_desc)
2733 {
2734         int res, i;
2735         struct scst_user_dev *dev, *d;
2736         int block;
2737
2738         TRACE_ENTRY();
2739
2740         res = dev_user_check_version(dev_desc);
2741         if (res != 0)
2742                 goto out;
2743
2744         switch (dev_desc->type) {
2745         case TYPE_DISK:
2746         case TYPE_ROM:
2747         case TYPE_MOD:
2748                 if (dev_desc->block_size == 0) {
2749                         PRINT_ERROR("Wrong block size %d",
2750                                     dev_desc->block_size);
2751                         res = -EINVAL;
2752                         goto out;
2753                 }
2754                 block = scst_calc_block_shift(dev_desc->block_size);
2755                 if (block == -1) {
2756                         res = -EINVAL;
2757                         goto out;
2758                 }
2759                 break;
2760         default:
2761                 block = dev_desc->block_size;
2762                 break;
2763         }
2764
2765         if (!try_module_get(THIS_MODULE)) {
2766                 PRINT_ERROR("%s", "Fail to get module");
2767                 res = -ETXTBSY;
2768                 goto out;
2769         }
2770
2771         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2772         if (dev == NULL) {
2773                 res = -ENOMEM;
2774                 goto out_put;
2775         }
2776
2777         init_rwsem(&dev->dev_rwsem);
2778         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2779         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2780         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2781         INIT_LIST_HEAD(&dev->ready_cmd_list);
2782         if (file->f_flags & O_NONBLOCK) {
2783                 TRACE_DBG("%s", "Non-blocking operations");
2784                 dev->blocking = 0;
2785         } else
2786                 dev->blocking = 1;
2787         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2788                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2789
2790         strlcpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2791
2792         scst_init_mem_lim(&dev->udev_mem_lim);
2793
2794         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2795                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2796                                                   dev_desc->sgv_name);
2797         dev->pool = sgv_pool_create(dev->devtype.name, sgv_no_clustering,
2798                                         dev_desc->sgv_single_alloc_pages,
2799                                         dev_desc->sgv_shared,
2800                                         dev_desc->sgv_purge_interval);
2801         if (dev->pool == NULL) {
2802                 res = -ENOMEM;
2803                 goto out_free_dev;
2804         }
2805         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2806                 dev_user_free_sg_entries);
2807
2808         if (!dev_desc->sgv_disable_clustered_pool) {
2809                 scnprintf(dev->devtype.name, sizeof(dev->devtype.name),
2810                         "%s-clust",
2811                         (dev_desc->sgv_name[0] == '\0') ? dev->name :
2812                                                           dev_desc->sgv_name);
2813                 dev->pool_clust = sgv_pool_create(dev->devtype.name,
2814                                         sgv_tail_clustering,
2815                                         dev_desc->sgv_single_alloc_pages,
2816                                         dev_desc->sgv_shared,
2817                                         dev_desc->sgv_purge_interval);
2818                 if (dev->pool_clust == NULL) {
2819                         res = -ENOMEM;
2820                         goto out_free0;
2821                 }
2822                 sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
2823                         dev_user_free_sg_entries);
2824         } else {
2825                 dev->pool_clust = dev->pool;
2826                 sgv_pool_get(dev->pool_clust);
2827         }
2828
2829         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2830                 dev->name);
2831         dev->devtype.type = dev_desc->type;
2832         dev->devtype.threads_num = -1;
2833         dev->devtype.parse_atomic = 1;
2834         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2835         dev->devtype.dev_done_atomic = 1;
2836 #ifdef CONFIG_SCST_PROC
2837         dev->devtype.no_proc = 1;
2838 #else
2839         dev->devtype.dev_attrs = dev_user_dev_attrs;
2840 #endif
2841         dev->devtype.attach = dev_user_attach;
2842         dev->devtype.detach = dev_user_detach;
2843         dev->devtype.attach_tgt = dev_user_attach_tgt;
2844         dev->devtype.detach_tgt = dev_user_detach_tgt;
2845         dev->devtype.exec = dev_user_exec;
2846         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2847         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2848
2849         dev->devtype.parent = &dev_user_devtype;
2850
2851         init_completion(&dev->cleanup_cmpl);
2852         dev->block = block;
2853         dev->def_block = block;
2854
2855         res = __dev_user_set_opt(dev, &dev_desc->opt);
2856         if (res != 0)
2857                 goto out_free;
2858
2859         TRACE_MEM("dev %p, name %s", dev, dev->name);
2860
2861         spin_lock(&dev_list_lock);
2862
2863         list_for_each_entry(d, &dev_list, dev_list_entry) {
2864                 if (strcmp(d->name, dev->name) == 0) {
2865                         PRINT_ERROR("Device %s already exist",
2866                                 dev->name);
2867                         res = -EEXIST;
2868                         spin_unlock(&dev_list_lock);
2869                         goto out_free;
2870                 }
2871         }
2872
2873         list_add_tail(&dev->dev_list_entry, &dev_list);
2874
2875         spin_unlock(&dev_list_lock);
2876
2877         res = scst_register_virtual_dev_driver(&dev->devtype);
2878         if (res < 0)
2879                 goto out_del_free;
2880
2881         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2882         if (dev->virt_id < 0) {
2883                 res = dev->virt_id;
2884                 goto out_unreg_handler;
2885         }
2886
2887         mutex_lock(&dev_priv_mutex);
2888         if (file->private_data != NULL) {
2889                 mutex_unlock(&dev_priv_mutex);
2890                 PRINT_ERROR("%s", "Device already registered");
2891                 res = -EINVAL;
2892                 goto out_unreg_drv;
2893         }
2894         file->private_data = dev;
2895         mutex_unlock(&dev_priv_mutex);
2896
2897 out:
2898         TRACE_EXIT_RES(res);
2899         return res;
2900
2901 out_unreg_drv:
2902         scst_unregister_virtual_device(dev->virt_id);
2903
2904 out_unreg_handler:
2905         scst_unregister_virtual_dev_driver(&dev->devtype);
2906
2907 out_del_free:
2908         spin_lock(&dev_list_lock);
2909         list_del(&dev->dev_list_entry);
2910         spin_unlock(&dev_list_lock);
2911
2912 out_free:
2913         sgv_pool_del(dev->pool_clust);
2914
2915 out_free0:
2916         sgv_pool_del(dev->pool);
2917
2918 out_free_dev:
2919         kfree(dev);
2920
2921 out_put:
2922         module_put(THIS_MODULE);
2923         goto out;
2924 }
2925
2926 static int dev_user_unregister_dev(struct file *file)
2927 {
2928         int res;
2929         struct scst_user_dev *dev;
2930
2931         TRACE_ENTRY();
2932
2933         mutex_lock(&dev_priv_mutex);
2934         dev = (struct scst_user_dev *)file->private_data;
2935         res = dev_user_check_reg(dev);
2936         if (res != 0) {
2937                 mutex_unlock(&dev_priv_mutex);
2938                 goto out;
2939         }
2940         down_read(&dev->dev_rwsem);
2941         mutex_unlock(&dev_priv_mutex);
2942
2943         res = scst_suspend_activity(true);
2944         if (res != 0)
2945                 goto out_up;
2946
2947         up_read(&dev->dev_rwsem);
2948
2949         mutex_lock(&dev_priv_mutex);
2950         dev = (struct scst_user_dev *)file->private_data;
2951         if (dev == NULL) {
2952                 mutex_unlock(&dev_priv_mutex);
2953                 goto out_resume;
2954         }
2955
2956         dev->blocking = 0;
2957         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
2958
2959         down_write(&dev->dev_rwsem);
2960         file->private_data = NULL;
2961         mutex_unlock(&dev_priv_mutex);
2962
2963         dev_user_exit_dev(dev);
2964
2965         up_write(&dev->dev_rwsem); /* to make lockdep happy */
2966
2967         kfree(dev);
2968
2969 out_resume:
2970         scst_resume_activity();
2971
2972 out:
2973         TRACE_EXIT_RES(res);
2974         return res;
2975
2976 out_up:
2977         up_read(&dev->dev_rwsem);
2978         goto out;
2979 }
2980
2981 static int dev_user_flush_cache(struct file *file)
2982 {
2983         int res;
2984         struct scst_user_dev *dev;
2985
2986         TRACE_ENTRY();
2987
2988         mutex_lock(&dev_priv_mutex);
2989         dev = (struct scst_user_dev *)file->private_data;
2990         res = dev_user_check_reg(dev);
2991         if (res != 0) {
2992                 mutex_unlock(&dev_priv_mutex);
2993                 goto out;
2994         }
2995         down_read(&dev->dev_rwsem);
2996         mutex_unlock(&dev_priv_mutex);
2997
2998         res = scst_suspend_activity(true);
2999         if (res != 0)
3000                 goto out_up;
3001
3002         sgv_pool_flush(dev->pool);
3003         sgv_pool_flush(dev->pool_clust);
3004
3005         scst_resume_activity();
3006
3007 out_up:
3008         up_read(&dev->dev_rwsem);
3009
3010 out:
3011         TRACE_EXIT_RES(res);
3012         return res;
3013 }
3014
3015 static int dev_user_capacity_changed(struct file *file)
3016 {
3017         int res;
3018         struct scst_user_dev *dev;
3019
3020         TRACE_ENTRY();
3021
3022         mutex_lock(&dev_priv_mutex);
3023         dev = (struct scst_user_dev *)file->private_data;
3024         res = dev_user_check_reg(dev);
3025         if (res != 0) {
3026                 mutex_unlock(&dev_priv_mutex);
3027                 goto out;
3028         }
3029         down_read(&dev->dev_rwsem);
3030         mutex_unlock(&dev_priv_mutex);
3031
3032         scst_capacity_data_changed(dev->sdev);
3033
3034         up_read(&dev->dev_rwsem);
3035
3036 out:
3037         TRACE_EXIT_RES(res);
3038         return res;
3039 }
3040
3041 static int dev_user_prealloc_buffer(struct file *file, void __user *arg)
3042 {
3043         int res = 0;
3044         struct scst_user_dev *dev;
3045         union scst_user_prealloc_buffer pre;
3046         aligned_u64 pbuf;
3047         uint32_t bufflen;
3048         struct scst_user_cmd *ucmd;
3049         int pages, sg_cnt;
3050         struct sgv_pool *pool;
3051         struct scatterlist *sg;
3052
3053         TRACE_ENTRY();
3054
3055         mutex_lock(&dev_priv_mutex);
3056         dev = (struct scst_user_dev *)file->private_data;
3057         res = dev_user_check_reg(dev);
3058         if (unlikely(res != 0)) {
3059                 mutex_unlock(&dev_priv_mutex);
3060                 goto out;
3061         }
3062         down_read(&dev->dev_rwsem);
3063         mutex_unlock(&dev_priv_mutex);
3064
3065         res = copy_from_user(&pre.in, arg, sizeof(pre.in));
3066         if (unlikely(res < 0))
3067                 goto out_up;
3068
3069         TRACE_MEM("Prealloc buffer with size %dKB for dev %s",
3070                 pre.in.bufflen / 1024, dev->name);
3071         TRACE_BUFFER("Input param", &pre.in, sizeof(pre.in));
3072
3073         pbuf = pre.in.pbuf;
3074         bufflen = pre.in.bufflen;
3075
3076         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
3077         if (ucmd == NULL) {
3078                 res = -ENOMEM;
3079                 goto out_up;
3080         }
3081
3082         ucmd->buff_cached = 1;
3083
3084         TRACE_MEM("ucmd %p, pbuf %llx", ucmd, pbuf);
3085
3086         if (unlikely((pbuf & ~PAGE_MASK) != 0)) {
3087                 PRINT_ERROR("Supplied pbuf %llx isn't page aligned", pbuf);
3088                 res = -EINVAL;
3089                 goto out_put;
3090         }
3091
3092         pages = calc_num_pg(pbuf, bufflen);
3093         res = dev_user_map_buf(ucmd, pbuf, pages);
3094         if (res != 0)
3095                 goto out_put;
3096
3097         if (pre.in.for_clust_pool)
3098                 pool = dev->pool_clust;
3099         else
3100                 pool = dev->pool;
3101
3102         sg = sgv_pool_alloc(pool, bufflen, GFP_KERNEL, SGV_POOL_ALLOC_GET_NEW,
3103                          &sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
3104         if (sg != NULL) {
3105                 struct scst_user_cmd *buf_ucmd =
3106                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
3107
3108                 TRACE_MEM("Buf ucmd %p (sg_cnt %d, last seg len %d, "
3109                         "bufflen %d)", buf_ucmd, sg_cnt,
3110                         sg[sg_cnt-1].length, bufflen);
3111
3112                 EXTRACHECKS_BUG_ON(ucmd != buf_ucmd);
3113
3114                 ucmd->buf_ucmd = buf_ucmd;
3115         } else {
3116                 res = -ENOMEM;
3117                 goto out_put;
3118         }
3119
3120         dev_user_free_sgv(ucmd);
3121
3122         pre.out.cmd_h = ucmd->h;
3123         res = copy_to_user(arg, &pre.out, sizeof(pre.out));
3124
3125 out_put:
3126         ucmd_put(ucmd);
3127
3128 out_up:
3129         up_read(&dev->dev_rwsem);
3130
3131 out:
3132         TRACE_EXIT_RES(res);
3133         return res;
3134 }
3135
3136 static int __dev_user_set_opt(struct scst_user_dev *dev,
3137         const struct scst_user_opt *opt)
3138 {
3139         int res = 0;
3140
3141         TRACE_ENTRY();
3142
3143         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3144                 "memory_reuse_type %x, partial_transfers_type %x, "
3145                 "partial_len %d", dev->name, opt->parse_type,
3146                 opt->on_free_cmd_type, opt->memory_reuse_type,
3147                 opt->partial_transfers_type, opt->partial_len);
3148
3149         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
3150             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
3151             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
3152             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
3153                 PRINT_ERROR("%s", "Invalid option");
3154                 res = -EINVAL;
3155                 goto out;
3156         }
3157
3158         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
3159              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
3160             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
3161              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
3162             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) ||
3163             (opt->d_sense > 1)) {
3164                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
3165                         " tas %x, d_sense %d, has_own_order_mgmt %x)", opt->tst,
3166                         opt->queue_alg, opt->swp, opt->tas, opt->d_sense,
3167                         opt->has_own_order_mgmt);
3168                 res = -EINVAL;
3169                 goto out;
3170         }
3171
3172         dev->parse_type = opt->parse_type;
3173         dev->on_free_cmd_type = opt->on_free_cmd_type;
3174         dev->memory_reuse_type = opt->memory_reuse_type;
3175         dev->partial_transfers_type = opt->partial_transfers_type;
3176         dev->partial_len = opt->partial_len;
3177
3178         dev->tst = opt->tst;
3179         dev->queue_alg = opt->queue_alg;
3180         dev->swp = opt->swp;
3181         dev->tas = opt->tas;
3182         dev->tst = opt->tst;
3183         dev->d_sense = opt->d_sense;
3184         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
3185         if (dev->sdev != NULL) {
3186                 dev->sdev->tst = opt->tst;
3187                 dev->sdev->queue_alg = opt->queue_alg;
3188                 dev->sdev->swp = opt->swp;
3189                 dev->sdev->tas = opt->tas;
3190                 dev->sdev->d_sense = opt->d_sense;
3191                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
3192         }
3193
3194         dev_user_setup_functions(dev);
3195
3196 out:
3197         TRACE_EXIT_RES(res);
3198         return res;
3199 }
3200
3201 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
3202 {
3203         int res;
3204         struct scst_user_dev *dev;
3205
3206         TRACE_ENTRY();
3207
3208         mutex_lock(&dev_priv_mutex);
3209         dev = (struct scst_user_dev *)file->private_data;
3210         res = dev_user_check_reg(dev);
3211         if (res != 0) {
3212                 mutex_unlock(&dev_priv_mutex);
3213                 goto out;
3214         }
3215         down_read(&dev->dev_rwsem);
3216         mutex_unlock(&dev_priv_mutex);
3217
3218         res = scst_suspend_activity(true);
3219         if (res != 0)
3220                 goto out_up;
3221
3222         res = __dev_user_set_opt(dev, opt);
3223
3224         scst_resume_activity();
3225
3226 out_up:
3227         up_read(&dev->dev_rwsem);
3228
3229 out:
3230         TRACE_EXIT_RES(res);
3231         return res;
3232 }
3233
3234 static int dev_user_get_opt(struct file *file, void __user *arg)
3235 {
3236         int res;
3237         struct scst_user_dev *dev;
3238         struct scst_user_opt opt;
3239
3240         TRACE_ENTRY();
3241
3242         mutex_lock(&dev_priv_mutex);
3243         dev = (struct scst_user_dev *)file->private_data;
3244         res = dev_user_check_reg(dev);
3245         if (res != 0) {
3246                 mutex_unlock(&dev_priv_mutex);
3247                 goto out;
3248         }
3249         down_read(&dev->dev_rwsem);
3250         mutex_unlock(&dev_priv_mutex);
3251
3252         opt.parse_type = dev->parse_type;
3253         opt.on_free_cmd_type = dev->on_free_cmd_type;
3254         opt.memory_reuse_type = dev->memory_reuse_type;
3255         opt.partial_transfers_type = dev->partial_transfers_type;
3256         opt.partial_len = dev->partial_len;
3257         opt.tst = dev->tst;
3258         opt.queue_alg = dev->queue_alg;
3259         opt.tas = dev->tas;
3260         opt.swp = dev->swp;
3261         opt.d_sense = dev->d_sense;
3262         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
3263
3264         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3265                 "memory_reuse_type %x, partial_transfers_type %x, "
3266                 "partial_len %d", dev->name, opt.parse_type,
3267                 opt.on_free_cmd_type, opt.memory_reuse_type,
3268                 opt.partial_transfers_type, opt.partial_len);
3269
3270         res = copy_to_user(arg, &opt, sizeof(opt));
3271
3272         up_read(&dev->dev_rwsem);
3273 out:
3274         TRACE_EXIT_RES(res);
3275         return res;
3276 }
3277
3278 static int dev_usr_parse(struct scst_cmd *cmd)
3279 {
3280         sBUG();
3281         return SCST_CMD_STATE_DEFAULT;
3282 }
3283
3284 static int dev_user_exit_dev(struct scst_user_dev *dev)
3285 {
3286         TRACE_ENTRY();
3287
3288         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
3289
3290         spin_lock(&dev_list_lock);
3291         list_del(&dev->dev_list_entry);
3292         spin_unlock(&dev_list_lock);
3293
3294         dev->blocking = 0;
3295         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
3296
3297         spin_lock(&cleanup_lock);
3298         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
3299         spin_unlock(&cleanup_lock);
3300
3301         wake_up(&cleanup_list_waitQ);
3302
3303         scst_unregister_virtual_device(dev->virt_id);
3304         scst_unregister_virtual_dev_driver(&dev->devtype);
3305
3306         sgv_pool_flush(dev->pool_clust);
3307         sgv_pool_flush(dev->pool);
3308
3309         TRACE_MGMT_DBG("Unregistering finished (dev %p)", dev);
3310
3311         dev->cleanup_done = 1;
3312
3313         wake_up(&cleanup_list_waitQ);
3314         wake_up(&dev->cmd_lists.cmd_list_waitQ);
3315
3316         wait_for_completion(&dev->cleanup_cmpl);
3317
3318         sgv_pool_del(dev->pool_clust);
3319         sgv_pool_del(dev->pool);
3320
3321         TRACE_MGMT_DBG("Releasing completed (dev %p)", dev);
3322
3323         module_put(THIS_MODULE);
3324
3325         TRACE_EXIT();
3326         return 0;
3327 }
3328
3329 static int __dev_user_release(void *arg)
3330 {
3331         struct scst_user_dev *dev = (struct scst_user_dev *)arg;
3332         dev_user_exit_dev(dev);
3333         kfree(dev);
3334         return 0;
3335 }
3336
3337 static int dev_user_release(struct inode *inode, struct file *file)
3338 {
3339         struct scst_user_dev *dev;
3340         struct task_struct *t;
3341
3342         TRACE_ENTRY();
3343
3344         dev = (struct scst_user_dev *)file->private_data;
3345         if (dev == NULL)
3346                 goto out;
3347         file->private_data = NULL;
3348
3349         TRACE_MGMT_DBG("Going to release dev %s", dev->name);
3350
3351         t = kthread_run(__dev_user_release, dev, "scst_usr_released");
3352         if (IS_ERR(t)) {
3353                 PRINT_CRIT_ERROR("kthread_run() failed (%ld), releasing device "
3354                         "%p directly. If you have several devices under load "
3355                         "it might deadlock!", PTR_ERR(t), dev);
3356                 __dev_user_release(dev);
3357         }
3358
3359 out:
3360         TRACE_EXIT();
3361         return 0;
3362 }
3363
3364 static int dev_user_process_cleanup(struct scst_user_dev *dev)
3365 {
3366         struct scst_user_cmd *ucmd;
3367         int rc = 0, res = 1;
3368
3369         TRACE_ENTRY();
3370
3371         sBUG_ON(dev->blocking);
3372         wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
3373
3374         while (1) {
3375                 int rc1;
3376
3377                 TRACE_DBG("Cleanuping dev %p", dev);
3378
3379                 rc1 = dev_user_unjam_dev(dev);
3380                 if ((rc1 == 0) && (rc == -EAGAIN) && dev->cleanup_done)
3381                         break;
3382
3383                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
3384
3385                 rc = dev_user_get_next_cmd(dev, &ucmd);
3386                 if (rc == 0)
3387                         dev_user_unjam_cmd(ucmd, 1, NULL);
3388
3389                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
3390
3391                 if (rc == -EAGAIN) {
3392                         if (!dev->cleanup_done) {
3393                                 TRACE_DBG("No more commands (dev %p)", dev);
3394                                 goto out;
3395                         }
3396                 }
3397         }
3398
3399 #ifdef CONFIG_SCST_EXTRACHECKS
3400 {
3401         int i;
3402         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3403                 struct list_head *head = &dev->ucmd_hash[i];
3404                 struct scst_user_cmd *ucmd2;
3405 again:
3406                 list_for_each_entry(ucmd2, head, hash_list_entry) {
3407                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
3408                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
3409                         ucmd_put(ucmd2);
3410                         goto again;
3411                 }
3412         }
3413 }
3414 #endif
3415
3416         TRACE_DBG("Cleanuping done (dev %p)", dev);
3417         complete_all(&dev->cleanup_cmpl);
3418         res = 0;
3419
3420 out:
3421         TRACE_EXIT_RES(res);
3422         return res;
3423 }
3424
3425 #ifndef CONFIG_SCST_PROC
3426
3427 static ssize_t dev_user_sysfs_commands_show(struct kobject *kobj,
3428         struct kobj_attribute *attr, char *buf)
3429 {
3430         int pos = 0, ppos, i;
3431         struct scst_device *dev;
3432         struct scst_user_dev *udev;
3433         unsigned long flags;
3434
3435         TRACE_ENTRY();
3436
3437         dev = container_of(kobj, struct scst_device, dev_kobj);
3438         udev = (struct scst_user_dev *)dev->dh_priv;
3439
3440         spin_lock_irqsave(&udev->cmd_lists.cmd_list_lock, flags);
3441         for (i = 0; i < (int)ARRAY_SIZE(udev->ucmd_hash); i++) {
3442                 struct list_head *head = &udev->ucmd_hash[i];
3443                 struct scst_user_cmd *ucmd;
3444                 list_for_each_entry(ucmd, head, hash_list_entry) {
3445                         ppos = pos;
3446                         pos += scnprintf(&buf[pos],
3447                                 SCST_SYSFS_BLOCK_SIZE - pos,
3448                                 "ucmd %p (state %x, ref %d), "
3449                                 "sent_to_user %d, seen_by_user %d, "
3450                                 "aborted %d, jammed %d, scst_cmd %p\n",
3451                                 ucmd, ucmd->state,
3452                                 atomic_read(&ucmd->ucmd_ref),
3453                                 ucmd->sent_to_user, ucmd->seen_by_user,
3454                                 ucmd->aborted, ucmd->jammed, ucmd->cmd);
3455                         if (pos >= SCST_SYSFS_BLOCK_SIZE-1) {
3456                                 ppos += scnprintf(&buf[ppos],
3457                                         SCST_SYSFS_BLOCK_SIZE - ppos, "...\n");
3458                                 pos = ppos;
3459                                 break;
3460                         }
3461                 }
3462         }
3463         spin_unlock_irqrestore(&udev->cmd_lists.cmd_list_lock, flags);
3464
3465         TRACE_EXIT_RES(pos);
3466         return pos;
3467 }
3468
3469 #else /* CONFIG_SCST_PROC */
3470
3471 /*
3472  * Called when a file in the /proc/scsi_tgt/scst_user is read
3473  */
3474 static int dev_user_read_proc(struct seq_file *seq, struct scst_dev_type *dev_type)
3475 {
3476         int res = 0;
3477         struct scst_user_dev *dev;
3478         unsigned long flags;
3479
3480         TRACE_ENTRY();
3481
3482         spin_lock(&dev_list_lock);
3483
3484         list_for_each_entry(dev, &dev_list, dev_list_entry) {
3485                 int i;
3486                 seq_printf(seq, "Device %s commands:\n", dev->name);
3487                 spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
3488                 for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3489                         struct list_head *head = &dev->ucmd_hash[i];
3490                         struct scst_user_cmd *ucmd;
3491                         list_for_each_entry(ucmd, head, hash_list_entry) {
3492                                 seq_printf(seq, "ucmd %p (state %x, ref %d), "
3493                                         "sent_to_user %d, seen_by_user %d, "
3494                                         "aborted %d, jammed %d, scst_cmd %p\n",
3495                                         ucmd, ucmd->state,
3496                                         atomic_read(&ucmd->ucmd_ref),
3497                                         ucmd->sent_to_user, ucmd->seen_by_user,
3498                                         ucmd->aborted, ucmd->jammed, ucmd->cmd);
3499                         }
3500                 }
3501                 spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
3502         }
3503         spin_unlock(&dev_list_lock);
3504
3505         TRACE_EXIT_RES(res);
3506         return res;
3507 }
3508 #endif /* CONFIG_SCST_PROC */
3509
3510 static inline int test_cleanup_list(void)
3511 {
3512         int res = !list_empty(&cleanup_list) ||
3513                   unlikely(kthread_should_stop());
3514         return res;
3515 }
3516
3517 static int dev_user_cleanup_thread(void *arg)
3518 {
3519         TRACE_ENTRY();
3520
3521         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
3522
3523         current->flags |= PF_NOFREEZE;
3524
3525         spin_lock(&cleanup_lock);
3526         while (!kthread_should_stop()) {
3527                 wait_queue_t wait;
3528                 init_waitqueue_entry(&wait, current);
3529
3530                 if (!test_cleanup_list()) {
3531                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
3532                         for (;;) {
3533                                 set_current_state(TASK_INTERRUPTIBLE);
3534                                 if (test_cleanup_list())
3535                                         break;
3536                                 spin_unlock(&cleanup_lock);
3537                                 schedule();
3538                                 spin_lock(&cleanup_lock);
3539                         }
3540                         set_current_state(TASK_RUNNING);
3541                         remove_wait_queue(&cleanup_list_waitQ, &wait);
3542                 }
3543
3544                 /*
3545                  * We have to poll devices, because commands can go from SCST
3546                  * core on cmd_list_waitQ and we have no practical way to
3547                  * detect them.
3548                  */
3549
3550                 while (1) {
3551                         struct scst_user_dev *dev;
3552                         LIST_HEAD(cl_devs);
3553
3554                         while (!list_empty(&cleanup_list)) {
3555                                 int rc;
3556
3557                                 dev = list_entry(cleanup_list.next,
3558                                         typeof(*dev), cleanup_list_entry);
3559                                 list_del(&dev->cleanup_list_entry);
3560
3561                                 spin_unlock(&cleanup_lock);
3562                                 rc = dev_user_process_cleanup(dev);
3563                                 spin_lock(&cleanup_lock);
3564
3565                                 if (rc != 0)
3566                                         list_add_tail(&dev->cleanup_list_entry,
3567                                                 &cl_devs);
3568                         }
3569
3570                         if (list_empty(&cl_devs))
3571                                 break;
3572
3573                         spin_unlock(&cleanup_lock);
3574                         msleep(100);
3575                         spin_lock(&cleanup_lock);
3576
3577                         while (!list_empty(&cl_devs)) {
3578                                 dev = list_entry(cl_devs.next, typeof(*dev),
3579                                         cleanup_list_entry);
3580                                 list_move_tail(&dev->cleanup_list_entry,
3581                                         &cleanup_list);
3582                         }
3583                 }
3584         }
3585         spin_unlock(&cleanup_lock);
3586
3587         /*
3588          * If kthread_should_stop() is true, we are guaranteed to be
3589          * on the module unload, so cleanup_list must be empty.
3590          */
3591         sBUG_ON(!list_empty(&cleanup_list));
3592
3593         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
3594
3595         TRACE_EXIT();
3596         return 0;
3597 }
3598
3599 static int __init init_scst_user(void)
3600 {
3601         int res = 0;
3602         struct max_get_reply {
3603                 union {
3604                         struct scst_user_get_cmd g;
3605                         struct scst_user_reply_cmd r;
3606                 };
3607         };
3608 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3609         struct class_device *class_member;
3610 #else
3611         struct device *dev;
3612 #endif
3613
3614         TRACE_ENTRY();
3615
3616 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
3617         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
3618                 "Consider changing VMSPLIT option or use a 64-bit "
3619                 "configuration instead. See README file for details.");
3620         res = -EINVAL;
3621         goto out;
3622 #endif
3623
3624         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
3625         if (user_cmd_cachep == NULL) {