Fix copy_to_user()/copy_from_user() wrong reeturn value processing
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2009 ID7 Ltd.
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23 #include <linux/stddef.h>
24
25 #define LOG_PREFIX              DEV_USER_NAME
26
27 #include "scst.h"
28 #include "scst_user.h"
29 #include "scst_dev_handler.h"
30
31 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
32 #warning "HIGHMEM kernel configurations are not supported by this module,\
33  because nowadays it doesn't worth the effort. Consider changing\
34  VMSPLIT option or use a 64-bit configuration instead. See README file\
35  for details."
36 #endif
37
38 #define DEV_USER_CMD_HASH_ORDER         6
39 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
40
41 struct scst_user_dev {
42         struct rw_semaphore dev_rwsem;
43
44         struct scst_cmd_lists cmd_lists;
45
46         /* Protected by cmd_lists.cmd_list_lock */
47         struct list_head ready_cmd_list;
48
49         /* Protected by dev_rwsem or don't need any protection */
50         unsigned int blocking:1;
51         unsigned int cleanup_done:1;
52         unsigned int tst:3;
53         unsigned int queue_alg:4;
54         unsigned int tas:1;
55         unsigned int swp:1;
56         unsigned int d_sense:1;
57         unsigned int has_own_order_mgmt:1;
58
59         int (*generic_parse)(struct scst_cmd *cmd,
60                 int (*get_block)(struct scst_cmd *cmd));
61
62         int block;
63         int def_block;
64
65         struct scst_mem_lim udev_mem_lim;
66         struct sgv_pool *pool;
67         struct sgv_pool *pool_clust;
68
69         uint8_t parse_type;
70         uint8_t on_free_cmd_type;
71         uint8_t memory_reuse_type;
72         uint8_t partial_transfers_type;
73         uint32_t partial_len;
74
75         struct scst_dev_type devtype;
76
77         /* Both protected by cmd_lists.cmd_list_lock */
78         unsigned int handle_counter;
79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
80
81         struct scst_device *sdev;
82
83         int virt_id;
84         struct list_head dev_list_entry;
85         char name[SCST_MAX_NAME];
86
87         struct list_head cleanup_list_entry;
88         struct completion cleanup_cmpl;
89 };
90
91 /* Most fields are unprotected, since only one thread at time can access them */
92 struct scst_user_cmd {
93         struct scst_cmd *cmd;
94         struct scst_user_dev *dev;
95
96         atomic_t ucmd_ref;
97
98         unsigned int buff_cached:1;
99         unsigned int buf_dirty:1;
100         unsigned int background_exec:1;
101         unsigned int aborted:1;
102
103         struct scst_user_cmd *buf_ucmd;
104
105         int cur_data_page;
106         int num_data_pages;
107         int first_page_offset;
108         unsigned long ubuff;
109         struct page **data_pages;
110         struct sgv_pool_obj *sgv;
111
112         /*
113          * Special flags, which can be accessed asynchronously (hence "long").
114          * Protected by cmd_lists.cmd_list_lock.
115          */
116         unsigned long sent_to_user:1;
117         unsigned long jammed:1;
118         unsigned long this_state_unjammed:1;
119         unsigned long seen_by_user:1; /* here only as a small optimization */
120
121         unsigned int state;
122
123         struct list_head ready_cmd_list_entry;
124
125         unsigned int h;
126         struct list_head hash_list_entry;
127
128         int user_cmd_payload_len;
129         struct scst_user_get_cmd user_cmd;
130
131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
132         union {
133                 struct completion *cmpl;
134                 struct scst_mgmt_cmd *mcmd;
135         };
136         int result;
137 };
138
139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
140         gfp_t gfp_mask);
141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
142
143 static int dev_user_parse(struct scst_cmd *cmd);
144 static int dev_user_exec(struct scst_cmd *cmd);
145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
147         struct scst_tgt_dev *tgt_dev);
148
149 static int dev_user_disk_done(struct scst_cmd *cmd);
150 static int dev_user_tape_done(struct scst_cmd *cmd);
151
152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
153         gfp_t gfp_mask, void *priv);
154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
155                                      void *priv);
156
157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
158
159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
160         unsigned long *flags);
161
162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
164         int status);
165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
166 static int dev_user_register_dev(struct file *file,
167         const struct scst_user_dev_desc *dev_desc);
168 static int dev_user_unregister_dev(struct file *file);
169 static int dev_user_flush_cache(struct file *file);
170 static int dev_user_capacity_changed(struct file *file);
171 static int dev_user_prealloc_buffer(struct file *file, void __user *arg);
172 static int __dev_user_set_opt(struct scst_user_dev *dev,
173         const struct scst_user_opt *opt);
174 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
175 static int dev_user_get_opt(struct file *file, void __user *arg);
176
177 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
178 static long dev_user_ioctl(struct file *file, unsigned int cmd,
179         unsigned long arg);
180 static int dev_user_release(struct inode *inode, struct file *file);
181 static int dev_user_exit_dev(struct scst_user_dev *dev);
182
183 #ifdef CONFIG_SCST_PROC
184
185 static int dev_user_read_proc(struct seq_file *seq,
186         struct scst_dev_type *dev_type);
187
188 #else /* CONFIG_SCST_PROC */
189
190 static ssize_t dev_user_sysfs_commands_show(struct kobject *kobj,
191         struct kobj_attribute *attr, char *buf);
192
193 static struct kobj_attribute dev_user_commands_attr =
194         __ATTR(commands, S_IRUGO, dev_user_sysfs_commands_show, NULL);
195
196 static const struct attribute *dev_user_dev_attrs[] = {
197         &dev_user_commands_attr.attr,
198         NULL,
199 };
200
201 #endif /* CONFIG_SCST_PROC */
202
203 static int dev_usr_parse(struct scst_cmd *cmd);
204
205 /** Data **/
206
207 static struct kmem_cache *user_cmd_cachep;
208 static struct kmem_cache *user_get_cmd_cachep;
209
210 static DEFINE_MUTEX(dev_priv_mutex);
211
212 static const struct file_operations dev_user_fops = {
213         .poll           = dev_user_poll,
214         .unlocked_ioctl = dev_user_ioctl,
215 #ifdef CONFIG_COMPAT
216         .compat_ioctl   = dev_user_ioctl,
217 #endif
218         .release        = dev_user_release,
219 };
220
221 static struct scst_dev_type dev_user_devtype = {
222         .name =         DEV_USER_NAME,
223         .type =         -1,
224         .parse =        dev_usr_parse,
225 #ifdef CONFIG_SCST_PROC
226         .read_proc =    dev_user_read_proc,
227 #endif
228 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
229         .default_trace_flags = SCST_DEFAULT_DEV_LOG_FLAGS,
230         .trace_flags = &trace_flag,
231 #endif
232 };
233
234 static int dev_user_major;
235
236 static struct class *dev_user_sysfs_class;
237
238 static DEFINE_SPINLOCK(dev_list_lock);
239 static LIST_HEAD(dev_list);
240
241 static DEFINE_SPINLOCK(cleanup_lock);
242 static LIST_HEAD(cleanup_list);
243 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
244 static struct task_struct *cleanup_thread;
245
246 /*
247  * Skip this command if result is not 0. Must be called under
248  * cmd_lists.cmd_list_lock and IRQ off.
249  */
250 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
251 {
252         int r = atomic_inc_return(&ucmd->ucmd_ref);
253         int res;
254         if (unlikely(r == 1)) {
255                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
256                 atomic_dec(&ucmd->ucmd_ref);
257                 res = true;
258                 /*
259                  * Necessary code is serialized by cmd_list_lock in
260                  * cmd_remove_hash()
261                  */
262         } else {
263                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
264                         atomic_read(&ucmd->ucmd_ref));
265                 res = false;
266         }
267         return res;
268 }
269
270 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
271 {
272         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
273         atomic_inc(&ucmd->ucmd_ref);
274         if (barrier)
275                 smp_mb__after_atomic_inc();
276 }
277
278 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
279 {
280         __ucmd_get(ucmd, true);
281 }
282
283 static inline void ucmd_get(struct scst_user_cmd *ucmd)
284 {
285         __ucmd_get(ucmd, false);
286 }
287
288 /* Must not be called under cmd_list_lock!! */
289 static inline void ucmd_put(struct scst_user_cmd *ucmd)
290 {
291         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
292
293         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
294
295         if (atomic_dec_and_test(&ucmd->ucmd_ref))
296                 dev_user_free_ucmd(ucmd);
297 }
298
299 static inline int calc_num_pg(unsigned long buf, int len)
300 {
301         len += buf & ~PAGE_MASK;
302         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
303 }
304
305 static void __dev_user_not_reg(void)
306 {
307         TRACE_MGMT_DBG("%s", "Device not registered");
308         return;
309 }
310
311 static inline int dev_user_check_reg(struct scst_user_dev *dev)
312 {
313         if (dev == NULL) {
314                 __dev_user_not_reg();
315                 return -ENODEV;
316         }
317         return 0;
318 }
319
320 static inline int scst_user_cmd_hashfn(int h)
321 {
322         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
323 }
324
325 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
326         unsigned int h)
327 {
328         struct list_head *head;
329         struct scst_user_cmd *ucmd;
330
331         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
332         list_for_each_entry(ucmd, head, hash_list_entry) {
333                 if (ucmd->h == h) {
334                         TRACE_DBG("Found ucmd %p", ucmd);
335                         return ucmd;
336                 }
337         }
338         return NULL;
339 }
340
341 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
342 {
343         struct list_head *head;
344         struct scst_user_dev *dev = ucmd->dev;
345         struct scst_user_cmd *u;
346         unsigned long flags;
347
348         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
349         do {
350                 ucmd->h = dev->handle_counter++;
351                 u = __ucmd_find_hash(dev, ucmd->h);
352         } while (u != NULL);
353         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
354         list_add_tail(&ucmd->hash_list_entry, head);
355         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
356
357         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
358         return;
359 }
360
361 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
362 {
363         unsigned long flags;
364
365         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
366         list_del(&ucmd->hash_list_entry);
367         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
368
369         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
370         return;
371 }
372
373 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
374 {
375         TRACE_ENTRY();
376
377         TRACE_MEM("Freeing ucmd %p", ucmd);
378
379         cmd_remove_hash(ucmd);
380         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
381
382         kmem_cache_free(user_cmd_cachep, ucmd);
383
384         TRACE_EXIT();
385         return;
386 }
387
388 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
389         gfp_t gfp_mask, void *priv)
390 {
391         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
392         int offset = 0;
393
394         TRACE_ENTRY();
395
396         /* *sg supposed to be zeroed */
397
398         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
399                 ucmd->ubuff, ucmd->cur_data_page);
400
401         if (ucmd->cur_data_page == 0) {
402                 TRACE_MEM("ucmd->first_page_offset %d",
403                         ucmd->first_page_offset);
404                 offset = ucmd->first_page_offset;
405                 ucmd_get(ucmd);
406         }
407
408         if (ucmd->cur_data_page >= ucmd->num_data_pages)
409                 goto out;
410
411         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
412                 PAGE_SIZE - offset, offset);
413         ucmd->cur_data_page++;
414
415         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
416                 sg->offset);
417         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
418
419 out:
420         TRACE_EXIT();
421         return sg_page(sg);
422 }
423
424 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
425 {
426         TRACE_ENTRY();
427
428         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
429                 ucmd, ucmd->h, ucmd->ubuff);
430
431         ucmd->user_cmd_payload_len =
432                 offsetof(struct scst_user_get_cmd, on_cached_mem_free) +
433                 sizeof(ucmd->user_cmd.on_cached_mem_free);
434         ucmd->user_cmd.cmd_h = ucmd->h;
435         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
436         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
437
438         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
439
440         dev_user_add_to_ready(ucmd);
441
442         TRACE_EXIT();
443         return;
444 }
445
446 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
447 {
448         int i;
449
450         TRACE_ENTRY();
451
452         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
453                 ucmd->ubuff, ucmd->num_data_pages);
454
455         for (i = 0; i < ucmd->num_data_pages; i++) {
456                 struct page *page = ucmd->data_pages[i];
457
458                 if (ucmd->buf_dirty)
459                         SetPageDirty(page);
460
461                 page_cache_release(page);
462         }
463
464         kfree(ucmd->data_pages);
465         ucmd->data_pages = NULL;
466
467         TRACE_EXIT();
468         return;
469 }
470
471 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
472 {
473         TRACE_ENTRY();
474
475         sBUG_ON(ucmd->data_pages == NULL);
476
477         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
478                 ucmd, ucmd->ubuff, ucmd->buff_cached);
479
480         dev_user_unmap_buf(ucmd);
481
482         if (ucmd->buff_cached)
483                 dev_user_on_cached_mem_free(ucmd);
484         else
485                 ucmd_put(ucmd);
486
487         TRACE_EXIT();
488         return;
489 }
490
491 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
492         void *priv)
493 {
494         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
495
496         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
497                 sg_count, ucmd);
498
499         __dev_user_free_sg_entries(ucmd);
500
501         return;
502 }
503
504 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
505 {
506         int mem_reuse_type = ucmd->dev->memory_reuse_type;
507
508         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
509             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
510              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
511             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
512              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
513                 return 1;
514         else
515                 return 0;
516 }
517
518 static inline int is_need_offs_page(unsigned long buf, int len)
519 {
520         return ((buf & ~PAGE_MASK) != 0) &&
521                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
522 }
523
524 /*
525  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
526  * Unmaps the buffer, if needed in case of error
527  */
528 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
529 {
530         int res = 0;
531         struct scst_cmd *cmd = ucmd->cmd;
532         struct scst_user_dev *dev = ucmd->dev;
533         struct sgv_pool *pool;
534         gfp_t gfp_mask;
535         int flags = 0;
536         int bufflen, orig_bufflen;
537         int last_len = 0;
538         int out_sg_pages = 0;
539
540         TRACE_ENTRY();
541
542         gfp_mask = __GFP_NOWARN;
543         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
544
545         if (cmd->data_direction != SCST_DATA_BIDI) {
546                 orig_bufflen = cmd->bufflen;
547                 pool = (struct sgv_pool *)cmd->tgt_dev->dh_priv;
548         } else {
549                 /* Make in_sg->offset 0 */
550                 int len = cmd->bufflen + ucmd->first_page_offset;
551                 out_sg_pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
552                 orig_bufflen = (out_sg_pages << PAGE_SHIFT) + cmd->in_bufflen;
553                 pool = dev->pool;
554         }
555         bufflen = orig_bufflen;
556
557         EXTRACHECKS_BUG_ON(bufflen == 0);
558
559         if (cached_buff) {
560                 flags |= SGV_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
561                 if (ucmd->ubuff == 0)
562                         flags |= SGV_POOL_NO_ALLOC_ON_CACHE_MISS;
563         } else {
564                 TRACE_MEM("%s", "Not cached buff");
565                 flags |= SGV_POOL_ALLOC_NO_CACHED;
566                 if (ucmd->ubuff == 0) {
567                         res = 1;
568                         goto out;
569                 }
570                 bufflen += ucmd->first_page_offset;
571                 if (is_need_offs_page(ucmd->ubuff, orig_bufflen))
572                         last_len = bufflen & ~PAGE_MASK;
573                 else
574                         last_len = orig_bufflen & ~PAGE_MASK;
575                 if (last_len == 0)
576                         last_len = PAGE_SIZE;
577         }
578         ucmd->buff_cached = cached_buff;
579
580         cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt,
581                         &ucmd->sgv, &dev->udev_mem_lim, ucmd);
582         if (cmd->sg != NULL) {
583                 struct scst_user_cmd *buf_ucmd =
584                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
585
586                 TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
587                         "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
588                         cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
589
590                 ucmd->ubuff = buf_ucmd->ubuff;
591                 ucmd->buf_ucmd = buf_ucmd;
592
593                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
594                                    (ucmd != buf_ucmd));
595
596                 if (last_len != 0) {
597                         cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
598                         cmd->sg[cmd->sg_cnt-1].length += last_len;
599                 }
600
601                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
602                         "last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
603                         cmd->sg[cmd->sg_cnt-1].length);
604
605                 if (cmd->data_direction == SCST_DATA_BIDI) {
606                         cmd->in_sg = &cmd->sg[out_sg_pages];
607                         cmd->in_sg_cnt = cmd->sg_cnt - out_sg_pages;
608                         cmd->sg_cnt = out_sg_pages;
609                         TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d",
610                                 cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt);
611                 }
612
613                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
614                         static int ll;
615                         if (ll < 10) {
616                                 PRINT_INFO("Unable to complete command due to "
617                                         "SG IO count limitation (requested %d, "
618                                         "available %d, tgt lim %d)",
619                                         cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt,
620                                         cmd->tgt->sg_tablesize);
621                                 ll++;
622                         }
623                         cmd->sg = NULL;
624                         /* sgv will be freed in dev_user_free_sgv() */
625                         res = -1;
626                 }
627         } else {
628                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
629                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
630                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
631                 if (unlikely(cmd->sg_cnt == 0)) {
632                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
633                         sBUG_ON(ucmd->sgv != NULL);
634                         res = -1;
635                 } else {
636                         switch (ucmd->state) {
637                         case UCMD_STATE_BUF_ALLOCING:
638                                 res = 1;
639                                 break;
640                         case UCMD_STATE_EXECING:
641                                 res = -1;
642                                 break;
643                         default:
644                                 sBUG();
645                                 break;
646                         }
647                 }
648         }
649
650 out:
651         TRACE_EXIT_RES(res);
652         return res;
653 }
654
655 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
656 {
657         int rc, res = SCST_CMD_STATE_DEFAULT;
658         struct scst_cmd *cmd = ucmd->cmd;
659
660         TRACE_ENTRY();
661
662         ucmd->state = UCMD_STATE_BUF_ALLOCING;
663         cmd->dh_data_buf_alloced = 1;
664
665         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
666         if (rc == 0)
667                 goto out;
668         else if (rc < 0) {
669                 scst_set_busy(cmd);
670                 res = scst_get_cmd_abnormal_done_state(cmd);
671                 goto out;
672         }
673
674         if (!(cmd->data_direction & SCST_DATA_WRITE) &&
675             !scst_is_cmd_local(cmd)) {
676                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
677                 goto out;
678         }
679
680         ucmd->user_cmd_payload_len =
681                 offsetof(struct scst_user_get_cmd, alloc_cmd) +
682                 sizeof(ucmd->user_cmd.alloc_cmd);
683         ucmd->user_cmd.cmd_h = ucmd->h;
684         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
685         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
686         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb, cmd->cdb_len);
687         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
688         ucmd->user_cmd.alloc_cmd.ext_cdb_len = cmd->ext_cdb_len;
689         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
690                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
691         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
692         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
693         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
694
695         dev_user_add_to_ready(ucmd);
696
697         res = SCST_CMD_STATE_STOP;
698
699 out:
700         TRACE_EXIT_RES(res);
701         return res;
702 }
703
704 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
705         gfp_t gfp_mask)
706 {
707         struct scst_user_cmd *ucmd = NULL;
708
709         TRACE_ENTRY();
710
711 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
712         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
713         if (ucmd != NULL)
714                 memset(ucmd, 0, sizeof(*ucmd));
715 #else
716         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
717 #endif
718         if (unlikely(ucmd == NULL)) {
719                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
720                         "user cmd (gfp_mask %x)", gfp_mask);
721                 goto out;
722         }
723         ucmd->dev = dev;
724         atomic_set(&ucmd->ucmd_ref, 1);
725
726         cmd_insert_hash(ucmd);
727
728         TRACE_MEM("ucmd %p allocated", ucmd);
729
730 out:
731         TRACE_EXIT_HRES((unsigned long)ucmd);
732         return ucmd;
733 }
734
735 static int dev_user_get_block(struct scst_cmd *cmd)
736 {
737         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
738         /*
739          * No need for locks here, since *_detach() can not be
740          * called, when there are existing commands.
741          */
742         TRACE_EXIT_RES(dev->block);
743         return dev->block;
744 }
745
746 static int dev_user_parse(struct scst_cmd *cmd)
747 {
748         int rc, res = SCST_CMD_STATE_DEFAULT;
749         struct scst_user_cmd *ucmd;
750         int atomic = scst_cmd_atomic(cmd);
751         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
752         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
753
754         TRACE_ENTRY();
755
756         if (cmd->dh_priv == NULL) {
757                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
758                 if (unlikely(ucmd == NULL)) {
759                         if (atomic) {
760                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
761                                 goto out;
762                         } else {
763                                 scst_set_busy(cmd);
764                                 goto out_error;
765                         }
766                 }
767                 ucmd->cmd = cmd;
768                 cmd->dh_priv = ucmd;
769         } else {
770                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
771                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
772         }
773
774         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
775
776         if (ucmd->state != UCMD_STATE_NEW)
777                 goto alloc;
778
779         switch (dev->parse_type) {
780         case SCST_USER_PARSE_STANDARD:
781                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
782                 rc = dev->generic_parse(cmd, dev_user_get_block);
783                 if ((rc != 0) || !(cmd->op_flags & SCST_INFO_VALID))
784                         goto out_invalid;
785                 break;
786
787         case SCST_USER_PARSE_EXCEPTION:
788                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
789                 rc = dev->generic_parse(cmd, dev_user_get_block);
790                 if ((rc == 0) && (cmd->op_flags & SCST_INFO_VALID))
791                         break;
792                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
793                         TRACE_MEM("Restarting PARSE to thread context "
794                                 "(ucmd %p)", ucmd);
795                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
796                         goto out;
797                 }
798                 /* else go through */
799
800         case SCST_USER_PARSE_CALL:
801                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
802                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
803                 ucmd->user_cmd_payload_len =
804                         offsetof(struct scst_user_get_cmd, parse_cmd) +
805                         sizeof(ucmd->user_cmd.parse_cmd);
806                 ucmd->user_cmd.cmd_h = ucmd->h;
807                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
808                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
809                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb, cmd->cdb_len);
810                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
811                 ucmd->user_cmd.parse_cmd.ext_cdb_len = cmd->ext_cdb_len;
812                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
813                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
814                 ucmd->user_cmd.parse_cmd.in_bufflen = cmd->in_bufflen;
815                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
816                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
817                 ucmd->user_cmd.parse_cmd.expected_values_set =
818                                         cmd->expected_values_set;
819                 ucmd->user_cmd.parse_cmd.expected_data_direction =
820                                         cmd->expected_data_direction;
821                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
822                                         cmd->expected_transfer_len;
823                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
824                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
825                 ucmd->user_cmd.parse_cmd.op_flags = cmd->op_flags;
826                 ucmd->state = UCMD_STATE_PARSING;
827                 dev_user_add_to_ready(ucmd);
828                 res = SCST_CMD_STATE_STOP;
829                 goto out;
830
831         default:
832                 sBUG();
833                 goto out;
834         }
835
836 alloc:
837         if (cmd->bufflen == 0) {
838                 /*
839                  * According to SPC bufflen 0 for data transfer commands isn't
840                  * an error, so we need to fix the transfer direction.
841                  */
842                 cmd->data_direction = SCST_DATA_NONE;
843         }
844
845         if (cmd->data_direction != SCST_DATA_NONE)
846                 res = dev_user_alloc_space(ucmd);
847
848 out:
849         TRACE_EXIT_RES(res);
850         return res;
851
852 out_invalid:
853         PRINT_ERROR("PARSE failed (ucmd %p, rc %d)", ucmd, rc);
854         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
855
856 out_error:
857         res = scst_get_cmd_abnormal_done_state(cmd);
858         goto out;
859 }
860
861 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
862 {
863         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
864         unsigned long start = buf_ucmd->ubuff;
865         int i, bufflen = ucmd->cmd->bufflen;
866
867         TRACE_ENTRY();
868
869         if (start == 0)
870                 goto out;
871
872         /*
873          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
874          * faster, since it should be cache hot, while ucmd->buf_ucmd and
875          * buf_ucmd->data_pages are cache cold. But, from other side,
876          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
877          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
878          * data_pages array can lead to less cache misses. So, real numbers are
879          * needed. ToDo.
880          */
881
882         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
883                 struct page *page;
884                 page = buf_ucmd->data_pages[i];
885 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
886                 struct vm_area_struct *vma = find_vma(current->mm, start);
887                 if (vma != NULL)
888                         flush_anon_page(vma, page, start);
889 #endif
890                 flush_dcache_page(page);
891                 start += PAGE_SIZE;
892                 bufflen -= PAGE_SIZE;
893         }
894
895 out:
896         TRACE_EXIT();
897         return;
898 }
899
900 static int dev_user_exec(struct scst_cmd *cmd)
901 {
902         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
903         int res = SCST_EXEC_COMPLETED;
904
905         TRACE_ENTRY();
906
907 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
908        * context to complete the necessary actions, but all we are going to
909        * do in this function is, in fact, atomic, so let's skip this check.
910        */
911         if (scst_cmd_atomic(cmd)) {
912                 TRACE_DBG("%s", "User exec() can not be called in atomic "
913                         "context, rescheduling to the thread");
914                 res = SCST_EXEC_NEED_THREAD;
915                 goto out;
916         }
917 #endif
918
919         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
920                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
921                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
922
923         if (cmd->data_direction & SCST_DATA_WRITE)
924                 dev_user_flush_dcache(ucmd);
925
926         BUILD_BUG_ON(sizeof(ucmd->user_cmd.exec_cmd.cdb) != sizeof(cmd->cdb));
927
928         ucmd->user_cmd_payload_len =
929                 offsetof(struct scst_user_get_cmd, exec_cmd) +
930                 sizeof(ucmd->user_cmd.exec_cmd);
931         ucmd->user_cmd.cmd_h = ucmd->h;
932         ucmd->user_cmd.subcode = SCST_USER_EXEC;
933         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
934         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb, cmd->cdb_len);
935         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
936         ucmd->user_cmd.exec_cmd.ext_cdb_len = cmd->ext_cdb_len;
937         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
938         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
939         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
940         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
941                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
942                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
943         }
944         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
945         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
946         ucmd->user_cmd.exec_cmd.partial = 0;
947         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
948         ucmd->user_cmd.exec_cmd.p_in_buf = ucmd->ubuff +
949                                                 (cmd->sg_cnt << PAGE_SHIFT);
950         ucmd->user_cmd.exec_cmd.in_bufflen = cmd->in_bufflen;
951         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
952
953         ucmd->state = UCMD_STATE_EXECING;
954
955         dev_user_add_to_ready(ucmd);
956
957         TRACE_EXIT_RES(res);
958         return res;
959 }
960
961 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
962 {
963         if (ucmd->sgv != NULL) {
964                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
965                 ucmd->sgv = NULL;
966         } else if (ucmd->data_pages != NULL) {
967                 /* We mapped pages, but for some reason didn't allocate them */
968                 ucmd_get(ucmd);
969                 __dev_user_free_sg_entries(ucmd);
970         }
971         return;
972 }
973
974 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
975 {
976         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
977
978         TRACE_ENTRY();
979
980         if (unlikely(ucmd == NULL))
981                 goto out;
982
983         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
984                 ucmd->buff_cached, ucmd->ubuff);
985
986         ucmd->cmd = NULL;
987         if ((cmd->data_direction & SCST_DATA_WRITE) && ucmd->buf_ucmd != NULL)
988                 ucmd->buf_ucmd->buf_dirty = 1;
989
990         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
991                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
992                 /* The state assignment must be before freeing sgv! */
993                 goto out_reply;
994         }
995
996         if (unlikely(!ucmd->seen_by_user)) {
997                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
998                 goto out_reply;
999         }
1000
1001         ucmd->user_cmd_payload_len =
1002                 offsetof(struct scst_user_get_cmd, on_free_cmd) +
1003                 sizeof(ucmd->user_cmd.on_free_cmd);
1004         ucmd->user_cmd.cmd_h = ucmd->h;
1005         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
1006         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
1007         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
1008         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
1009         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
1010         ucmd->user_cmd.on_free_cmd.status = cmd->status;
1011         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
1012
1013         ucmd->state = UCMD_STATE_ON_FREEING;
1014
1015         dev_user_add_to_ready(ucmd);
1016
1017 out:
1018         TRACE_EXIT();
1019         return;
1020
1021 out_reply:
1022         dev_user_process_reply_on_free(ucmd);
1023         goto out;
1024 }
1025
1026 static void dev_user_set_block(struct scst_cmd *cmd, int block)
1027 {
1028         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
1029         /*
1030          * No need for locks here, since *_detach() can not be
1031          * called, when there are existing commands.
1032          */
1033         TRACE_DBG("dev %p, new block %d", dev, block);
1034         if (block != 0)
1035                 dev->block = block;
1036         else
1037                 dev->block = dev->def_block;
1038         return;
1039 }
1040
1041 static int dev_user_disk_done(struct scst_cmd *cmd)
1042 {
1043         int res = SCST_CMD_STATE_DEFAULT;
1044
1045         TRACE_ENTRY();
1046
1047         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
1048
1049         TRACE_EXIT_RES(res);
1050         return res;
1051 }
1052
1053 static int dev_user_tape_done(struct scst_cmd *cmd)
1054 {
1055         int res = SCST_CMD_STATE_DEFAULT;
1056
1057         TRACE_ENTRY();
1058
1059         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
1060
1061         TRACE_EXIT_RES(res);
1062         return res;
1063 }
1064
1065 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
1066 {
1067         struct scst_user_dev *dev = ucmd->dev;
1068         unsigned long flags;
1069         int do_wake = in_interrupt();
1070
1071         TRACE_ENTRY();
1072
1073         if (ucmd->cmd)
1074                 do_wake |= ucmd->cmd->preprocessing_only;
1075
1076         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1077
1078         ucmd->this_state_unjammed = 0;
1079
1080         if ((ucmd->state == UCMD_STATE_PARSING) ||
1081             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1082                 /*
1083                  * If we don't put such commands in the queue head, then under
1084                  * high load we might delay threads, waiting for memory
1085                  * allocations, for too long and start loosing NOPs, which
1086                  * would lead to consider us by remote initiators as
1087                  * unresponsive and stuck => broken connections, etc. If none
1088                  * of our commands completed in NOP timeout to allow the head
1089                  * commands to go, then we are really overloaded and/or stuck.
1090                  */
1091                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1092                         "cmd list", ucmd, ucmd->state);
1093                 list_add(&ucmd->ready_cmd_list_entry,
1094                         &dev->ready_cmd_list);
1095         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1096                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1097                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1098                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1099                         "ready cmd list", ucmd, ucmd->state);
1100                 list_add(&ucmd->ready_cmd_list_entry,
1101                         &dev->ready_cmd_list);
1102                 do_wake = 1;
1103         } else {
1104                 if ((ucmd->cmd != NULL) &&
1105                     unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1106                         TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list",
1107                                 ucmd);
1108                         list_add(&ucmd->ready_cmd_list_entry,
1109                                 &dev->ready_cmd_list);
1110                 } else {
1111                         TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1112                         list_add_tail(&ucmd->ready_cmd_list_entry,
1113                                       &dev->ready_cmd_list);
1114                 }
1115                 do_wake |= ((ucmd->state == UCMD_STATE_ON_CACHE_FREEING) ||
1116                             (ucmd->state == UCMD_STATE_ON_FREEING));
1117         }
1118
1119         if (do_wake) {
1120                 TRACE_DBG("Waking up dev %p", dev);
1121                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1122         }
1123
1124         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1125
1126         TRACE_EXIT();
1127         return;
1128 }
1129
1130 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1131         int num_pg)
1132 {
1133         int res = 0, rc;
1134         int i;
1135         struct task_struct *tsk = current;
1136
1137         TRACE_ENTRY();
1138
1139         if (unlikely(ubuff == 0))
1140                 goto out_nomem;
1141
1142         sBUG_ON(ucmd->data_pages != NULL);
1143
1144         ucmd->num_data_pages = num_pg;
1145
1146         ucmd->data_pages =
1147                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1148                           GFP_KERNEL);
1149         if (ucmd->data_pages == NULL) {
1150                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1151                         "(num_data_pages=%d)", ucmd->num_data_pages);
1152                 res = -ENOMEM;
1153                 goto out_nomem;
1154         }
1155
1156         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1157                 " first_page_offset %d, len %d)", ucmd, ubuff,
1158                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1159                 (ucmd->cmd != NULL) ? ucmd->cmd->bufflen : -1);
1160
1161         down_read(&tsk->mm->mmap_sem);
1162         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1163                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1164         up_read(&tsk->mm->mmap_sem);
1165
1166         /* get_user_pages() flushes dcache */
1167
1168         if (rc < ucmd->num_data_pages)
1169                 goto out_unmap;
1170
1171         ucmd->ubuff = ubuff;
1172         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1173
1174 out:
1175         TRACE_EXIT_RES(res);
1176         return res;
1177
1178 out_nomem:
1179         if (ucmd->cmd != NULL)
1180                 scst_set_busy(ucmd->cmd);
1181         /* go through */
1182
1183 out_err:
1184         if (ucmd->cmd != NULL)
1185                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1186         goto out;
1187
1188 out_unmap:
1189         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1190                 ucmd->num_data_pages, rc);
1191         if (rc > 0) {
1192                 for (i = 0; i < rc; i++)
1193                         page_cache_release(ucmd->data_pages[i]);
1194         }
1195         kfree(ucmd->data_pages);
1196         ucmd->data_pages = NULL;
1197         res = -EFAULT;
1198         if (ucmd->cmd != NULL)
1199                 scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1200         goto out_err;
1201 }
1202
1203 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1204         struct scst_user_reply_cmd *reply)
1205 {
1206         int res = 0;
1207         struct scst_cmd *cmd = ucmd->cmd;
1208
1209         TRACE_ENTRY();
1210
1211         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1212
1213         if (likely(reply->alloc_reply.pbuf != 0)) {
1214                 int pages;
1215                 if (ucmd->buff_cached) {
1216                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1217                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1218                                         "page aligned",
1219                                         reply->alloc_reply.pbuf);
1220                                 goto out_hwerr;
1221                         }
1222                         pages = cmd->sg_cnt;
1223                 } else
1224                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1225                                             cmd->bufflen);
1226                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1227         } else {
1228                 scst_set_busy(ucmd->cmd);
1229                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1230         }
1231
1232 out_process:
1233         scst_post_parse_process_active_cmd(cmd, false);
1234
1235         TRACE_EXIT_RES(res);
1236         return res;
1237
1238 out_hwerr:
1239         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1240         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1241         res = -EINVAL;
1242         goto out_process;
1243 }
1244
1245 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1246         struct scst_user_reply_cmd *reply)
1247 {
1248         int res = 0, rc;
1249         struct scst_user_scsi_cmd_reply_parse *preply =
1250                 &reply->parse_reply;
1251         struct scst_cmd *cmd = ucmd->cmd;
1252
1253         TRACE_ENTRY();
1254
1255         if (preply->status != 0)
1256                 goto out_status;
1257
1258         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1259                 goto out_inval;
1260
1261         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1262                      (preply->data_direction != SCST_DATA_READ) &&
1263                      (preply->data_direction != SCST_DATA_BIDI) &&
1264                      (preply->data_direction != SCST_DATA_NONE)))
1265                 goto out_inval;
1266
1267         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1268                      (preply->bufflen == 0)))
1269                 goto out_inval;
1270
1271         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1272                 goto out_inval;
1273
1274         if (unlikely(preply->cdb_len > SCST_MAX_CDB_SIZE))
1275                 goto out_inval;
1276
1277         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1278                 "data_len %d, pbuf %llx, cdb_len %d, op_flags %x", ucmd,
1279                 preply->queue_type, preply->data_direction, preply->bufflen,
1280                 preply->data_len, reply->alloc_reply.pbuf, preply->cdb_len,
1281                 preply->op_flags);
1282
1283         cmd->queue_type = preply->queue_type;
1284         cmd->data_direction = preply->data_direction;
1285         cmd->bufflen = preply->bufflen;
1286         cmd->data_len = preply->data_len;
1287         if (preply->cdb_len > 0)
1288                 cmd->cdb_len = preply->cdb_len;
1289         if (preply->op_flags & SCST_INFO_VALID)
1290                 cmd->op_flags = preply->op_flags;
1291
1292 out_process:
1293         scst_post_parse_process_active_cmd(cmd, false);
1294
1295         TRACE_EXIT_RES(res);
1296         return res;
1297
1298 out_inval:
1299         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1300                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1301         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1302         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1303         res = -EINVAL;
1304         goto out_abnormal;
1305
1306 out_hwerr_res_set:
1307         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1308
1309 out_abnormal:
1310         scst_set_cmd_abnormal_done_state(cmd);
1311         goto out_process;
1312
1313 out_status:
1314         TRACE_DBG("ucmd %p returned with error from user status %x",
1315                 ucmd, preply->status);
1316
1317         if (preply->sense_len != 0) {
1318                 int sense_len;
1319
1320                 res = scst_alloc_sense(cmd, 0);
1321                 if (res != 0)
1322                         goto out_hwerr_res_set;
1323
1324                 sense_len = min_t(int, cmd->sense_buflen, preply->sense_len);
1325
1326                 rc = copy_from_user(cmd->sense,
1327                         (void __user *)(unsigned long)preply->psense_buffer,
1328                         sense_len);
1329                 if (rc != 0) {
1330                         PRINT_ERROR("Failed to copy %d sense's bytes", rc);
1331                         res = -EFAULT;
1332                         goto out_hwerr_res_set;
1333                 }
1334                 cmd->sense_valid_len = sense_len;
1335         }
1336         scst_set_cmd_error_status(cmd, preply->status);
1337         goto out_abnormal;
1338 }
1339
1340 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1341 {
1342         int res = 0;
1343
1344         TRACE_ENTRY();
1345
1346         TRACE_DBG("ON FREE ucmd %p", ucmd);
1347
1348         dev_user_free_sgv(ucmd);
1349         ucmd_put(ucmd);
1350
1351         TRACE_EXIT_RES(res);
1352         return res;
1353 }
1354
1355 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1356 {
1357         int res = 0;
1358
1359         TRACE_ENTRY();
1360
1361         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1362
1363         ucmd_put(ucmd);
1364
1365         TRACE_EXIT_RES(res);
1366         return res;
1367 }
1368
1369 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1370         struct scst_user_reply_cmd *reply)
1371 {
1372         int res = 0;
1373         struct scst_user_scsi_cmd_reply_exec *ereply =
1374                 &reply->exec_reply;
1375         struct scst_cmd *cmd = ucmd->cmd;
1376
1377         TRACE_ENTRY();
1378
1379         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1380                 if (ucmd->background_exec) {
1381                         TRACE_DBG("Background ucmd %p finished", ucmd);
1382                         ucmd_put(ucmd);
1383                         goto out;
1384                 }
1385                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1386                         goto out_inval;
1387                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1388                              (ereply->resp_data_len != 0)))
1389                         goto out_inval;
1390         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1391                 if (unlikely(ucmd->background_exec))
1392                         goto out_inval;
1393                 if (unlikely((cmd->data_direction & SCST_DATA_READ) ||
1394                              (cmd->resp_data_len != 0)))
1395                         goto out_inval;
1396                 /*
1397                  * background_exec assignment must be after ucmd get.
1398                  * Otherwise, due to reorder, in dev_user_process_reply()
1399                  * it is possible that ucmd is destroyed before it "got" here.
1400                  */
1401                 ucmd_get_ordered(ucmd);
1402                 ucmd->background_exec = 1;
1403                 TRACE_DBG("Background ucmd %p", ucmd);
1404                 goto out_compl;
1405         } else
1406                 goto out_inval;
1407
1408         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1409                 ereply->status, ereply->resp_data_len);
1410
1411         cmd->atomic = 0;
1412
1413          if (ereply->resp_data_len != 0) {
1414                 if (ucmd->ubuff == 0) {
1415                         int pages, rc;
1416                         if (unlikely(ereply->pbuf == 0))
1417                                 goto out_busy;
1418                         if (ucmd->buff_cached) {
1419                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1420                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1421                                                 "page aligned", ereply->pbuf);
1422                                         goto out_hwerr;
1423                                 }
1424                                 pages = cmd->sg_cnt;
1425                         } else
1426                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1427                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1428                         if ((rc != 0) || (ucmd->ubuff == 0))
1429                                 goto out_compl;
1430
1431                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1432                         if (unlikely(rc != 0))
1433                                 goto out_busy;
1434                 } else
1435                         dev_user_flush_dcache(ucmd);
1436                 cmd->may_need_dma_sync = 1;
1437                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1438         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1439                 if (ucmd->ubuff == 0)
1440                         cmd->resp_data_len = ereply->resp_data_len;
1441                 else
1442                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1443         }
1444
1445         cmd->status = ereply->status;
1446         if (ereply->sense_len != 0) {
1447                 int sense_len, rc;
1448
1449                 res = scst_alloc_sense(cmd, 0);
1450                 if (res != 0)
1451                         goto out_compl;
1452
1453                 sense_len = min((int)cmd->sense_buflen, (int)ereply->sense_len);
1454
1455                 rc = copy_from_user(cmd->sense,
1456                         (void __user *)(unsigned long)ereply->psense_buffer,
1457                         sense_len);
1458                 if (rc != 0) {
1459                         PRINT_ERROR("Failed to copy %d sense's bytes", rc);
1460                         res = -EFAULT;
1461                         goto out_hwerr_res_set;
1462                 }
1463                 cmd->sense_valid_len = sense_len;
1464         }
1465
1466 out_compl:
1467         cmd->completed = 1;
1468         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1469         /* !! At this point cmd can be already freed !! */
1470
1471 out:
1472         TRACE_EXIT_RES(res);
1473         return res;
1474
1475 out_inval:
1476         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1477                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1478         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1479
1480 out_hwerr:
1481         res = -EINVAL;
1482
1483 out_hwerr_res_set:
1484         if (ucmd->background_exec) {
1485                 ucmd_put(ucmd);
1486                 goto out;
1487         } else {
1488                 scst_set_cmd_error(cmd,
1489                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1490                 goto out_compl;
1491         }
1492
1493 out_busy:
1494         scst_set_busy(cmd);
1495         goto out_compl;
1496 }
1497
1498 static int dev_user_process_reply(struct scst_user_dev *dev,
1499         struct scst_user_reply_cmd *reply)
1500 {
1501         int res = 0;
1502         struct scst_user_cmd *ucmd;
1503         int state;
1504
1505         TRACE_ENTRY();
1506
1507         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1508
1509         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1510         if (unlikely(ucmd == NULL)) {
1511                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1512                 res = -ESRCH;
1513                 goto out_unlock;
1514         }
1515
1516         if (unlikely(ucmd_get_check(ucmd))) {
1517                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1518                 res = -ESRCH;
1519                 goto out_unlock;
1520         }
1521
1522         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1523         smp_mb();
1524         if (ucmd->background_exec) {
1525                 state = UCMD_STATE_EXECING;
1526                 goto unlock_process;
1527         }
1528
1529         if (unlikely(ucmd->this_state_unjammed)) {
1530                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1531                         ucmd);
1532                 goto out_unlock_put;
1533         }
1534
1535         if (unlikely(!ucmd->sent_to_user)) {
1536                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1537                         "state %x", ucmd, ucmd->state);
1538                 res = -EINVAL;
1539                 goto out_unlock_put;
1540         }
1541
1542         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1543                 goto out_wrong_state;
1544
1545         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1546                 goto out_wrong_state;
1547
1548         state = ucmd->state;
1549         ucmd->sent_to_user = 0;
1550
1551 unlock_process:
1552         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1553
1554         switch (state) {
1555         case UCMD_STATE_PARSING:
1556                 res = dev_user_process_reply_parse(ucmd, reply);
1557                 break;
1558
1559         case UCMD_STATE_BUF_ALLOCING:
1560                 res = dev_user_process_reply_alloc(ucmd, reply);
1561                 break;
1562
1563         case UCMD_STATE_EXECING:
1564                 res = dev_user_process_reply_exec(ucmd, reply);
1565                 break;
1566
1567         case UCMD_STATE_ON_FREEING:
1568                 res = dev_user_process_reply_on_free(ucmd);
1569                 break;
1570
1571         case UCMD_STATE_ON_CACHE_FREEING:
1572                 res = dev_user_process_reply_on_cache_free(ucmd);
1573                 break;
1574
1575         case UCMD_STATE_TM_EXECING:
1576                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1577                 break;
1578
1579         case UCMD_STATE_ATTACH_SESS:
1580         case UCMD_STATE_DETACH_SESS:
1581                 res = dev_user_process_reply_sess(ucmd, reply->result);
1582                 break;
1583
1584         default:
1585                 sBUG();
1586                 break;
1587         }
1588
1589 out_put:
1590         ucmd_put(ucmd);
1591
1592 out:
1593         TRACE_EXIT_RES(res);
1594         return res;
1595
1596 out_wrong_state:
1597         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1598                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1599                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1600                 reply->subcode, ucmd->user_cmd.subcode);
1601         res = -EINVAL;
1602         dev_user_unjam_cmd(ucmd, 0, NULL);
1603
1604 out_unlock_put:
1605         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1606         goto out_put;
1607
1608 out_unlock:
1609         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1610         goto out;
1611 }
1612
1613 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1614 {
1615         int res = 0, rc;
1616         struct scst_user_dev *dev;
1617         struct scst_user_reply_cmd reply;
1618
1619         TRACE_ENTRY();
1620
1621         mutex_lock(&dev_priv_mutex);
1622         dev = (struct scst_user_dev *)file->private_data;
1623         res = dev_user_check_reg(dev);
1624         if (unlikely(res != 0)) {
1625                 mutex_unlock(&dev_priv_mutex);
1626                 goto out;
1627         }
1628         down_read(&dev->dev_rwsem);
1629         mutex_unlock(&dev_priv_mutex);
1630
1631         rc = copy_from_user(&reply, arg, sizeof(reply));
1632         if (unlikely(rc != 0)) {
1633                 PRINT_ERROR("Failed to copy %d user's bytes", rc);
1634                 res = -EFAULT;
1635                 goto out_up;
1636         }
1637
1638         TRACE_DBG("Reply for dev %s", dev->name);
1639
1640         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1641
1642         res = dev_user_process_reply(dev, &reply);
1643         if (unlikely(res < 0))
1644                 goto out_up;
1645
1646 out_up:
1647         up_read(&dev->dev_rwsem);
1648
1649 out:
1650         TRACE_EXIT_RES(res);
1651         return res;
1652 }
1653
1654 static int dev_user_get_ext_cdb(struct file *file, void __user *arg)
1655 {
1656         int res = 0, rc;
1657         struct scst_user_dev *dev;
1658         struct scst_user_cmd *ucmd;
1659         struct scst_cmd *cmd = NULL;
1660         struct scst_user_get_ext_cdb get;
1661
1662         TRACE_ENTRY();
1663
1664         mutex_lock(&dev_priv_mutex);
1665         dev = (struct scst_user_dev *)file->private_data;
1666         res = dev_user_check_reg(dev);
1667         if (unlikely(res != 0)) {
1668                 mutex_unlock(&dev_priv_mutex);
1669                 goto out;
1670         }
1671         down_read(&dev->dev_rwsem);
1672         mutex_unlock(&dev_priv_mutex);
1673
1674         rc = copy_from_user(&get, arg, sizeof(get));
1675         if (unlikely(rc != 0)) {
1676                 PRINT_ERROR("Failed to copy %d user's bytes", rc);
1677                 res = -EFAULT;
1678                 goto out_up;
1679         }
1680
1681         TRACE_MGMT_DBG("Get ext cdb for dev %s", dev->name);
1682
1683         TRACE_BUFFER("Get ext cdb", &get, sizeof(get));
1684
1685         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1686
1687         ucmd = __ucmd_find_hash(dev, get.cmd_h);
1688         if (unlikely(ucmd == NULL)) {
1689                 TRACE_MGMT_DBG("cmd_h %d not found", get.cmd_h);
1690                 res = -ESRCH;
1691                 goto out_unlock;
1692         }
1693
1694         if (unlikely(ucmd_get_check(ucmd))) {
1695                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", get.cmd_h);
1696                 res = -ESRCH;
1697                 goto out_unlock;
1698         }
1699
1700         if ((ucmd->cmd != NULL) && (ucmd->state <= UCMD_STATE_EXECING) &&
1701             (ucmd->sent_to_user || ucmd->background_exec)) {
1702                 cmd = ucmd->cmd;
1703                 scst_cmd_get(cmd);
1704         } else {
1705                 TRACE_MGMT_DBG("Invalid ucmd state %d for cmd_h %d",
1706                         ucmd->state, get.cmd_h);
1707                 res = -EINVAL;
1708                 goto out_unlock;
1709         }
1710
1711         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1712
1713         if (cmd == NULL)
1714                 goto out_put;
1715
1716         if (cmd->ext_cdb == NULL)
1717                 goto out_cmd_put;
1718
1719         TRACE_BUFFER("EXT CDB", cmd->ext_cdb, cmd->ext_cdb_len);
1720         rc = copy_to_user((void __user *)(unsigned long)get.ext_cdb_buffer,
1721                 cmd->ext_cdb, cmd->ext_cdb_len);
1722         if (unlikely(rc != 0)) {
1723                 PRINT_ERROR("Failed to copy to user %d bytes", rc);
1724                 res = -EFAULT;
1725                 goto out_cmd_put;
1726         }
1727
1728 out_cmd_put:
1729         scst_cmd_put(cmd);
1730
1731 out_put:
1732         ucmd_put(ucmd);
1733
1734 out_up:
1735         up_read(&dev->dev_rwsem);
1736
1737 out:
1738         TRACE_EXIT_RES(res);
1739         return res;
1740
1741 out_unlock:
1742         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1743         goto out_up;
1744 }
1745
1746 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1747         __releases(&dev->cmd_lists.cmd_list_lock)
1748         __acquires(&dev->cmd_lists.cmd_list_lock)
1749 {
1750         int res = 0;
1751
1752         TRACE_ENTRY();
1753
1754         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1755                 struct scst_cmd *cmd = list_entry(
1756                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1757                         cmd_list_entry);
1758                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1759                 list_del(&cmd->cmd_list_entry);
1760                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1761                 scst_process_active_cmd(cmd, false);
1762                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1763                 res++;
1764         }
1765
1766         TRACE_EXIT_RES(res);
1767         return res;
1768 }
1769
1770 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1771 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1772         __releases(&dev->cmd_lists.cmd_list_lock)
1773         __acquires(&dev->cmd_lists.cmd_list_lock)
1774 {
1775         struct scst_user_cmd *u;
1776
1777 again:
1778         u = NULL;
1779         if (!list_empty(cmd_list)) {
1780                 u = list_entry(cmd_list->next, typeof(*u),
1781                                ready_cmd_list_entry);
1782
1783                 TRACE_DBG("Found ready ucmd %p", u);
1784                 list_del(&u->ready_cmd_list_entry);
1785
1786                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1787
1788                 if (u->cmd != NULL) {
1789                         if (u->state == UCMD_STATE_EXECING) {
1790                                 struct scst_user_dev *dev = u->dev;
1791                                 int rc;
1792
1793                                 EXTRACHECKS_BUG_ON(u->jammed);
1794
1795                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1796
1797                                 rc = scst_check_local_events(u->cmd);
1798                                 if (unlikely(rc != 0)) {
1799                                         u->cmd->scst_cmd_done(u->cmd,
1800                                                 SCST_CMD_STATE_DEFAULT,
1801                                                 SCST_CONTEXT_DIRECT);
1802                                         /*
1803                                          * !! At this point cmd & u can be !!
1804                                          * !! already freed                !!
1805                                          */
1806                                         spin_lock_irq(
1807                                                 &dev->cmd_lists.cmd_list_lock);
1808                                         goto again;
1809                                 }
1810
1811                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1812                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1813                                         &u->cmd->cmd_flags))) {
1814                                 switch (u->state) {
1815                                 case UCMD_STATE_PARSING:
1816                                 case UCMD_STATE_BUF_ALLOCING:
1817                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1818                                         dev_user_unjam_cmd(u, 0, NULL);
1819                                         goto again;
1820                                 case UCMD_STATE_EXECING:
1821                                         EXTRACHECKS_BUG_ON(1);
1822                                 }
1823                         }
1824                 }
1825                 u->sent_to_user = 1;
1826                 u->seen_by_user = 1;
1827         }
1828         return u;
1829 }
1830
1831 static inline int test_cmd_lists(struct scst_user_dev *dev)
1832 {
1833         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1834                   !list_empty(&dev->ready_cmd_list) ||
1835                   !dev->blocking || dev->cleanup_done ||
1836                   signal_pending(current);
1837         return res;
1838 }
1839
1840 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1841 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1842         struct scst_user_cmd **ucmd)
1843 {
1844         int res = 0;
1845         wait_queue_t wait;
1846
1847         TRACE_ENTRY();
1848
1849         init_waitqueue_entry(&wait, current);
1850
1851         while (1) {
1852                 if (!test_cmd_lists(dev)) {
1853                         add_wait_queue_exclusive_head(
1854                                 &dev->cmd_lists.cmd_list_waitQ,
1855                                 &wait);
1856                         for (;;) {
1857                                 set_current_state(TASK_INTERRUPTIBLE);
1858                                 if (test_cmd_lists(dev))
1859                                         break;
1860                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1861                                 schedule();
1862                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1863                         }
1864                         set_current_state(TASK_RUNNING);
1865                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1866                                 &wait);
1867                 }
1868
1869                 dev_user_process_scst_commands(dev);
1870
1871                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1872                 if (*ucmd != NULL)
1873                         break;
1874
1875                 if (!dev->blocking || dev->cleanup_done) {
1876                         res = -EAGAIN;
1877                         TRACE_DBG("No ready commands, returning %d", res);
1878                         break;
1879                 }
1880
1881                 if (signal_pending(current)) {
1882                         res = -EINTR;
1883                         TRACE_DBG("Signal pending, returning %d", res);
1884                         break;
1885                 }
1886         }
1887
1888         TRACE_EXIT_RES(res);
1889         return res;
1890 }
1891
1892 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1893 {
1894         int res = 0, rc;
1895         struct scst_user_dev *dev;
1896         struct scst_user_get_cmd *cmd;
1897         struct scst_user_reply_cmd *reply;
1898         struct scst_user_cmd *ucmd;
1899         uint64_t ureply;
1900
1901         TRACE_ENTRY();
1902
1903         mutex_lock(&dev_priv_mutex);
1904         dev = (struct scst_user_dev *)file->private_data;
1905         res = dev_user_check_reg(dev);
1906         if (unlikely(res != 0)) {
1907                 mutex_unlock(&dev_priv_mutex);
1908                 goto out;
1909         }
1910         down_read(&dev->dev_rwsem);
1911         mutex_unlock(&dev_priv_mutex);
1912
1913         /* get_user() can't be used with 64-bit values on x86_32 */
1914         rc = copy_from_user(&ureply, (uint64_t __user *)
1915                 &((struct scst_user_get_cmd __user *)arg)->preply,
1916                 sizeof(ureply));
1917         if (unlikely(rc != 0)) {
1918                 PRINT_ERROR("Failed to copy %d user's bytes", rc);
1919                 res = -EFAULT;
1920                 goto out_up;
1921         }
1922
1923         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1924                 dev->name);
1925
1926         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1927         if (unlikely(cmd == NULL)) {
1928                 res = -ENOMEM;
1929                 goto out_up;
1930         }
1931
1932         if (ureply != 0) {
1933                 unsigned long u = (unsigned long)ureply;
1934                 reply = (struct scst_user_reply_cmd *)cmd;
1935                 rc = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1936                 if (unlikely(rc != 0)) {
1937                         PRINT_ERROR("Failed to copy %d user's bytes", rc);
1938                         res = -EFAULT;
1939                         goto out_free;
1940                 }
1941
1942                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1943
1944                 res = dev_user_process_reply(dev, reply);
1945                 if (unlikely(res < 0))
1946                         goto out_free;
1947         }
1948
1949         kmem_cache_free(user_get_cmd_cachep, cmd);
1950
1951         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1952 again:
1953         res = dev_user_get_next_cmd(dev, &ucmd);
1954         if (res == 0) {
1955                 int len;
1956                 /*
1957                  * A misbehaving user space handler can make ucmd to get dead
1958                  * immediately after we released the lock, which can lead to
1959                  * copy of dead data to the user space, which can lead to a
1960                  * leak of sensitive information.
1961                  */
1962                 if (unlikely(ucmd_get_check(ucmd))) {
1963                         /* Oops, this ucmd is already being destroyed. Retry. */
1964                         goto again;
1965                 }
1966                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1967
1968                 EXTRACHECKS_BUG_ON(ucmd->user_cmd_payload_len == 0);
1969
1970                 len = ucmd->user_cmd_payload_len;
1971                 TRACE_DBG("ucmd %p (user_cmd %p), payload_len %d (len %d)",
1972                         ucmd, &ucmd->user_cmd, ucmd->user_cmd_payload_len, len);
1973                 TRACE_BUFFER("UCMD", &ucmd->user_cmd, len);
1974                 rc = copy_to_user(arg, &ucmd->user_cmd, len);
1975                 if (unlikely(rc != 0)) {
1976                         PRINT_ERROR("Copy to user failed (%d), requeuing ucmd "
1977                                 "%p back to head of ready cmd list", rc, ucmd);
1978                         res = -EFAULT;
1979                         /* Requeue ucmd back */
1980                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1981                         list_add(&ucmd->ready_cmd_list_entry,
1982                                 &dev->ready_cmd_list);
1983                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1984                 }
1985 #ifdef CONFIG_SCST_EXTRACHECKS
1986                 else
1987                         ucmd->user_cmd_payload_len = 0;
1988 #endif
1989                 ucmd_put(ucmd);
1990         } else
1991                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1992
1993 out_up:
1994         up_read(&dev->dev_rwsem);
1995
1996 out:
1997         TRACE_EXIT_RES(res);
1998         return res;
1999
2000 out_free:
2001         kmem_cache_free(user_get_cmd_cachep, cmd);
2002         goto out_up;
2003 }
2004
2005 static long dev_user_ioctl(struct file *file, unsigned int cmd,
2006         unsigned long arg)
2007 {
2008         long res, rc;
2009
2010         TRACE_ENTRY();
2011
2012         switch (cmd) {
2013         case SCST_USER_REPLY_AND_GET_CMD:
2014                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
2015                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
2016                 break;
2017
2018         case SCST_USER_REPLY_CMD:
2019                 TRACE_DBG("%s", "REPLY_CMD");
2020                 res = dev_user_reply_cmd(file, (void __user *)arg);
2021                 break;
2022
2023         case SCST_USER_GET_EXTENDED_CDB:
2024                 TRACE_DBG("%s", "GET_EXTENDED_CDB");
2025                 res = dev_user_get_ext_cdb(file, (void __user *)arg);
2026                 break;
2027
2028         case SCST_USER_REGISTER_DEVICE:
2029         {
2030                 struct scst_user_dev_desc *dev_desc;
2031                 TRACE_DBG("%s", "REGISTER_DEVICE");
2032                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
2033                 if (dev_desc == NULL) {
2034                         res = -ENOMEM;
2035                         goto out;
2036                 }
2037                 rc = copy_from_user(dev_desc, (void __user *)arg,
2038                                      sizeof(*dev_desc));
2039                 if (rc != 0) {
2040                         PRINT_ERROR("Failed to copy %ld user's bytes", rc);
2041                         res = -EFAULT;
2042                         kfree(dev_desc);
2043                         goto out;
2044                 }
2045                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
2046                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
2047                 dev_desc->sgv_name[sizeof(dev_desc->sgv_name)-1] = '\0';
2048                 res = dev_user_register_dev(file, dev_desc);
2049                 kfree(dev_desc);
2050                 break;
2051         }
2052
2053         case SCST_USER_UNREGISTER_DEVICE:
2054                 TRACE_DBG("%s", "UNREGISTER_DEVICE");
2055                 res = dev_user_unregister_dev(file);
2056                 break;
2057
2058         case SCST_USER_FLUSH_CACHE:
2059                 TRACE_DBG("%s", "FLUSH_CACHE");
2060                 res = dev_user_flush_cache(file);
2061                 break;
2062
2063         case SCST_USER_SET_OPTIONS:
2064         {
2065                 struct scst_user_opt opt;
2066                 TRACE_DBG("%s", "SET_OPTIONS");
2067                 rc = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
2068                 if (rc != 0) {
2069                         PRINT_ERROR("Failed to copy %ld user's bytes", rc);
2070                         res = -EFAULT;
2071                         goto out;
2072                 }
2073                 TRACE_BUFFER("opt", &opt, sizeof(opt));
2074                 res = dev_user_set_opt(file, &opt);
2075                 break;
2076         }
2077
2078         case SCST_USER_GET_OPTIONS:
2079                 TRACE_DBG("%s", "GET_OPTIONS");
2080                 res = dev_user_get_opt(file, (void __user *)arg);
2081                 break;
2082
2083         case SCST_USER_DEVICE_CAPACITY_CHANGED:
2084                 TRACE_DBG("%s", "CAPACITY_CHANGED");
2085                 res = dev_user_capacity_changed(file);
2086                 break;
2087
2088         case SCST_USER_PREALLOC_BUFFER:
2089                 TRACE_DBG("%s", "PREALLOC_BUFFER");
2090                 res = dev_user_prealloc_buffer(file, (void __user *)arg);
2091                 break;
2092
2093         default:
2094                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
2095                 res = -EINVAL;
2096                 goto out;
2097         }
2098
2099 out:
2100         TRACE_EXIT_RES(res);
2101         return res;
2102 }
2103
2104 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
2105 {
2106         int res = 0;
2107         struct scst_user_dev *dev;
2108
2109         TRACE_ENTRY();
2110
2111         mutex_lock(&dev_priv_mutex);
2112         dev = (struct scst_user_dev *)file->private_data;
2113         res = dev_user_check_reg(dev);
2114         if (unlikely(res != 0)) {
2115                 mutex_unlock(&dev_priv_mutex);
2116                 goto out;
2117         }
2118         down_read(&dev->dev_rwsem);
2119         mutex_unlock(&dev_priv_mutex);
2120
2121         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2122
2123         if (!list_empty(&dev->ready_cmd_list) ||
2124             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2125                 res |= POLLIN | POLLRDNORM;
2126                 goto out_unlock;
2127         }
2128
2129         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2130
2131         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
2132         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
2133         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
2134
2135         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2136
2137         if (!list_empty(&dev->ready_cmd_list) ||
2138             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2139                 res |= POLLIN | POLLRDNORM;
2140                 goto out_unlock;
2141         }
2142
2143 out_unlock:
2144         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2145
2146         up_read(&dev->dev_rwsem);
2147
2148 out:
2149         TRACE_EXIT_HRES(res);
2150         return res;
2151 }
2152
2153 /*
2154  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
2155  */
2156 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
2157         unsigned long *flags)
2158         __releases(&dev->cmd_lists.cmd_list_lock)
2159         __acquires(&dev->cmd_lists.cmd_list_lock)
2160 {
2161         int state = ucmd->state;
2162         struct scst_user_dev *dev = ucmd->dev;
2163
2164         TRACE_ENTRY();
2165
2166         if (ucmd->this_state_unjammed)
2167                 goto out;
2168
2169         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
2170                 state);
2171
2172         ucmd->jammed = 1;
2173         ucmd->this_state_unjammed = 1;
2174         ucmd->sent_to_user = 0;
2175
2176         switch (state) {
2177         case UCMD_STATE_PARSING:
2178         case UCMD_STATE_BUF_ALLOCING:
2179                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2180                         ucmd->aborted = 1;
2181                 else {
2182                         if (busy)
2183                                 scst_set_busy(ucmd->cmd);
2184                         else
2185                                 scst_set_cmd_error(ucmd->cmd,
2186                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2187                 }
2188                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
2189
2190                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
2191                 list_add(&ucmd->cmd->cmd_list_entry,
2192                         &ucmd->cmd->cmd_lists->active_cmd_list);
2193                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
2194                 break;
2195
2196         case UCMD_STATE_EXECING:
2197                 if (flags != NULL)
2198                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2199                                                *flags);
2200                 else
2201                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2202
2203                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
2204
2205                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2206                         ucmd->aborted = 1;
2207                 else {
2208                         if (busy)
2209                                 scst_set_busy(ucmd->cmd);
2210                         else
2211                                 scst_set_cmd_error(ucmd->cmd,
2212                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2213                 }
2214
2215                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
2216                         SCST_CONTEXT_DIRECT);
2217                 /* !! At this point cmd and ucmd can be already freed !! */
2218
2219                 if (flags != NULL)
2220                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2221                                           *flags);
2222                 else
2223                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2224                 break;
2225
2226         case UCMD_STATE_ON_FREEING:
2227         case UCMD_STATE_ON_CACHE_FREEING:
2228         case UCMD_STATE_TM_EXECING:
2229         case UCMD_STATE_ATTACH_SESS:
2230         case UCMD_STATE_DETACH_SESS:
2231                 if (flags != NULL)
2232                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2233                                                *flags);
2234                 else
2235                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2236
2237                 switch (state) {
2238                 case UCMD_STATE_ON_FREEING:
2239                         dev_user_process_reply_on_free(ucmd);
2240                         break;
2241
2242                 case UCMD_STATE_ON_CACHE_FREEING:
2243                         dev_user_process_reply_on_cache_free(ucmd);
2244                         break;
2245
2246                 case UCMD_STATE_TM_EXECING:
2247                         dev_user_process_reply_tm_exec(ucmd,
2248                                                        SCST_MGMT_STATUS_FAILED);
2249                         break;
2250
2251                 case UCMD_STATE_ATTACH_SESS:
2252                 case UCMD_STATE_DETACH_SESS:
2253                         dev_user_process_reply_sess(ucmd, -EFAULT);
2254                         break;
2255                 }
2256
2257                 if (flags != NULL)
2258                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2259                                           *flags);
2260                 else
2261                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2262                 break;
2263
2264         default:
2265                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
2266                 sBUG();
2267                 break;
2268         }
2269
2270 out:
2271         TRACE_EXIT();
2272         return;
2273 }
2274
2275 static int dev_user_unjam_dev(struct scst_user_dev *dev)
2276         __releases(&dev->cmd_lists.cmd_list_lock)
2277         __acquires(&dev->cmd_lists.cmd_list_lock)
2278 {
2279         int i, res = 0;
2280         struct scst_user_cmd *ucmd;
2281
2282         TRACE_ENTRY();
2283
2284         TRACE_MGMT_DBG("Unjamming dev %p", dev);
2285
2286         sgv_pool_flush(dev->pool);
2287         sgv_pool_flush(dev->pool_clust);
2288
2289         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2290
2291 repeat:
2292         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2293                 struct list_head *head = &dev->ucmd_hash[i];
2294
2295                 list_for_each_entry(ucmd, head, hash_list_entry) {
2296                         res++;
2297
2298                         if (!ucmd->sent_to_user)
2299                                 continue;
2300
2301                         if (ucmd_get_check(ucmd))
2302                                 continue;
2303
2304                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2305                                 ucmd->state, ucmd->cmd);
2306
2307                         dev_user_unjam_cmd(ucmd, 0, NULL);
2308
2309                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2310                         ucmd_put(ucmd);
2311                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2312
2313                         goto repeat;
2314                 }
2315         }
2316
2317         if (dev_user_process_scst_commands(dev) != 0)
2318                 goto repeat;
2319
2320         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2321
2322         TRACE_EXIT_RES(res);
2323         return res;
2324 }
2325
2326 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2327         int status)
2328 {
2329         int res = 0;
2330
2331         TRACE_ENTRY();
2332
2333         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2334                 ucmd->user_cmd.tm_cmd.fn, status);
2335
2336         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2337                 /*
2338                  * It is possible that user space seen TM cmd before cmd
2339                  * to abort or will never see it at all, because it was
2340                  * aborted on the way there. So, it is safe to return
2341                  * success instead, because, if there is the TM cmd at this
2342                  * point, then the cmd to abort apparrently does exist.
2343                  */
2344                 status = SCST_MGMT_STATUS_SUCCESS;
2345         }
2346
2347         scst_async_mcmd_completed(ucmd->mcmd, status);
2348
2349         ucmd_put(ucmd);
2350
2351         TRACE_EXIT_RES(res);
2352         return res;
2353 }
2354
2355 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2356 {
2357         struct scst_user_cmd *ucmd;
2358         unsigned long flags;
2359
2360         TRACE_ENTRY();
2361
2362         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2363 again:
2364         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2365                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2366                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2367                         switch (ucmd->state) {
2368                         case UCMD_STATE_PARSING:
2369                         case UCMD_STATE_BUF_ALLOCING:
2370                         case UCMD_STATE_EXECING:
2371                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2372                                 list_del(&ucmd->ready_cmd_list_entry);
2373                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2374                                 goto again;
2375                         }
2376                 }
2377         }
2378
2379         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2380
2381         TRACE_EXIT();
2382         return;
2383 }
2384
2385 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2386         struct scst_tgt_dev *tgt_dev)
2387 {
2388         struct scst_user_cmd *ucmd;
2389         struct scst_user_dev *dev =
2390                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2391         struct scst_user_cmd *ucmd_to_abort = NULL;
2392
2393         TRACE_ENTRY();
2394
2395         /*
2396          * In the used approach we don't do anything with hung devices, which
2397          * stopped responding and/or have stuck commands. We forcedly abort such
2398          * commands only if they not yet sent to the user space or if the device
2399          * is getting unloaded, e.g. if its handler program gets killed. This is
2400          * because it's pretty hard to distinguish between stuck and temporary
2401          * overloaded states of the device. There are several reasons for that:
2402          *
2403          * 1. Some commands need a lot of time to complete (several hours),
2404          *    so for an impatient user such command(s) will always look as
2405          *    stuck.
2406          *
2407          * 2. If we forcedly abort, i.e. abort before it's actually completed
2408          *    in the user space, just one command, we will have to put the whole
2409          *    device offline until we are sure that no more previously aborted
2410          *    commands will get executed. Otherwise, we might have a possibility
2411          *    for data corruption, when aborted and reported as completed
2412          *    command actually gets executed *after* new commands sent
2413          *    after the force abort was done. Many journaling file systems and
2414          *    databases use "provide required commands order via queue draining"
2415          *    approach and not putting the whole device offline after the forced
2416          *    abort will break it. This makes our decision, if a command stuck
2417          *    or not, cost a lot.
2418          *
2419          * So, we leave policy definition if a device stuck or not to
2420          * the user space and simply let all commands live until they are
2421          * completed or their devices get closed/killed. This approach is very
2422          * much OK, but can affect management commands, which need activity
2423          * suspending via scst_suspend_activity() function such as devices or
2424          * targets registration/removal. But during normal life such commands
2425          * should be rare. Plus, when possible, scst_suspend_activity() will
2426          * return after timeout EBUSY status to allow caller to not stuck
2427          * forever as well.
2428          *
2429          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2430          * stuck commands would affect only related devices.
2431          */
2432
2433         dev_user_abort_ready_commands(dev);
2434
2435         /* We can't afford missing TM command due to memory shortage */
2436         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2437
2438         ucmd->user_cmd_payload_len =
2439                 offsetof(struct scst_user_get_cmd, tm_cmd) +
2440                 sizeof(ucmd->user_cmd.tm_cmd);
2441         ucmd->user_cmd.cmd_h = ucmd->h;
2442         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2443         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2444         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2445         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2446         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2447
2448         if (mcmd->cmd_to_abort != NULL) {
2449                 ucmd_to_abort =
2450                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2451                 if (ucmd_to_abort != NULL)
2452                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2453         }
2454
2455         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2456                 "ucmd_to_abort %p, cmd_h_to_abort %d, mcmd %p)", ucmd, ucmd->h,
2457                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2458                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort, mcmd);
2459
2460         ucmd->mcmd = mcmd;
2461         ucmd->state = UCMD_STATE_TM_EXECING;
2462
2463         scst_prepare_async_mcmd(mcmd);
2464
2465         dev_user_add_to_ready(ucmd);
2466
2467         TRACE_EXIT();
2468         return SCST_DEV_TM_NOT_COMPLETED;
2469 }
2470
2471 static int dev_user_attach(struct scst_device *sdev)
2472 {
2473         int res = 0;
2474         struct scst_user_dev *dev = NULL, *d;
2475
2476         TRACE_ENTRY();
2477
2478         spin_lock(&dev_list_lock);
2479         list_for_each_entry(d, &dev_list, dev_list_entry) {
2480                 if (strcmp(d->name, sdev->virt_name) == 0) {
2481                         dev = d;
2482                         break;
2483                 }
2484         }
2485         spin_unlock(&dev_list_lock);
2486         if (dev == NULL) {
2487                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2488                 res = -EINVAL;
2489                 goto out;
2490         }
2491
2492         sdev->p_cmd_lists = &dev->cmd_lists;
2493         sdev->dh_priv = dev;
2494         sdev->tst = dev->tst;
2495         sdev->queue_alg = dev->queue_alg;
2496         sdev->swp = dev->swp;
2497         sdev->tas = dev->tas;
2498         sdev->d_sense = dev->d_sense;
2499         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2500
2501         dev->sdev = sdev;
2502
2503         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2504                 dev->name);
2505
2506 out:
2507         TRACE_EXIT();
2508         return res;
2509 }
2510
2511 static void dev_user_detach(struct scst_device *sdev)
2512 {
2513         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2514
2515         TRACE_ENTRY();
2516
2517         TRACE_DBG("virt_id %d", sdev->virt_id);
2518
2519         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2520                 dev->name);
2521
2522         /* dev will be freed by the caller */
2523         sdev->dh_priv = NULL;
2524         dev->sdev = NULL;
2525
2526         TRACE_EXIT();
2527         return;
2528 }
2529
2530 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2531 {
2532         int res = 0;
2533         unsigned long flags;
2534
2535         TRACE_ENTRY();
2536
2537         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2538
2539         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2540
2541         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2542                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2543                 ucmd->result = status;
2544         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2545                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2546         } else
2547                 sBUG();
2548
2549         if (ucmd->cmpl != NULL)
2550                 complete_all(ucmd->cmpl);
2551
2552         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2553
2554         ucmd_put(ucmd);
2555
2556         TRACE_EXIT_RES(res);
2557         return res;
2558 }
2559
2560 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2561 {
2562         struct scst_user_dev *dev =
2563                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2564         int res = 0, rc;
2565         struct scst_user_cmd *ucmd;
2566         DECLARE_COMPLETION_ONSTACK(cmpl);
2567
2568         TRACE_ENTRY();
2569
2570         /*
2571          * We can't replace tgt_dev->pool, because it can be used to allocate
2572          * memory for SCST local commands, like REPORT LUNS, where there is no
2573          * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
2574          */
2575         if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
2576                 tgt_dev->dh_priv = dev->pool_clust;
2577         else
2578                 tgt_dev->dh_priv = dev->pool;
2579
2580         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2581         if (ucmd == NULL)
2582                 goto out_nomem;
2583
2584         ucmd->cmpl = &cmpl;
2585
2586         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2587                 sizeof(ucmd->user_cmd.sess);
2588         ucmd->user_cmd.cmd_h = ucmd->h;
2589         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2590         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2591         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2592         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2593         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only;
2594         strlcpy(ucmd->user_cmd.sess.initiator_name,
2595                 tgt_dev->sess->initiator_name,
2596                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2597         strlcpy(ucmd->user_cmd.sess.target_name,
2598                 tgt_dev->sess->tgt->tgt_name,
2599                 sizeof(ucmd->user_cmd.sess.target_name)-1);
2600
2601         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2602                 "threads_num %d, rd_only %d, initiator %s, target %s)",
2603                 ucmd, ucmd->h, ucmd->user_cmd.sess.sess_h,
2604                 ucmd->user_cmd.sess.lun, ucmd->user_cmd.sess.threads_num,
2605                 ucmd->user_cmd.sess.rd_only, ucmd->user_cmd.sess.initiator_name,
2606                 ucmd->user_cmd.sess.target_name);
2607
2608         ucmd->state = UCMD_STATE_ATTACH_SESS;
2609
2610         ucmd_get(ucmd);
2611
2612         dev_user_add_to_ready(ucmd);
2613
2614         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2615         if (rc > 0)
2616                 res = ucmd->result;
2617         else {
2618                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2619                 res = -EFAULT;
2620         }
2621
2622         sBUG_ON(irqs_disabled());
2623
2624         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2625         ucmd->cmpl = NULL;
2626         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2627
2628         ucmd_put(ucmd);
2629
2630 out:
2631         TRACE_EXIT_RES(res);
2632         return res;
2633
2634 out_nomem:
2635         res = -ENOMEM;
2636         goto out;
2637 }
2638
2639 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2640 {
2641         struct scst_user_dev *dev =
2642                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2643         struct scst_user_cmd *ucmd;
2644
2645         TRACE_ENTRY();
2646
2647         /*
2648          * We can't miss TM command due to memory shortage, because it might
2649          * lead to a memory leak in the user space handler.
2650          */
2651         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2652         if (ucmd == NULL)
2653                 goto out;
2654
2655         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2656                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2657
2658         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2659                 sizeof(ucmd->user_cmd.sess);
2660         ucmd->user_cmd.cmd_h = ucmd->h;
2661         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2662         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2663
2664         ucmd->state = UCMD_STATE_DETACH_SESS;
2665
2666         dev_user_add_to_ready(ucmd);
2667
2668 out:
2669         TRACE_EXIT();
2670         return;
2671 }
2672
2673 /* No locks are needed, but the activity must be suspended */
2674 static void dev_user_setup_functions(struct scst_user_dev *dev)
2675 {
2676         TRACE_ENTRY();
2677
2678         dev->devtype.parse = dev_user_parse;
2679         dev->devtype.dev_done = NULL;
2680
2681         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2682                 switch (dev->devtype.type) {
2683                 case TYPE_DISK:
2684                         dev->generic_parse = scst_sbc_generic_parse;
2685                         dev->devtype.dev_done = dev_user_disk_done;
2686                         break;
2687
2688                 case TYPE_TAPE:
2689                         dev->generic_parse = scst_tape_generic_parse;
2690                         dev->devtype.dev_done = dev_user_tape_done;
2691                         break;
2692
2693                 case TYPE_MOD:
2694                         dev->generic_parse = scst_modisk_generic_parse;
2695                         dev->devtype.dev_done = dev_user_disk_done;
2696                         break;
2697
2698                 case TYPE_ROM:
2699                         dev->generic_parse = scst_cdrom_generic_parse;
2700                         dev->devtype.dev_done = dev_user_disk_done;
2701                         break;
2702
2703                 case TYPE_MEDIUM_CHANGER:
2704                         dev->generic_parse = scst_changer_generic_parse;
2705                         break;
2706
2707                 case TYPE_PROCESSOR:
2708                         dev->generic_parse = scst_processor_generic_parse;
2709                         break;
2710
2711                 case TYPE_RAID:
2712                         dev->generic_parse = scst_raid_generic_parse;
2713                         break;
2714
2715                 default:
2716                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2717                                 "for it", dev->devtype.type);
2718                         dev->parse_type = SCST_USER_PARSE_CALL;
2719                         break;
2720                 }
2721         } else {
2722                 dev->generic_parse = NULL;
2723                 dev->devtype.dev_done = NULL;
2724         }
2725
2726         TRACE_EXIT();
2727         return;
2728 }
2729
2730 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2731 {
2732         char ver[sizeof(DEV_USER_VERSION)+1];
2733         int res = 0, rc;
2734
2735         rc = copy_from_user(ver,
2736                         (void __user *)(unsigned long)dev_desc->version_str,
2737                         sizeof(ver));
2738         if (rc != 0) {
2739                 PRINT_ERROR("%s", "Unable to get version string");
2740                 res = -EFAULT;
2741                 goto out;
2742         }
2743         ver[sizeof(ver)-1] = '\0';
2744
2745         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2746                 /* ->name already 0-terminated in dev_user_ioctl() */
2747                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2748                         dev_desc->name, ver);
2749                 res = -EINVAL;
2750                 goto out;
2751         }
2752
2753 out:
2754         return res;
2755 }
2756
2757 static int dev_user_register_dev(struct file *file,
2758         const struct scst_user_dev_desc *dev_desc)
2759 {
2760         int res, i;
2761         struct scst_user_dev *dev, *d;
2762         int block;
2763
2764         TRACE_ENTRY();
2765
2766         res = dev_user_check_version(dev_desc);
2767         if (res != 0)
2768                 goto out;
2769
2770         switch (dev_desc->type) {
2771         case TYPE_DISK:
2772         case TYPE_ROM:
2773         case TYPE_MOD:
2774                 if (dev_desc->block_size == 0) {
2775                         PRINT_ERROR("Wrong block size %d",
2776                                     dev_desc->block_size);
2777                         res = -EINVAL;
2778                         goto out;
2779                 }
2780                 block = scst_calc_block_shift(dev_desc->block_size);
2781                 if (block == -1) {
2782                         res = -EINVAL;
2783                         goto out;
2784                 }
2785                 break;
2786         default:
2787                 block = dev_desc->block_size;
2788                 break;
2789         }
2790
2791         if (!try_module_get(THIS_MODULE)) {
2792                 PRINT_ERROR("%s", "Fail to get module");
2793                 res = -ETXTBSY;
2794                 goto out;
2795         }
2796
2797         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2798         if (dev == NULL) {
2799                 res = -ENOMEM;
2800                 goto out_put;
2801         }
2802
2803         init_rwsem(&dev->dev_rwsem);
2804         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2805         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2806         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2807         INIT_LIST_HEAD(&dev->ready_cmd_list);
2808         if (file->f_flags & O_NONBLOCK) {
2809                 TRACE_DBG("%s", "Non-blocking operations");
2810                 dev->blocking = 0;
2811         } else
2812                 dev->blocking = 1;
2813         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2814                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2815
2816         strlcpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2817
2818         scst_init_mem_lim(&dev->udev_mem_lim);
2819
2820         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2821                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2822                                                   dev_desc->sgv_name);
2823         dev->pool = sgv_pool_create(dev->devtype.name, sgv_no_clustering,
2824                                         dev_desc->sgv_single_alloc_pages,
2825                                         dev_desc->sgv_shared,
2826                                         dev_desc->sgv_purge_interval);
2827         if (dev->pool == NULL) {
2828                 res = -ENOMEM;
2829                 goto out_free_dev;
2830         }
2831         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2832                 dev_user_free_sg_entries);
2833
2834         if (!dev_desc->sgv_disable_clustered_pool) {
2835                 scnprintf(dev->devtype.name, sizeof(dev->devtype.name),
2836                         "%s-clust",
2837                         (dev_desc->sgv_name[0] == '\0') ? dev->name :
2838                                                           dev_desc->sgv_name);
2839                 dev->pool_clust = sgv_pool_create(dev->devtype.name,
2840                                         sgv_tail_clustering,
2841                                         dev_desc->sgv_single_alloc_pages,
2842                                         dev_desc->sgv_shared,
2843                                         dev_desc->sgv_purge_interval);
2844                 if (dev->pool_clust == NULL) {
2845                         res = -ENOMEM;
2846                         goto out_free0;
2847                 }
2848                 sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
2849                         dev_user_free_sg_entries);
2850         } else {
2851                 dev->pool_clust = dev->pool;
2852                 sgv_pool_get(dev->pool_clust);
2853         }
2854
2855         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2856                 dev->name);
2857         dev->devtype.type = dev_desc->type;
2858         dev->devtype.threads_num = -1;
2859         dev->devtype.parse_atomic = 1;
2860         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2861         dev->devtype.dev_done_atomic = 1;
2862 #ifdef CONFIG_SCST_PROC
2863         dev->devtype.no_proc = 1;
2864 #else
2865         dev->devtype.dev_attrs = dev_user_dev_attrs;
2866 #endif
2867         dev->devtype.attach = dev_user_attach;
2868         dev->devtype.detach = dev_user_detach;
2869         dev->devtype.attach_tgt = dev_user_attach_tgt;
2870         dev->devtype.detach_tgt = dev_user_detach_tgt;
2871         dev->devtype.exec = dev_user_exec;
2872         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2873         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2874
2875         dev->devtype.parent = &dev_user_devtype;
2876
2877         init_completion(&dev->cleanup_cmpl);
2878         dev->block = block;
2879         dev->def_block = block;
2880
2881         res = __dev_user_set_opt(dev, &dev_desc->opt);
2882         if (res != 0)
2883                 goto out_free;
2884
2885         TRACE_MEM("dev %p, name %s", dev, dev->name);
2886
2887         spin_lock(&dev_list_lock);
2888
2889         list_for_each_entry(d, &dev_list, dev_list_entry) {
2890                 if (strcmp(d->name, dev->name) == 0) {
2891                         PRINT_ERROR("Device %s already exist",
2892                                 dev->name);
2893                         res = -EEXIST;
2894                         spin_unlock(&dev_list_lock);
2895                         goto out_free;
2896                 }
2897         }
2898
2899         list_add_tail(&dev->dev_list_entry, &dev_list);
2900
2901         spin_unlock(&dev_list_lock);
2902
2903         res = scst_register_virtual_dev_driver(&dev->devtype);
2904         if (res < 0)
2905                 goto out_del_free;
2906
2907         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2908         if (dev->virt_id < 0) {
2909                 res = dev->virt_id;
2910                 goto out_unreg_handler;
2911         }
2912
2913         mutex_lock(&dev_priv_mutex);
2914         if (file->private_data != NULL) {
2915                 mutex_unlock(&dev_priv_mutex);
2916                 PRINT_ERROR("%s", "Device already registered");
2917                 res = -EINVAL;
2918                 goto out_unreg_drv;
2919         }
2920         file->private_data = dev;
2921         mutex_unlock(&dev_priv_mutex);
2922
2923 out:
2924         TRACE_EXIT_RES(res);
2925         return res;
2926
2927 out_unreg_drv:
2928         scst_unregister_virtual_device(dev->virt_id);
2929
2930 out_unreg_handler:
2931         scst_unregister_virtual_dev_driver(&dev->devtype);
2932
2933 out_del_free:
2934         spin_lock(&dev_list_lock);
2935         list_del(&dev->dev_list_entry);
2936         spin_unlock(&dev_list_lock);
2937
2938 out_free:
2939         sgv_pool_del(dev->pool_clust);
2940
2941 out_free0:
2942         sgv_pool_del(dev->pool);
2943
2944 out_free_dev:
2945         kfree(dev);
2946
2947 out_put:
2948         module_put(THIS_MODULE);
2949         goto out;
2950 }
2951
2952 static int dev_user_unregister_dev(struct file *file)
2953 {
2954         int res;
2955         struct scst_user_dev *dev;
2956
2957         TRACE_ENTRY();
2958
2959         mutex_lock(&dev_priv_mutex);
2960         dev = (struct scst_user_dev *)file->private_data;
2961         res = dev_user_check_reg(dev);
2962         if (res != 0) {
2963                 mutex_unlock(&dev_priv_mutex);
2964                 goto out;
2965         }
2966         down_read(&dev->dev_rwsem);
2967         mutex_unlock(&dev_priv_mutex);
2968
2969         res = scst_suspend_activity(true);
2970         if (res != 0)
2971                 goto out_up;
2972
2973         up_read(&dev->dev_rwsem);
2974
2975         mutex_lock(&dev_priv_mutex);
2976         dev = (struct scst_user_dev *)file->private_data;
2977         if (dev == NULL) {
2978                 mutex_unlock(&dev_priv_mutex);
2979                 goto out_resume;
2980         }
2981
2982         dev->blocking = 0;
2983         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
2984
2985         down_write(&dev->dev_rwsem);
2986         file->private_data = NULL;
2987         mutex_unlock(&dev_priv_mutex);
2988
2989         dev_user_exit_dev(dev);
2990
2991         up_write(&dev->dev_rwsem); /* to make lockdep happy */
2992
2993         kfree(dev);
2994
2995 out_resume:
2996         scst_resume_activity();
2997
2998 out:
2999         TRACE_EXIT_RES(res);
3000         return res;
3001
3002 out_up:
3003         up_read(&dev->dev_rwsem);
3004         goto out;
3005 }
3006
3007 static int dev_user_flush_cache(struct file *file)
3008 {
3009         int res;
3010         struct scst_user_dev *dev;
3011
3012         TRACE_ENTRY();
3013
3014         mutex_lock(&dev_priv_mutex);
3015         dev = (struct scst_user_dev *)file->private_data;
3016         res = dev_user_check_reg(dev);
3017         if (res != 0) {
3018                 mutex_unlock(&dev_priv_mutex);
3019                 goto out;
3020         }
3021         down_read(&dev->dev_rwsem);
3022         mutex_unlock(&dev_priv_mutex);
3023
3024         res = scst_suspend_activity(true);
3025         if (res != 0)
3026                 goto out_up;
3027
3028         sgv_pool_flush(dev->pool);
3029         sgv_pool_flush(dev->pool_clust);
3030
3031         scst_resume_activity();
3032
3033 out_up:
3034         up_read(&dev->dev_rwsem);
3035
3036 out:
3037         TRACE_EXIT_RES(res);
3038         return res;
3039 }
3040
3041 static int dev_user_capacity_changed(struct file *file)
3042 {
3043         int res;
3044         struct scst_user_dev *dev;
3045
3046         TRACE_ENTRY();
3047
3048         mutex_lock(&dev_priv_mutex);
3049         dev = (struct scst_user_dev *)file->private_data;
3050         res = dev_user_check_reg(dev);
3051         if (res != 0) {
3052                 mutex_unlock(&dev_priv_mutex);
3053                 goto out;
3054         }
3055         down_read(&dev->dev_rwsem);
3056         mutex_unlock(&dev_priv_mutex);
3057
3058         scst_capacity_data_changed(dev->sdev);
3059
3060         up_read(&dev->dev_rwsem);
3061
3062 out:
3063         TRACE_EXIT_RES(res);
3064         return res;
3065 }
3066
3067 static int dev_user_prealloc_buffer(struct file *file, void __user *arg)
3068 {
3069         int res = 0, rc;
3070         struct scst_user_dev *dev;
3071         union scst_user_prealloc_buffer pre;
3072         aligned_u64 pbuf;
3073         uint32_t bufflen;
3074         struct scst_user_cmd *ucmd;
3075         int pages, sg_cnt;
3076         struct sgv_pool *pool;
3077         struct scatterlist *sg;
3078
3079         TRACE_ENTRY();
3080
3081         mutex_lock(&dev_priv_mutex);
3082         dev = (struct scst_user_dev *)file->private_data;
3083         res = dev_user_check_reg(dev);
3084         if (unlikely(res != 0)) {
3085                 mutex_unlock(&dev_priv_mutex);
3086                 goto out;
3087         }
3088         down_read(&dev->dev_rwsem);
3089         mutex_unlock(&dev_priv_mutex);
3090
3091         rc = copy_from_user(&pre.in, arg, sizeof(pre.in));
3092         if (unlikely(rc != 0)) {
3093                 PRINT_ERROR("Failed to copy %d user's bytes", rc);
3094                 res = -EFAULT;
3095                 goto out_up;
3096         }
3097
3098         TRACE_MEM("Prealloc buffer with size %dKB for dev %s",
3099                 pre.in.bufflen / 1024, dev->name);
3100         TRACE_BUFFER("Input param", &pre.in, sizeof(pre.in));
3101
3102         pbuf = pre.in.pbuf;
3103         bufflen = pre.in.bufflen;
3104
3105         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
3106         if (ucmd == NULL) {
3107                 res = -ENOMEM;
3108                 goto out_up;
3109         }
3110
3111         ucmd->buff_cached = 1;
3112
3113         TRACE_MEM("ucmd %p, pbuf %llx", ucmd, pbuf);
3114
3115         if (unlikely((pbuf & ~PAGE_MASK) != 0)) {
3116                 PRINT_ERROR("Supplied pbuf %llx isn't page aligned", pbuf);
3117                 res = -EINVAL;
3118                 goto out_put;
3119         }
3120
3121         pages = calc_num_pg(pbuf, bufflen);
3122         res = dev_user_map_buf(ucmd, pbuf, pages);
3123         if (res != 0)
3124                 goto out_put;
3125
3126         if (pre.in.for_clust_pool)
3127                 pool = dev->pool_clust;
3128         else
3129                 pool = dev->pool;
3130
3131         sg = sgv_pool_alloc(pool, bufflen, GFP_KERNEL, SGV_POOL_ALLOC_GET_NEW,
3132                          &sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
3133         if (sg != NULL) {
3134                 struct scst_user_cmd *buf_ucmd =
3135                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
3136
3137                 TRACE_MEM("Buf ucmd %p (sg_cnt %d, last seg len %d, "
3138                         "bufflen %d)", buf_ucmd, sg_cnt,
3139                         sg[sg_cnt-1].length, bufflen);
3140
3141                 EXTRACHECKS_BUG_ON(ucmd != buf_ucmd);
3142
3143                 ucmd->buf_ucmd = buf_ucmd;
3144         } else {
3145                 res = -ENOMEM;
3146                 goto out_put;
3147         }
3148
3149         dev_user_free_sgv(ucmd);
3150
3151         pre.out.cmd_h = ucmd->h;
3152         rc = copy_to_user(arg, &pre.out, sizeof(pre.out));
3153         if (unlikely(rc != 0)) {
3154                 PRINT_ERROR("Failed to copy to user %d bytes", rc);
3155                 res = -EFAULT;
3156                 goto out_put;
3157         }
3158
3159 out_put:
3160         ucmd_put(ucmd);
3161
3162 out_up:
3163         up_read(&dev->dev_rwsem);
3164
3165 out:
3166         TRACE_EXIT_RES(res);
3167         return res;
3168 }
3169
3170 static int __dev_user_set_opt(struct scst_user_dev *dev,
3171         const struct scst_user_opt *opt)
3172 {
3173         int res = 0;
3174
3175         TRACE_ENTRY();
3176
3177         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3178                 "memory_reuse_type %x, partial_transfers_type %x, "
3179                 "partial_len %d", dev->name, opt->parse_type,
3180                 opt->on_free_cmd_type, opt->memory_reuse_type,
3181                 opt->partial_transfers_type, opt->partial_len);
3182
3183         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
3184             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
3185             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
3186             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
3187                 PRINT_ERROR("%s", "Invalid option");
3188                 res = -EINVAL;
3189                 goto out;
3190         }
3191
3192         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
3193              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
3194             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
3195              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
3196             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) ||
3197             (opt->d_sense > 1)) {
3198                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
3199                         " tas %x, d_sense %d, has_own_order_mgmt %x)", opt->tst,
3200                         opt->queue_alg, opt->swp, opt->tas, opt->d_sense,
3201                         opt->has_own_order_mgmt);
3202                 res = -EINVAL;
3203                 goto out;
3204         }
3205
3206         dev->parse_type = opt->parse_type;
3207         dev->on_free_cmd_type = opt->on_free_cmd_type;
3208         dev->memory_reuse_type = opt->memory_reuse_type;
3209         dev->partial_transfers_type = opt->partial_transfers_type;
3210         dev->partial_len = opt->partial_len;
3211
3212         dev->tst = opt->tst;
3213         dev->queue_alg = opt->queue_alg;
3214         dev->swp = opt->swp;
3215         dev->tas = opt->tas;
3216         dev->tst = opt->tst;
3217         dev->d_sense = opt->d_sense;
3218         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
3219         if (dev->sdev != NULL) {
3220                 dev->sdev->tst = opt->tst;
3221                 dev->sdev->queue_alg = opt->queue_alg;
3222                 dev->sdev->swp = opt->swp;
3223                 dev->sdev->tas = opt->tas;
3224                 dev->sdev->d_sense = opt->d_sense;
3225                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
3226         }
3227
3228         dev_user_setup_functions(dev);
3229
3230 out:
3231         TRACE_EXIT_RES(res);
3232         return res;
3233 }
3234
3235 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
3236 {
3237         int res;
3238         struct scst_user_dev *dev;
3239
3240         TRACE_ENTRY();
3241
3242         mutex_lock(&dev_priv_mutex);
3243         dev = (struct scst_user_dev *)file->private_data;
3244         res = dev_user_check_reg(dev);
3245         if (res != 0) {
3246                 mutex_unlock(&dev_priv_mutex);
3247                 goto out;
3248         }
3249         down_read(&dev->dev_rwsem);
3250         mutex_unlock(&dev_priv_mutex);
3251
3252         res = scst_suspend_activity(true);
3253         if (res != 0)
3254                 goto out_up;
3255
3256         res = __dev_user_set_opt(dev, opt);
3257
3258         scst_resume_activity();
3259
3260 out_up:
3261         up_read(&dev->dev_rwsem);
3262
3263 out:
3264         TRACE_EXIT_RES(res);
3265         return res;
3266 }
3267
3268 static int dev_user_get_opt(struct file *file, void __user *arg)
3269 {
3270         int res, rc;
3271         struct scst_user_dev *dev;
3272         struct scst_user_opt opt;
3273
3274         TRACE_ENTRY();
3275
3276         mutex_lock(&dev_priv_mutex);
3277         dev = (struct scst_user_dev *)file->private_data;
3278         res = dev_user_check_reg(dev);
3279         if (res != 0) {
3280                 mutex_unlock(&dev_priv_mutex);
3281                 goto out;
3282         }
3283         down_read(&dev->dev_rwsem);
3284         mutex_unlock(&dev_priv_mutex);
3285
3286         opt.parse_type = dev->parse_type;
3287         opt.on_free_cmd_type = dev->on_free_cmd_type;
3288         opt.memory_reuse_type = dev->memory_reuse_type;
3289         opt.partial_transfers_type = dev->partial_transfers_type;
3290         opt.partial_len = dev->partial_len;
3291         opt.tst = dev->tst;
3292         opt.queue_alg = dev->queue_alg;
3293         opt.tas = dev->tas;
3294         opt.swp = dev->swp;
3295         opt.d_sense = dev->d_sense;
3296         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
3297
3298         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3299                 "memory_reuse_type %x, partial_transfers_type %x, "
3300                 "partial_len %d", dev->name, opt.parse_type,
3301                 opt.on_free_cmd_type, opt.memory_reuse_type,
3302                 opt.partial_transfers_type, opt.partial_len);
3303
3304         rc = copy_to_user(arg, &opt, sizeof(opt));
3305         if (unlikely(rc != 0)) {
3306                 PRINT_ERROR("Failed to copy to user %d bytes", rc);
3307                 res = -EFAULT;
3308                 goto out_up;
3309         }
3310
3311 out_up:
3312         up_read(&dev->dev_rwsem);
3313
3314 out:
3315         TRACE_EXIT_RES(res);
3316         return res;
3317 }
3318
3319 static int dev_usr_parse(struct scst_cmd *cmd)
3320 {
3321         sBUG();
3322         return SCST_CMD_STATE_DEFAULT;
3323 }
3324
3325 static int dev_user_exit_dev(struct scst_user_dev *dev)
3326 {
3327         TRACE_ENTRY();
3328
3329         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
3330
3331         spin_lock(&dev_list_lock);
3332         list_del(&dev->dev_list_entry);
3333         spin_unlock(&dev_list_lock);
3334
3335         dev->blocking = 0;
3336         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
3337
3338         spin_lock(&cleanup_lock);
3339         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
3340         spin_unlock(&cleanup_lock);
3341
3342         wake_up(&cleanup_list_waitQ);
3343
3344         scst_unregister_virtual_device(dev->virt_id);
3345         scst_unregister_virtual_dev_driver(&dev->devtype);
3346
3347         sgv_pool_flush(dev->pool_clust);
3348         sgv_pool_flush(dev->pool);
3349
3350         TRACE_MGMT_DBG("Unregistering finished (dev %p)", dev);
3351
3352         dev->cleanup_done = 1;
3353
3354         wake_up(&cleanup_list_waitQ);
3355         wake_up(&dev->cmd_lists.cmd_list_waitQ);
3356
3357         wait_for_completion(&dev->cleanup_cmpl);
3358
3359         sgv_pool_del(dev->pool_clust);
3360         sgv_pool_del(dev->pool);
3361
3362         TRACE_MGMT_DBG("Releasing completed (dev %p)", dev);
3363
3364         module_put(THIS_MODULE);
3365
3366         TRACE_EXIT();
3367         return 0;
3368 }
3369
3370 static int __dev_user_release(void *arg)
3371 {
3372         struct scst_user_dev *dev = (struct scst_user_dev *)arg;
3373         dev_user_exit_dev(dev);
3374         kfree(dev);
3375         return 0;
3376 }
3377
3378 static int dev_user_release(struct inode *inode, struct file *file)
3379 {
3380         struct scst_user_dev *dev;
3381         struct task_struct *t;
3382
3383         TRACE_ENTRY();
3384
3385         dev = (struct scst_user_dev *)file->private_data;
3386         if (dev == NULL)
3387                 goto out;
3388         file->private_data = NULL;
3389
3390         TRACE_MGMT_DBG("Going to release dev %s", dev->name);
3391
3392         t = kthread_run(__dev_user_release, dev, "scst_usr_released");
3393         if (IS_ERR(t)) {
3394                 PRINT_CRIT_ERROR("kthread_run() failed (%ld), releasing device "
3395                         "%p directly. If you have several devices under load "
3396                         "it might deadlock!", PTR_ERR(t), dev);
3397                 __dev_user_release(dev);
3398         }
3399
3400 out:
3401         TRACE_EXIT();
3402         return 0;
3403 }
3404
3405 static int dev_user_process_cleanup(struct scst_user_dev *dev)
3406 {
3407         struct scst_user_cmd *ucmd;
3408         int rc = 0, res = 1;
3409
3410         TRACE_ENTRY();
3411
3412         sBUG_ON(dev->blocking);
3413         wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
3414
3415         while (1) {
3416                 int rc1;
3417
3418                 TRACE_DBG("Cleanuping dev %p", dev);
3419
3420                 rc1 = dev_user_unjam_dev(dev);
3421                 if ((rc1 == 0) && (rc == -EAGAIN) && dev->cleanup_done)
3422                         break;
3423
3424                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
3425
3426                 rc = dev_user_get_next_cmd(dev, &ucmd);
3427                 if (rc == 0)
3428                         dev_user_unjam_cmd(ucmd, 1, NULL);
3429
3430                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
3431
3432                 if (rc == -EAGAIN) {
3433                         if (!dev->cleanup_done) {
3434                                 TRACE_DBG("No more commands (dev %p)", dev);
3435                                 goto out;
3436                         }
3437                 }
3438         }
3439
3440 #ifdef CONFIG_SCST_EXTRACHECKS
3441 {
3442         int i;
3443         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3444                 struct list_head *head = &dev->ucmd_hash[i];
3445                 struct scst_user_cmd *ucmd2;
3446 again:
3447                 list_for_each_entry(ucmd2, head, hash_list_entry) {
3448                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
3449                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
3450                         ucmd_put(ucmd2);
3451                         goto again;
3452                 }
3453         }
3454 }
3455 #endif
3456
3457         TRACE_DBG("Cleanuping done (dev %p)", dev);
3458         complete_all(&dev->cleanup_cmpl);
3459         res = 0;
3460
3461 out:
3462         TRACE_EXIT_RES(res);
3463         return res;
3464 }
3465
3466 #ifndef CONFIG_SCST_PROC
3467
3468 static ssize_t dev_user_sysfs_commands_show(struct kobject *kobj,
3469         struct kobj_attribute *attr, char *buf)
3470 {
3471         int pos = 0, ppos, i;
3472         struct scst_device *dev;
3473         struct scst_user_dev *udev;
3474         unsigned long flags;
3475
3476         TRACE_ENTRY();
3477
3478         dev = container_of(kobj, struct scst_device, dev_kobj);
3479         udev = (struct scst_user_dev *)dev->dh_priv;
3480
3481         spin_lock_irqsave(&udev->cmd_lists.cmd_list_lock, flags);
3482         for (i = 0; i < (int)ARRAY_SIZE(udev->ucmd_hash); i++) {
3483                 struct list_head *head = &udev->ucmd_hash[i];
3484                 struct scst_user_cmd *ucmd;
3485                 list_for_each_entry(ucmd, head, hash_list_entry) {
3486                         ppos = pos;
3487                         pos += scnprintf(&buf[pos],
3488                                 SCST_SYSFS_BLOCK_SIZE - pos,
3489                                 "ucmd %p (state %x, ref %d), "
3490                                 "sent_to_user %d, seen_by_user %d, "
3491                                 "aborted %d, jammed %d, scst_cmd %p\n",
3492                                 ucmd, ucmd->state,
3493                                 atomic_read(&ucmd->ucmd_ref),
3494                                 ucmd->sent_to_user, ucmd->seen_by_user,
3495                                 ucmd->aborted, ucmd->jammed, ucmd->cmd);
3496                         if (pos >= SCST_SYSFS_BLOCK_SIZE-1) {
3497                                 ppos += scnprintf(&buf[ppos],
3498                                         SCST_SYSFS_BLOCK_SIZE - ppos, "...\n");
3499                                 pos = ppos;
3500                                 break;
3501                         }
3502                 }
3503         }
3504         spin_unlock_irqrestore(&udev->cmd_lists.cmd_list_lock, flags);
3505
3506         TRACE_EXIT_RES(pos);
3507         return pos;
3508 }
3509
3510 #else /* CONFIG_SCST_PROC */
3511
3512 /*
3513  * Called when a file in the /proc/scsi_tgt/scst_user is read
3514  */
3515 static int dev_user_read_proc(struct seq_file *seq, struct scst_dev_type *dev_type)
3516 {
3517         int res = 0;
3518         struct scst_user_dev *dev;
3519         unsigned long flags;
3520
3521         TRACE_ENTRY();
3522
3523         spin_lock(&dev_list_lock);
3524
3525         list_for_each_entry(dev, &dev_list, dev_list_entry) {
3526                 int i;
3527                 seq_printf(seq, "Device %s commands:\n", dev->name);
3528                 spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
3529                 for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3530                         struct list_head *head = &dev->ucmd_hash[i];
3531                         struct scst_user_cmd *ucmd;
3532                         list_for_each_entry(ucmd, head, hash_list_entry) {
3533                                 seq_printf(seq, "ucmd %p (state %x, ref %d), "
3534                                         "sent_to_user %d, seen_by_user %d, "
3535                                         "aborted %d, jammed %d, scst_cmd %p\n",
3536                                         ucmd, ucmd->state,
3537                                         atomic_read(&ucmd->ucmd_ref),
3538                                         ucmd->sent_to_user, ucmd->seen_by_user,
3539                                         ucmd->aborted, ucmd->jammed, ucmd->cmd);
3540                         }
3541                 }
3542                 spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
3543         }
3544         spin_unlock(&dev_list_lock);
3545
3546         TRACE_EXIT_RES(res);
3547         return res;
3548 }
3549 #endif /* CONFIG_SCST_PROC */
3550
3551 static inline int test_cleanup_list(void)
3552 {
3553         int res = !list_empty(&cleanup_list) ||
3554                   unlikely(kthread_should_stop());
3555         return res;
3556 }
3557
3558 static int dev_user_cleanup_thread(void *arg)
3559 {
3560         TRACE_ENTRY();
3561
3562         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
3563
3564         current->flags |= PF_NOFREEZE;
3565
3566         spin_lock(&cleanup_lock);
3567         while (!kthread_should_stop()) {
3568                 wait_queue_t wait;
3569                 init_waitqueue_entry(&wait, current);
3570
3571                 if (!test_cleanup_list()) {
3572                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
3573                         for (;;) {
3574                                 set_current_state(TASK_INTERRUPTIBLE);
3575                                 if (test_cleanup_list())
3576                                         break;
3577                                 spin_unlock(&cleanup_lock);
3578                                 schedule();
3579                                 spin_lock(&cleanup_lock);
3580                         }
3581                         set_current_state(TASK_RUNNING);
3582                         remove_wait_queue(&cleanup_list_waitQ, &wait);
3583                 }
3584
3585                 /*
3586                  * We have to poll devices, because commands can go from SCST
3587                  * core on cmd_list_waitQ and we have no practical way to
3588                  * detect them.
3589                  */
3590
3591                 while (1) {
3592                         struct scst_user_dev *dev;
3593                         LIST_HEAD(cl_devs);
3594
3595                         while (!list_empty(&cleanup_list)) {
3596                                 int rc;
3597
3598                                 dev = list_entry(cleanup_list.next,
3599                                         typeof(*dev), cleanup_list_entry);
3600                                 list_del(&dev->cleanup_list_entry);
3601
3602                                 spin_unlock(&cleanup_lock);
3603                                 rc = dev_user_process_cleanup(dev);
3604                                 spin_lock(&cleanup_lock);
3605
3606                                 if (rc != 0)
3607                                         list_add_tail(&dev->cleanup_list_entry,
3608                                                 &cl_devs);
3609                         }
3610
3611                         if (list_empty(&cl_devs))
3612                                 break;
3613
3614                         spin_unlock(&cleanup_lock);
3615                         msleep(100);
3616                         spin_lock(&cleanup_lock);
3617
3618                         while (!list_empty(&cl_devs)) {
3619                                 dev = list_entry(cl_devs.next, typeof(*dev),
3620                                         cleanup_list_entry);
3621                                 list_move_tail(&dev->cleanup_list_entry,
3622                                         &cleanup_list);
3623                         }
3624                 }
3625         }