d820aab259bf1437d81435559b1812be3f5c75d4
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23
24 #define LOG_PREFIX              DEV_USER_NAME
25
26 #include "scst.h"
27 #include "scst_user.h"
28 #include "scst_dev_handler.h"
29
30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
31 #warning "HIGHMEM kernel configurations are not supported by this module, \
32         because nowadays it doesn't worth the effort. Consider changing \
33         VMSPLIT option or use a 64-bit configuration instead. See README file \
34         for details."
35 #endif
36
37 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18))
38 #define COMPLETION_INITIALIZER_ONSTACK(work) \
39         ({ init_completion(&work); work; })
40         
41 /*
42  * Lockdep needs to run a non-constant initializer for on-stack
43  * completions - so we use the _ONSTACK() variant for those that
44  * are on the kernel stack:
45  */
46 #ifdef CONFIG_LOCKDEP
47 # define DECLARE_COMPLETION_ONSTACK(work) \
48         struct completion work = COMPLETION_INITIALIZER_ONSTACK(work)
49 #else
50 # define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work)
51 #endif
52
53 #endif
54
55 #define DEV_USER_MAJOR                  237
56
57 #define DEV_USER_CMD_HASH_ORDER         6
58
59 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
60
61 struct scst_user_dev {
62         struct rw_semaphore dev_rwsem;
63
64         struct scst_cmd_lists cmd_lists;
65
66         /* Protected by cmd_lists.cmd_list_lock */
67         struct list_head ready_cmd_list;
68
69         /* Protected by dev_rwsem or don't need any protection */
70         unsigned int blocking:1;
71         unsigned int cleanup_done:1;
72         unsigned int tst:3;
73         unsigned int queue_alg:4;
74         unsigned int tas:1;
75         unsigned int swp:1;
76         unsigned int has_own_order_mgmt:1;
77
78         int (*generic_parse)(struct scst_cmd *cmd,
79                 int (*get_block)(struct scst_cmd *cmd));
80
81         int block;
82         int def_block;
83
84         struct scst_mem_lim udev_mem_lim;
85         struct sgv_pool *pool;
86
87         uint8_t parse_type;
88         uint8_t on_free_cmd_type;
89         uint8_t memory_reuse_type;
90         uint8_t partial_transfers_type;
91         uint32_t partial_len;
92
93         struct scst_dev_type devtype;
94
95         /* Both protected by cmd_lists.cmd_list_lock */
96         unsigned int handle_counter;
97         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
98
99         struct scst_device *sdev;
100
101         int virt_id;
102         struct list_head dev_list_entry;
103         char name[SCST_MAX_NAME];
104
105         struct list_head cleanup_list_entry;
106         /* ToDo: make it on-stack */
107         struct completion cleanup_cmpl;
108 };
109
110 /* Most fields are unprotected, since only one thread at time can access them */
111 struct scst_user_cmd {
112         struct scst_cmd *cmd;
113         struct scst_user_dev *dev;
114
115         atomic_t ucmd_ref;
116
117         unsigned int buff_cached:1;
118         unsigned int buf_dirty:1;
119         unsigned int background_exec:1;
120         unsigned int aborted:1;
121
122         struct scst_user_cmd *buf_ucmd;
123
124         int cur_data_page;
125         int num_data_pages;
126         int first_page_offset;
127         unsigned long ubuff;
128         struct page **data_pages;
129         struct sgv_pool_obj *sgv;
130
131         /*
132          * Special flags, which can be accessed asynchronously (hence "long").
133          * Protected by cmd_lists.cmd_list_lock.
134          */
135         unsigned long sent_to_user:1;
136         unsigned long jammed:1;
137         unsigned long this_state_unjammed:1;
138         unsigned long seen_by_user:1; /* here only as a small optimization */
139
140         unsigned int state;
141
142         struct list_head ready_cmd_list_entry;
143
144         unsigned int h;
145         struct list_head hash_list_entry;
146
147         struct scst_user_get_cmd user_cmd;
148
149         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
150         union {
151                 struct completion *cmpl;
152                 struct scst_mgmt_cmd *mcmd;
153         };
154         int result;
155 };
156
157 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
158         gfp_t gfp_mask);
159 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
160
161 static int dev_user_parse(struct scst_cmd *cmd);
162 static int dev_user_exec(struct scst_cmd *cmd);
163 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
164 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
165         struct scst_tgt_dev *tgt_dev);
166
167 static int dev_user_disk_done(struct scst_cmd *cmd);
168 static int dev_user_tape_done(struct scst_cmd *cmd);
169
170 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
171         gfp_t gfp_mask, void *priv);
172 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
173                                      void *priv);
174
175 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
176
177 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
178         unsigned long *flags);
179
180 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
181 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
182         int status);
183 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
184 static int dev_user_register_dev(struct file *file,
185         const struct scst_user_dev_desc *dev_desc);
186 static int __dev_user_set_opt(struct scst_user_dev *dev,
187         const struct scst_user_opt *opt);
188 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
189 static int dev_user_get_opt(struct file *file, void __user *arg);
190
191 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
192 static long dev_user_ioctl(struct file *file, unsigned int cmd,
193         unsigned long arg);
194 static int dev_user_release(struct inode *inode, struct file *file);
195
196 /** Data **/
197
198 static struct kmem_cache *user_cmd_cachep;
199 static struct kmem_cache *user_get_cmd_cachep;
200
201 static DEFINE_MUTEX(dev_priv_mutex);
202
203 static struct file_operations dev_user_fops = {
204         .poll           = dev_user_poll,
205         .unlocked_ioctl = dev_user_ioctl,
206 #ifdef CONFIG_COMPAT
207         .compat_ioctl   = dev_user_ioctl,
208 #endif
209         .release        = dev_user_release,
210 };
211
212 static struct class *dev_user_sysfs_class;
213
214 static DEFINE_SPINLOCK(dev_list_lock);
215 static LIST_HEAD(dev_list);
216
217 static DEFINE_SPINLOCK(cleanup_lock);
218 static LIST_HEAD(cleanup_list);
219 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
220 static struct task_struct *cleanup_thread;
221
222 /*
223  * Skip this command if result is not 0. Must be called under
224  * cmd_lists.cmd_list_lock and IRQ off.
225  */
226 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
227 {
228         int r = atomic_inc_return(&ucmd->ucmd_ref);
229         int res;
230         if (unlikely(r == 1)) {
231                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
232                 atomic_dec(&ucmd->ucmd_ref);
233                 res = true;
234                 /*
235                  * Necessary code is serialized by cmd_list_lock in
236                  * cmd_remove_hash()
237                  */
238         } else {
239                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
240                         atomic_read(&ucmd->ucmd_ref));
241                 res = false;
242         }
243         return res;
244 }
245
246 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
247 {
248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
249         atomic_inc(&ucmd->ucmd_ref);
250         if (barrier)
251                 smp_mb__after_atomic_inc();
252 }
253
254 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
255 {
256         __ucmd_get(ucmd, true);
257 }
258
259 static inline void ucmd_get(struct scst_user_cmd *ucmd)
260 {
261         __ucmd_get(ucmd, false);
262 }
263
264 /* Must not be called under cmd_list_lock!! */
265 static inline void ucmd_put(struct scst_user_cmd *ucmd)
266 {
267         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
268
269         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
270
271         if (atomic_dec_and_test(&ucmd->ucmd_ref))
272                 dev_user_free_ucmd(ucmd);
273 }
274
275 static inline int calc_num_pg(unsigned long buf, int len)
276 {
277         len += buf & ~PAGE_MASK;
278         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
279 }
280
281 static inline int is_need_offs_page(unsigned long buf, int len)
282 {
283         return ((buf & ~PAGE_MASK) != 0) &&
284                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
285 }
286
287 static void __dev_user_not_reg(void)
288 {
289         PRINT_ERROR("%s", "Device not registered");
290         return;
291 }
292
293 static inline int dev_user_check_reg(struct scst_user_dev *dev)
294 {
295         if (dev == NULL) {
296                 __dev_user_not_reg();
297                 return -EINVAL;
298         }
299         return 0;
300 }
301
302 static inline int scst_user_cmd_hashfn(int h)
303 {
304         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
305 }
306
307 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
308         unsigned int h)
309 {
310         struct list_head *head;
311         struct scst_user_cmd *ucmd;
312
313         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
314         list_for_each_entry(ucmd, head, hash_list_entry) {
315                 if (ucmd->h == h) {
316                         TRACE_DBG("Found ucmd %p", ucmd);
317                         return ucmd;
318                 }
319         }
320         return NULL;
321 }
322
323 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
324 {
325         struct list_head *head;
326         struct scst_user_dev *dev = ucmd->dev;
327         struct scst_user_cmd *u;
328         unsigned long flags;
329
330         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
331         do {
332                 ucmd->h = dev->handle_counter++;
333                 u = __ucmd_find_hash(dev, ucmd->h);
334         } while (u != NULL);
335         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
336         list_add_tail(&ucmd->hash_list_entry, head);
337         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
338
339         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
340         return;
341 }
342
343 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
344 {
345         unsigned long flags;
346
347         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
348         list_del(&ucmd->hash_list_entry);
349         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
350
351         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
352         return;
353 }
354
355 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
356 {
357         TRACE_ENTRY();
358
359         TRACE_MEM("Freeing ucmd %p", ucmd);
360
361         cmd_remove_hash(ucmd);
362         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
363
364         kmem_cache_free(user_cmd_cachep, ucmd);
365
366         TRACE_EXIT();
367         return;
368 }
369
370 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
371         gfp_t gfp_mask, void *priv)
372 {
373         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
374         int offset = 0;
375
376         TRACE_ENTRY();
377
378         /* *sg supposed to be zeroed */
379
380         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
381                 ucmd->ubuff, ucmd->cur_data_page);
382
383         if (ucmd->cur_data_page == 0) {
384                 TRACE_MEM("ucmd->first_page_offset %d",
385                         ucmd->first_page_offset);
386                 offset = ucmd->first_page_offset;
387                 ucmd_get(ucmd);
388         }
389
390         if (ucmd->cur_data_page >= ucmd->num_data_pages)
391                 goto out;
392
393         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
394                 PAGE_SIZE - offset, offset);
395         ucmd->cur_data_page++;
396
397         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
398                 sg->offset);
399         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
400
401 out:
402         TRACE_EXIT();
403         return sg_page(sg);
404 }
405
406 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
407 {
408         TRACE_ENTRY();
409
410         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
411                 ucmd, ucmd->h, ucmd->ubuff);
412
413         ucmd->user_cmd.cmd_h = ucmd->h;
414         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
415         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
416
417         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
418
419         dev_user_add_to_ready(ucmd);
420
421         TRACE_EXIT();
422         return;
423 }
424
425 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
426 {
427         int i;
428
429         TRACE_ENTRY();
430
431         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
432                 ucmd->ubuff, ucmd->num_data_pages);
433
434         for (i = 0; i < ucmd->num_data_pages; i++) {
435                 struct page *page = ucmd->data_pages[i];
436
437                 if (ucmd->buf_dirty)
438                         SetPageDirty(page);
439
440                 page_cache_release(page);
441         }
442
443         kfree(ucmd->data_pages);
444         ucmd->data_pages = NULL;
445
446         TRACE_EXIT();
447         return;
448 }
449
450 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
451 {
452         TRACE_ENTRY();
453
454         sBUG_ON(ucmd->data_pages == NULL);
455
456         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
457                 ucmd, ucmd->ubuff, ucmd->buff_cached);
458
459         dev_user_unmap_buf(ucmd);
460
461         if (ucmd->buff_cached)
462                 dev_user_on_cached_mem_free(ucmd);
463         else
464                 ucmd_put(ucmd);
465
466         TRACE_EXIT();
467         return;
468 }
469
470 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
471         void *priv)
472 {
473         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
474
475         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
476                 sg_count, ucmd);
477
478         __dev_user_free_sg_entries(ucmd);
479
480         return;
481 }
482
483 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
484 {
485         int mem_reuse_type = ucmd->dev->memory_reuse_type;
486
487         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
488             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
489              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
490             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
491              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
492                 return 1;
493         else
494                 return 0;
495 }
496
497 /*
498  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
499  * Unmaps the buffer, if needed in case of error
500  */
501 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
502 {
503         int res = 0;
504         struct scst_cmd *cmd = ucmd->cmd;
505         struct scst_user_dev *dev = ucmd->dev;
506         gfp_t gfp_mask;
507         int flags = 0;
508         int bufflen = cmd->bufflen;
509         int last_len = 0;
510
511         TRACE_ENTRY();
512
513         sBUG_ON(bufflen == 0);
514
515         gfp_mask = __GFP_NOWARN;
516         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
517
518         if (cached_buff) {
519                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
520                 if (ucmd->ubuff == 0)
521                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
522         } else {
523                 TRACE_MEM("%s", "Not cached buff");
524                 flags |= SCST_POOL_ALLOC_NO_CACHED;
525                 if (ucmd->ubuff == 0) {
526                         res = 1;
527                         goto out;
528                 }
529                 bufflen += ucmd->first_page_offset;
530                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
531                         last_len = bufflen & ~PAGE_MASK;
532                 else
533                         last_len = cmd->bufflen & ~PAGE_MASK;
534                 if (last_len == 0)
535                         last_len = PAGE_SIZE;
536         }
537         ucmd->buff_cached = cached_buff;
538
539         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
540                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
541         if (cmd->sg != NULL) {
542                 struct scst_user_cmd *buf_ucmd =
543                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
544
545                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
546
547                 ucmd->ubuff = buf_ucmd->ubuff;
548                 ucmd->buf_ucmd = buf_ucmd;
549
550                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
551                                    (ucmd != buf_ucmd));
552
553                 if (last_len != 0) {
554                         /* We don't use clustering, so the assignment is safe */
555                         cmd->sg[cmd->sg_cnt-1].length = last_len;
556                 }
557
558                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
559                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
560                         last_len, cmd->sg[cmd->sg_cnt-1].length);
561
562                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
563                         static int ll;
564                         if (ll < 10) {
565                                 PRINT_INFO("Unable to complete command due to "
566                                         "SG IO count limitation (requested %d, "
567                                         "available %d, tgt lim %d)",
568                                         cmd->sg_cnt,
569                                         cmd->tgt_dev->max_sg_cnt,
570                                         cmd->tgt->sg_tablesize);
571                                 ll++;
572                         }
573                         cmd->sg = NULL;
574                         /* sgv will be freed in dev_user_free_sgv() */
575                         res = -1;
576                 }
577         } else {
578                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
579                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
580                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
581                 if (unlikely(cmd->sg_cnt == 0)) {
582                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
583                         sBUG_ON(ucmd->sgv != NULL);
584                         res = -1;
585                 } else {
586                         switch (ucmd->state) {
587                         case UCMD_STATE_BUF_ALLOCING:
588                                 res = 1;
589                                 break;
590                         case UCMD_STATE_EXECING:
591                                 res = -1;
592                                 break;
593                         default:
594                                 sBUG();
595                                 break;
596                         }
597                 }
598         }
599
600 out:
601         TRACE_EXIT_RES(res);
602         return res;
603 }
604
605 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
606 {
607         int rc, res = SCST_CMD_STATE_DEFAULT;
608         struct scst_cmd *cmd = ucmd->cmd;
609
610         TRACE_ENTRY();
611
612         ucmd->state = UCMD_STATE_BUF_ALLOCING;
613         cmd->dh_data_buf_alloced = 1;
614
615         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
616         if (rc == 0)
617                 goto out;
618         else if (rc < 0) {
619                 scst_set_busy(cmd);
620                 res = scst_get_cmd_abnormal_done_state(cmd);
621                 goto out;
622         }
623
624         if ((cmd->data_direction != SCST_DATA_WRITE) &&
625             !scst_is_cmd_local(cmd)) {
626                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
627                 goto out;
628         }
629
630         ucmd->user_cmd.cmd_h = ucmd->h;
631         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
632         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
633         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
634                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
635         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
636         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
637                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
638         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
639         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
640         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
641
642         dev_user_add_to_ready(ucmd);
643
644         res = SCST_CMD_STATE_STOP;
645
646 out:
647         TRACE_EXIT_RES(res);
648         return res;
649 }
650
651 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
652         gfp_t gfp_mask)
653 {
654         struct scst_user_cmd *ucmd = NULL;
655
656         TRACE_ENTRY();
657
658 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
659         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
660         if (ucmd != NULL)
661                 memset(ucmd, 0, sizeof(*ucmd));
662 #else
663         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
664 #endif
665         if (unlikely(ucmd == NULL)) {
666                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
667                         "user cmd (gfp_mask %x)", gfp_mask);
668                 goto out;
669         }
670         ucmd->dev = dev;
671         atomic_set(&ucmd->ucmd_ref, 1);
672
673         cmd_insert_hash(ucmd);
674
675         TRACE_MEM("ucmd %p allocated", ucmd);
676
677 out:
678         TRACE_EXIT_HRES((unsigned long)ucmd);
679         return ucmd;
680 }
681
682 static int dev_user_get_block(struct scst_cmd *cmd)
683 {
684         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
685         /*
686          * No need for locks here, since *_detach() can not be
687          * called, when there are existing commands.
688          */
689         TRACE_EXIT_RES(dev->block);
690         return dev->block;
691 }
692
693 static int dev_user_parse(struct scst_cmd *cmd)
694 {
695         int rc, res = SCST_CMD_STATE_DEFAULT;
696         struct scst_user_cmd *ucmd;
697         int atomic = scst_cmd_atomic(cmd);
698         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
699         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
700
701         TRACE_ENTRY();
702
703         if (cmd->dh_priv == NULL) {
704                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
705                 if (unlikely(ucmd == NULL)) {
706                         if (atomic) {
707                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
708                                 goto out;
709                         } else {
710                                 scst_set_busy(cmd);
711                                 goto out_error;
712                         }
713                 }
714                 ucmd->cmd = cmd;
715                 cmd->dh_priv = ucmd;
716         } else {
717                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
718                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
719         }
720
721         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
722
723         if (ucmd->state != UCMD_STATE_NEW)
724                 goto alloc;
725
726         switch (dev->parse_type) {
727         case SCST_USER_PARSE_STANDARD:
728                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
729                 rc = dev->generic_parse(cmd, dev_user_get_block);
730                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
731                         goto out_invalid;
732                 break;
733
734         case SCST_USER_PARSE_EXCEPTION:
735                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
736                 rc = dev->generic_parse(cmd, dev_user_get_block);
737                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
738                         break;
739                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
740                         TRACE_MEM("Restarting PARSE to thread context "
741                                 "(ucmd %p)", ucmd);
742                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
743                         goto out;
744                 }
745                 /* else go through */
746
747         case SCST_USER_PARSE_CALL:
748                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
749                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
750                 ucmd->user_cmd.cmd_h = ucmd->h;
751                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
752                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
753                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
754                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
755                             sizeof(cmd->cdb)));
756                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
757                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
758                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
759                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
760                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
761                 ucmd->user_cmd.parse_cmd.expected_values_set =
762                                         cmd->expected_values_set;
763                 ucmd->user_cmd.parse_cmd.expected_data_direction =
764                                         cmd->expected_data_direction;
765                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
766                                         cmd->expected_transfer_len;
767                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
768                 ucmd->state = UCMD_STATE_PARSING;
769                 dev_user_add_to_ready(ucmd);
770                 res = SCST_CMD_STATE_STOP;
771                 goto out;
772
773         default:
774                 sBUG();
775                 goto out;
776         }
777
778 alloc:
779         if (cmd->data_direction != SCST_DATA_NONE)
780                 res = dev_user_alloc_space(ucmd);
781
782 out:
783         TRACE_EXIT_RES(res);
784         return res;
785
786 out_invalid:
787         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
788                 cmd->op_flags & SCST_INFO_INVALID);
789         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
790
791 out_error:
792         res = scst_get_cmd_abnormal_done_state(cmd);
793         goto out;
794 }
795
796 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
797 {
798         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
799         unsigned long start = buf_ucmd->ubuff;
800         int i, bufflen = ucmd->cmd->bufflen;
801
802         TRACE_ENTRY();
803
804         if (start == 0)
805                 goto out;
806
807         /*
808          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
809          * faster, since it should be cache hot, while ucmd->buf_ucmd and
810          * buf_ucmd->data_pages are cache cold. But, from other side,
811          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
812          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
813          * data_pages array can lead to less cache misses. So, real numbers are
814          * needed. ToDo.
815          */
816
817         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
818                 struct page *page;
819                 page = buf_ucmd->data_pages[i];
820 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
821                 struct vm_area_struct *vma = find_vma(current->mm, start);
822                 if (vma != NULL)
823                         flush_anon_page(vma, page, start);
824 #endif
825                 flush_dcache_page(page);
826                 start += PAGE_SIZE;
827                 bufflen -= PAGE_SIZE;
828         }
829
830 out:
831         TRACE_EXIT();
832         return;
833 }
834
835 static int dev_user_exec(struct scst_cmd *cmd)
836 {
837         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
838         int res = SCST_EXEC_COMPLETED;
839
840         TRACE_ENTRY();
841
842 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
843        * context to complete the necessary actions, but all we are going to
844        * do in this function is, in fact, atomic, so let's skip this check.
845        */
846         if (scst_cmd_atomic(cmd)) {
847                 TRACE_DBG("%s", "User exec() can not be called in atomic "
848                         "context, rescheduling to the thread");
849                 res = SCST_EXEC_NEED_THREAD;
850                 goto out;
851         }
852 #endif
853
854         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
855                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
856                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
857
858         if (cmd->data_direction == SCST_DATA_WRITE)
859                 dev_user_flush_dcache(ucmd);
860
861         ucmd->user_cmd.cmd_h = ucmd->h;
862         ucmd->user_cmd.subcode = SCST_USER_EXEC;
863         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
864         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
865                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
866                     sizeof(cmd->cdb)));
867         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
868         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
869         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
870         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
871         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
872                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
873                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
874         }
875         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
876         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
877         ucmd->user_cmd.exec_cmd.partial = 0;
878         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
879         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
880
881         ucmd->state = UCMD_STATE_EXECING;
882
883         dev_user_add_to_ready(ucmd);
884
885         TRACE_EXIT_RES(res);
886         return res;
887 }
888
889 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
890 {
891         if (ucmd->sgv != NULL) {
892                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
893                 ucmd->sgv = NULL;
894         } else if (ucmd->data_pages != NULL) {
895                 /* We mapped pages, but for some reason didn't allocate them */
896                 ucmd_get(ucmd);
897                 __dev_user_free_sg_entries(ucmd);
898         }
899         return;
900 }
901
902 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
903 {
904         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
905
906         TRACE_ENTRY();
907
908         if (unlikely(ucmd == NULL))
909                 goto out;
910
911         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
912                 ucmd->buff_cached, ucmd->ubuff);
913
914         ucmd->cmd = NULL;
915         if (cmd->data_direction == SCST_DATA_WRITE && ucmd->buf_ucmd != NULL)
916                 ucmd->buf_ucmd->buf_dirty = 1;
917
918         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
919                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
920                 /* The state assignment must be before freeing sgv! */
921                 goto out_reply;
922         }
923
924         if (unlikely(!ucmd->seen_by_user)) {
925                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
926                 goto out_reply;
927         }
928
929         ucmd->user_cmd.cmd_h = ucmd->h;
930         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
931
932         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
933         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
934         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
935         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
936         ucmd->user_cmd.on_free_cmd.status = cmd->status;
937         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
938
939         ucmd->state = UCMD_STATE_ON_FREEING;
940
941         dev_user_add_to_ready(ucmd);
942
943 out:
944         TRACE_EXIT();
945         return;
946
947 out_reply:
948         dev_user_process_reply_on_free(ucmd);
949         goto out;
950 }
951
952 static void dev_user_set_block(struct scst_cmd *cmd, int block)
953 {
954         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
955         /*
956          * No need for locks here, since *_detach() can not be
957          * called, when there are existing commands.
958          */
959         TRACE_DBG("dev %p, new block %d", dev, block);
960         if (block != 0)
961                 dev->block = block;
962         else
963                 dev->block = dev->def_block;
964         return;
965 }
966
967 static int dev_user_disk_done(struct scst_cmd *cmd)
968 {
969         int res = SCST_CMD_STATE_DEFAULT;
970
971         TRACE_ENTRY();
972
973         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
974
975         TRACE_EXIT_RES(res);
976         return res;
977 }
978
979 static int dev_user_tape_done(struct scst_cmd *cmd)
980 {
981         int res = SCST_CMD_STATE_DEFAULT;
982
983         TRACE_ENTRY();
984
985         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
986
987         TRACE_EXIT_RES(res);
988         return res;
989 }
990
991 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
992 {
993         struct scst_user_dev *dev = ucmd->dev;
994         unsigned long flags;
995         int do_wake;
996
997         TRACE_ENTRY();
998
999         do_wake = (in_interrupt() ||
1000                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
1001         if (ucmd->cmd)
1002                 do_wake |= ucmd->cmd->preprocessing_only;
1003
1004         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1005
1006         ucmd->this_state_unjammed = 0;
1007
1008         if ((ucmd->state == UCMD_STATE_PARSING) ||
1009             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1010                 /*
1011                  * If we don't put such commands in the queue head, then under
1012                  * high load we might delay threads, waiting for memory
1013                  * allocations, for too long and start loosing NOPs, which
1014                  * would lead to consider us by remote initiators as
1015                  * unresponsive and stuck => broken connections, etc. If none
1016                  * of our commands completed in NOP timeout to allow the head
1017                  * commands to go, then we are really overloaded and/or stuck.
1018                  */
1019                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1020                         "cmd list", ucmd, ucmd->state);
1021                 list_add(&ucmd->ready_cmd_list_entry,
1022                         &dev->ready_cmd_list);
1023                 do_wake = 1;
1024         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1025                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1026                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1027                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1028                         "ready cmd list", ucmd, ucmd->state);
1029                 list_add(&ucmd->ready_cmd_list_entry,
1030                         &dev->ready_cmd_list);
1031                 do_wake = 1;
1032         } else if ((ucmd->cmd != NULL) &&
1033            unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1034                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1035                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1036         } else {
1037                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1038                 list_add_tail(&ucmd->ready_cmd_list_entry,
1039                               &dev->ready_cmd_list);
1040         }
1041
1042         if (do_wake) {
1043                 TRACE_DBG("Waking up dev %p", dev);
1044                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1045         }
1046
1047         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1048
1049         TRACE_EXIT();
1050         return;
1051 }
1052
1053 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1054         int num_pg)
1055 {
1056         int res = 0, rc;
1057         int i;
1058         struct task_struct *tsk = current;
1059
1060         TRACE_ENTRY();
1061
1062         if (unlikely(ubuff == 0))
1063                 goto out_nomem;
1064
1065         sBUG_ON(ucmd->data_pages != NULL);
1066
1067         ucmd->num_data_pages = num_pg;
1068
1069         ucmd->data_pages =
1070                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1071                           GFP_KERNEL);
1072         if (ucmd->data_pages == NULL) {
1073                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1074                         "(num_data_pages=%d)", ucmd->num_data_pages);
1075                 res = -ENOMEM;
1076                 goto out_nomem;
1077         }
1078
1079         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1080                 " first_page_offset %d, len %d)", ucmd, ubuff,
1081                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1082                 ucmd->cmd->bufflen);
1083
1084         down_read(&tsk->mm->mmap_sem);
1085         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1086                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1087         up_read(&tsk->mm->mmap_sem);
1088
1089         /* get_user_pages() flushes dcache */
1090
1091         if (rc < ucmd->num_data_pages)
1092                 goto out_unmap;
1093
1094         ucmd->ubuff = ubuff;
1095         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1096
1097 out:
1098         TRACE_EXIT_RES(res);
1099         return res;
1100
1101 out_nomem:
1102         scst_set_busy(ucmd->cmd);
1103         /* go through */
1104
1105 out_err:
1106         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1107         goto out;
1108
1109 out_unmap:
1110         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1111                 ucmd->num_data_pages, rc);
1112         if (rc > 0) {
1113                 for (i = 0; i < rc; i++)
1114                         page_cache_release(ucmd->data_pages[i]);
1115         }
1116         kfree(ucmd->data_pages);
1117         ucmd->data_pages = NULL;
1118         res = -EFAULT;
1119         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1120         goto out_err;
1121 }
1122
1123 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1124         struct scst_user_reply_cmd *reply)
1125 {
1126         int res = 0;
1127         struct scst_cmd *cmd = ucmd->cmd;
1128
1129         TRACE_ENTRY();
1130
1131         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1132
1133         if (likely(reply->alloc_reply.pbuf != 0)) {
1134                 int pages;
1135                 if (ucmd->buff_cached) {
1136                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1137                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1138                                         "page aligned",
1139                                         reply->alloc_reply.pbuf);
1140                                 goto out_hwerr;
1141                         }
1142                         pages = cmd->sg_cnt;
1143                 } else
1144                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1145                                             cmd->bufflen);
1146                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1147         } else {
1148                 scst_set_busy(ucmd->cmd);
1149                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1150         }
1151
1152 out_process:
1153         scst_process_active_cmd(cmd, false);
1154
1155         TRACE_EXIT_RES(res);
1156         return res;
1157
1158 out_hwerr:
1159         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1160         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1161         res = -EINVAL;
1162         goto out_process;
1163 }
1164
1165 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1166         struct scst_user_reply_cmd *reply)
1167 {
1168         int res = 0;
1169         struct scst_user_scsi_cmd_reply_parse *preply =
1170                 &reply->parse_reply;
1171         struct scst_cmd *cmd = ucmd->cmd;
1172
1173         TRACE_ENTRY();
1174
1175         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1176                 goto out_inval;
1177
1178         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1179                      (preply->data_direction != SCST_DATA_READ) &&
1180                      (preply->data_direction != SCST_DATA_NONE)))
1181                 goto out_inval;
1182
1183         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1184                      (preply->bufflen == 0)))
1185                 goto out_inval;
1186
1187         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1188                 goto out_inval;
1189
1190         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1191                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1192                 preply->data_direction, preply->bufflen, preply->data_len,
1193                 reply->alloc_reply.pbuf);
1194
1195         cmd->queue_type = preply->queue_type;
1196         cmd->data_direction = preply->data_direction;
1197         cmd->bufflen = preply->bufflen;
1198         cmd->data_len = preply->data_len;
1199
1200 out_process:
1201         scst_process_active_cmd(cmd, false);
1202
1203         TRACE_EXIT_RES(res);
1204         return res;
1205
1206 out_inval:
1207         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1208                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1209         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1210         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1211         scst_set_cmd_abnormal_done_state(cmd);
1212         res = -EINVAL;
1213         goto out_process;
1214 }
1215
1216 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1217 {
1218         int res = 0;
1219
1220         TRACE_ENTRY();
1221
1222         TRACE_DBG("ON FREE ucmd %p", ucmd);
1223
1224         dev_user_free_sgv(ucmd);
1225         ucmd_put(ucmd);
1226
1227         TRACE_EXIT_RES(res);
1228         return res;
1229 }
1230
1231 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1232 {
1233         int res = 0;
1234
1235         TRACE_ENTRY();
1236
1237         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1238
1239         ucmd_put(ucmd);
1240
1241         TRACE_EXIT_RES(res);
1242         return res;
1243 }
1244
1245 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1246         struct scst_user_reply_cmd *reply)
1247 {
1248         int res = 0;
1249         struct scst_user_scsi_cmd_reply_exec *ereply =
1250                 &reply->exec_reply;
1251         struct scst_cmd *cmd = ucmd->cmd;
1252
1253         TRACE_ENTRY();
1254
1255         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1256                 if (ucmd->background_exec) {
1257                         TRACE_DBG("Background ucmd %p finished", ucmd);
1258                         ucmd_put(ucmd);
1259                         goto out;
1260                 }
1261                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1262                         goto out_inval;
1263                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1264                              (ereply->resp_data_len != 0)))
1265                         goto out_inval;
1266         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1267                 if (unlikely(ucmd->background_exec))
1268                         goto out_inval;
1269                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1270                              (cmd->resp_data_len != 0)))
1271                         goto out_inval;
1272                 /*
1273                  * background_exec assignment must be after ucmd get.
1274                  * Otherwise, due to reorder, in dev_user_process_reply()
1275                  * it is possible that ucmd is destroyed before it "got" here.
1276                  */
1277                 ucmd_get_ordered(ucmd);
1278                 ucmd->background_exec = 1;
1279                 TRACE_DBG("Background ucmd %p", ucmd);
1280                 goto out_compl;
1281         } else
1282                 goto out_inval;
1283
1284         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1285                 ereply->status, ereply->resp_data_len);
1286
1287          if (ereply->resp_data_len != 0) {
1288                 if (ucmd->ubuff == 0) {
1289                         int pages, rc;
1290                         if (unlikely(ereply->pbuf == 0))
1291                                 goto out_busy;
1292                         if (ucmd->buff_cached) {
1293                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1294                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1295                                                 "page aligned", ereply->pbuf);
1296                                         goto out_hwerr;
1297                                 }
1298                                 pages = cmd->sg_cnt;
1299                         } else
1300                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1301                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1302                         if ((rc != 0) || (ucmd->ubuff == 0))
1303                                 goto out_compl;
1304
1305                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1306                         if (unlikely(rc != 0))
1307                                 goto out_busy;
1308                 } else
1309                         dev_user_flush_dcache(ucmd);
1310                 cmd->may_need_dma_sync = 1;
1311                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1312         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1313                 if (ucmd->ubuff == 0)
1314                         cmd->resp_data_len = ereply->resp_data_len;
1315                 else
1316                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1317         }
1318
1319         cmd->status = ereply->status;
1320         if (ereply->sense_len != 0) {
1321                 res = scst_alloc_sense(cmd, 0);
1322                 if (res != 0)
1323                         goto out_compl;
1324                 res = copy_from_user(cmd->sense,
1325                         (void __user *)(unsigned long)ereply->psense_buffer,
1326                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1327                                 (unsigned int)ereply->sense_len));
1328                 if (res < 0) {
1329                         PRINT_ERROR("%s", "Unable to get sense data");
1330                         goto out_hwerr_res_set;
1331                 }
1332         }
1333
1334 out_compl:
1335         cmd->completed = 1;
1336         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1337         /* !! At this point cmd can be already freed !! */
1338
1339 out:
1340         TRACE_EXIT_RES(res);
1341         return res;
1342
1343 out_inval:
1344         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1345                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1346         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1347
1348 out_hwerr:
1349         res = -EINVAL;
1350
1351 out_hwerr_res_set:
1352         if (ucmd->background_exec) {
1353                 ucmd_put(ucmd);
1354                 goto out;
1355         } else {
1356                 scst_set_cmd_error(cmd,
1357                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1358                 goto out_compl;
1359         }
1360
1361 out_busy:
1362         scst_set_busy(cmd);
1363         goto out_compl;
1364 }
1365
1366 static int dev_user_process_reply(struct scst_user_dev *dev,
1367         struct scst_user_reply_cmd *reply)
1368 {
1369         int res = 0;
1370         struct scst_user_cmd *ucmd;
1371         int state;
1372
1373         TRACE_ENTRY();
1374
1375         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1376
1377         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1378         if (unlikely(ucmd == NULL)) {
1379                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1380                 res = -ESRCH;
1381                 goto out_unlock;
1382         }
1383
1384         if (unlikely(ucmd_get_check(ucmd))) {
1385                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1386                 res = -ESRCH;
1387                 goto out_unlock;
1388         }
1389
1390         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1391         smp_mb();
1392         if (ucmd->background_exec) {
1393                 state = UCMD_STATE_EXECING;
1394                 goto unlock_process;
1395         }
1396
1397         if (unlikely(ucmd->this_state_unjammed)) {
1398                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1399                         ucmd);
1400                 goto out_unlock_put;
1401         }
1402
1403         if (unlikely(!ucmd->sent_to_user)) {
1404                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1405                         "state %x", ucmd, ucmd->state);
1406                 res = -EINVAL;
1407                 goto out_unlock_put;
1408         }
1409
1410         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1411                 goto out_wrong_state;
1412
1413         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1414                 goto out_wrong_state;
1415
1416         state = ucmd->state;
1417         ucmd->sent_to_user = 0;
1418
1419 unlock_process:
1420         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1421
1422         switch (state) {
1423         case UCMD_STATE_PARSING:
1424                 res = dev_user_process_reply_parse(ucmd, reply);
1425                 break;
1426
1427         case UCMD_STATE_BUF_ALLOCING:
1428                 res = dev_user_process_reply_alloc(ucmd, reply);
1429                 break;
1430
1431         case UCMD_STATE_EXECING:
1432                 res = dev_user_process_reply_exec(ucmd, reply);
1433                 break;
1434
1435         case UCMD_STATE_ON_FREEING:
1436                 res = dev_user_process_reply_on_free(ucmd);
1437                 break;
1438
1439         case UCMD_STATE_ON_CACHE_FREEING:
1440                 res = dev_user_process_reply_on_cache_free(ucmd);
1441                 break;
1442
1443         case UCMD_STATE_TM_EXECING:
1444                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1445                 break;
1446
1447         case UCMD_STATE_ATTACH_SESS:
1448         case UCMD_STATE_DETACH_SESS:
1449                 res = dev_user_process_reply_sess(ucmd, reply->result);
1450                 break;
1451
1452         default:
1453                 sBUG();
1454                 break;
1455         }
1456
1457 out_put:
1458         ucmd_put(ucmd);
1459
1460 out:
1461         TRACE_EXIT_RES(res);
1462         return res;
1463
1464 out_wrong_state:
1465         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1466                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1467                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1468                 reply->subcode, ucmd->user_cmd.subcode);
1469         res = -EINVAL;
1470         dev_user_unjam_cmd(ucmd, 0, NULL);
1471
1472 out_unlock_put:
1473         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1474         goto out_put;
1475
1476 out_unlock:
1477         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1478         goto out;
1479 }
1480
1481 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1482 {
1483         int res = 0;
1484         struct scst_user_dev *dev;
1485         struct scst_user_reply_cmd reply;
1486
1487         TRACE_ENTRY();
1488
1489         mutex_lock(&dev_priv_mutex);
1490         dev = (struct scst_user_dev *)file->private_data;
1491         res = dev_user_check_reg(dev);
1492         if (unlikely(res != 0)) {
1493                 mutex_unlock(&dev_priv_mutex);
1494                 goto out;
1495         }
1496         down_read(&dev->dev_rwsem);
1497         mutex_unlock(&dev_priv_mutex);
1498
1499         res = copy_from_user(&reply, arg, sizeof(reply));
1500         if (unlikely(res < 0))
1501                 goto out_up;
1502
1503         TRACE_MGMT_DBG("Reply for dev %s", dev->name);
1504
1505         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1506
1507         res = dev_user_process_reply(dev, &reply);
1508         if (unlikely(res < 0))
1509                 goto out_up;
1510
1511 out_up:
1512         up_read(&dev->dev_rwsem);
1513
1514 out:
1515         TRACE_EXIT_RES(res);
1516         return res;
1517 }
1518
1519 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1520         __releases(&dev->cmd_lists.cmd_list_lock)
1521         __acquires(&dev->cmd_lists.cmd_list_lock)
1522 {
1523         int res = 0;
1524
1525         TRACE_ENTRY();
1526
1527         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1528                 struct scst_cmd *cmd = list_entry(
1529                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1530                         cmd_list_entry);
1531                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1532                 list_del(&cmd->cmd_list_entry);
1533                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1534                 scst_process_active_cmd(cmd, false);
1535                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1536                 res++;
1537         }
1538
1539         TRACE_EXIT_RES(res);
1540         return res;
1541 }
1542
1543 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1544 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1545         __releases(&dev->cmd_lists.cmd_list_lock)
1546         __acquires(&dev->cmd_lists.cmd_list_lock)
1547 {
1548         struct scst_user_cmd *u;
1549
1550 again:
1551         u = NULL;
1552         if (!list_empty(cmd_list)) {
1553                 u = list_entry(cmd_list->next, typeof(*u),
1554                                ready_cmd_list_entry);
1555
1556                 TRACE_DBG("Found ready ucmd %p", u);
1557                 list_del(&u->ready_cmd_list_entry);
1558
1559                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1560
1561                 if (u->cmd != NULL) {
1562                         if (u->state == UCMD_STATE_EXECING) {
1563                                 struct scst_user_dev *dev = u->dev;
1564                                 int rc;
1565
1566                                 EXTRACHECKS_BUG_ON(u->jammed);
1567
1568                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1569
1570                                 rc = scst_check_local_events(u->cmd);
1571                                 if (unlikely(rc != 0)) {
1572                                         u->cmd->scst_cmd_done(u->cmd,
1573                                                 SCST_CMD_STATE_DEFAULT,
1574                                                 SCST_CONTEXT_DIRECT);
1575                                         /*
1576                                          * !! At this point cmd & u can be !!
1577                                          * !! already freed                !!
1578                                          */
1579                                         spin_lock_irq(
1580                                                 &dev->cmd_lists.cmd_list_lock);
1581                                         goto again;
1582                                 }
1583
1584                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1585                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1586                                         &u->cmd->cmd_flags))) {
1587                                 switch (u->state) {
1588                                 case UCMD_STATE_PARSING:
1589                                 case UCMD_STATE_BUF_ALLOCING:
1590                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1591                                         dev_user_unjam_cmd(u, 0, NULL);
1592                                         goto again;
1593                                 case UCMD_STATE_EXECING:
1594                                         EXTRACHECKS_BUG_ON(1);
1595                                 }
1596                         }
1597                 }
1598                 u->sent_to_user = 1;
1599                 u->seen_by_user = 1;
1600         }
1601         return u;
1602 }
1603
1604 static inline int test_cmd_lists(struct scst_user_dev *dev)
1605 {
1606         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1607                   !list_empty(&dev->ready_cmd_list) ||
1608                   !dev->blocking || dev->cleanup_done ||
1609                   signal_pending(current);
1610         return res;
1611 }
1612
1613 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1614 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1615         struct scst_user_cmd **ucmd)
1616 {
1617         int res = 0;
1618         wait_queue_t wait;
1619
1620         TRACE_ENTRY();
1621
1622         init_waitqueue_entry(&wait, current);
1623
1624         while (1) {
1625                 if (!test_cmd_lists(dev)) {
1626                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1627                                 &wait);
1628                         for (;;) {
1629                                 set_current_state(TASK_INTERRUPTIBLE);
1630                                 if (test_cmd_lists(dev))
1631                                         break;
1632                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1633                                 schedule();
1634                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1635                         }
1636                         set_current_state(TASK_RUNNING);
1637                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1638                                 &wait);
1639                 }
1640
1641                 dev_user_process_scst_commands(dev);
1642
1643                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1644                 if (*ucmd != NULL)
1645                         break;
1646
1647                 if (!dev->blocking || dev->cleanup_done) {
1648                         res = -EAGAIN;
1649                         TRACE_DBG("No ready commands, returning %d", res);
1650                         break;
1651                 }
1652
1653                 if (signal_pending(current)) {
1654                         res = -EINTR;
1655                         TRACE_DBG("Signal pending, returning %d", res);
1656                         break;
1657                 }
1658         }
1659
1660         TRACE_EXIT_RES(res);
1661         return res;
1662 }
1663
1664 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1665 {
1666         int res = 0;
1667         struct scst_user_dev *dev;
1668         struct scst_user_get_cmd *cmd;
1669         struct scst_user_reply_cmd *reply;
1670         struct scst_user_cmd *ucmd;
1671         uint64_t ureply;
1672
1673         TRACE_ENTRY();
1674
1675         mutex_lock(&dev_priv_mutex);
1676         dev = (struct scst_user_dev *)file->private_data;
1677         res = dev_user_check_reg(dev);
1678         if (unlikely(res != 0)) {
1679                 mutex_unlock(&dev_priv_mutex);
1680                 goto out;
1681         }
1682         down_read(&dev->dev_rwsem);
1683         mutex_unlock(&dev_priv_mutex);
1684
1685         /* get_user() can't be used with 64-bit values on x86_32 */
1686         res = copy_from_user(&ureply, (uint64_t __user *)
1687                 &((struct scst_user_get_cmd *)arg)->preply,
1688                 sizeof(ureply));
1689         if (unlikely(res < 0))
1690                 goto out_up;
1691
1692         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1693                 dev->name);
1694
1695         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1696         if (unlikely(cmd == NULL)) {
1697                 res = -ENOMEM;
1698                 goto out_up;
1699         }
1700
1701         if (ureply != 0) {
1702                 unsigned long u = (unsigned long)ureply;
1703                 reply = (struct scst_user_reply_cmd *)cmd;
1704                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1705                 if (unlikely(res < 0))
1706                         goto out_free;
1707
1708                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1709
1710                 res = dev_user_process_reply(dev, reply);
1711                 if (unlikely(res < 0))
1712                         goto out_free;
1713         }
1714
1715         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1716         res = dev_user_get_next_cmd(dev, &ucmd);
1717         if (res == 0) {
1718                 *cmd = ucmd->user_cmd;
1719                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1720                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1721                 res = copy_to_user(arg, cmd, sizeof(*cmd));
1722         } else
1723                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1724
1725 out_free:
1726         kmem_cache_free(user_get_cmd_cachep, cmd);
1727
1728 out_up:
1729         up_read(&dev->dev_rwsem);
1730
1731 out:
1732         TRACE_EXIT_RES(res);
1733         return res;
1734 }
1735
1736 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1737         unsigned long arg)
1738 {
1739         long res;
1740
1741         TRACE_ENTRY();
1742
1743         switch (cmd) {
1744         case SCST_USER_REPLY_AND_GET_CMD:
1745                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1746                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1747                 break;
1748
1749         case SCST_USER_REPLY_CMD:
1750                 TRACE_DBG("%s", "REPLY_CMD");
1751                 res = dev_user_reply_cmd(file, (void __user *)arg);
1752                 break;
1753
1754         case SCST_USER_REGISTER_DEVICE:
1755         {
1756                 struct scst_user_dev_desc *dev_desc;
1757                 TRACE_DBG("%s", "REGISTER_DEVICE");
1758                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1759                 if (dev_desc == NULL) {
1760                         res = -ENOMEM;
1761                         goto out;
1762                 }
1763                 res = copy_from_user(dev_desc, (void __user *)arg,
1764                                      sizeof(*dev_desc));
1765                 if (res < 0) {
1766                         kfree(dev_desc);
1767                         goto out;
1768                 }
1769                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1770                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1771                 res = dev_user_register_dev(file, dev_desc);
1772                 kfree(dev_desc);
1773                 break;
1774         }
1775
1776         case SCST_USER_SET_OPTIONS:
1777         {
1778                 struct scst_user_opt opt;
1779                 TRACE_DBG("%s", "SET_OPTIONS");
1780                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1781                 if (res < 0)
1782                         goto out;
1783                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1784                 res = dev_user_set_opt(file, &opt);
1785                 break;
1786         }
1787
1788         case SCST_USER_GET_OPTIONS:
1789                 TRACE_DBG("%s", "GET_OPTIONS");
1790                 res = dev_user_get_opt(file, (void __user *)arg);
1791                 break;
1792
1793         default:
1794                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1795                 res = -EINVAL;
1796                 goto out;
1797         }
1798
1799 out:
1800         TRACE_EXIT_RES(res);
1801         return res;
1802 }
1803
1804 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1805 {
1806         int res = 0;
1807         struct scst_user_dev *dev;
1808
1809         TRACE_ENTRY();
1810
1811         mutex_lock(&dev_priv_mutex);
1812         dev = (struct scst_user_dev *)file->private_data;
1813         res = dev_user_check_reg(dev);
1814         if (unlikely(res != 0)) {
1815                 mutex_unlock(&dev_priv_mutex);
1816                 goto out;
1817         }
1818         down_read(&dev->dev_rwsem);
1819         mutex_unlock(&dev_priv_mutex);
1820
1821         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1822
1823         if (!list_empty(&dev->ready_cmd_list) ||
1824             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1825                 res |= POLLIN | POLLRDNORM;
1826                 goto out_unlock;
1827         }
1828
1829         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1830
1831         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
1832         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1833         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
1834
1835         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1836
1837         if (!list_empty(&dev->ready_cmd_list) ||
1838             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1839                 res |= POLLIN | POLLRDNORM;
1840                 goto out_unlock;
1841         }
1842
1843 out_unlock:
1844         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1845
1846         up_read(&dev->dev_rwsem);
1847
1848 out:
1849         TRACE_EXIT_HRES(res);
1850         return res;
1851 }
1852
1853 /*
1854  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
1855  */
1856 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1857         unsigned long *flags)
1858         __releases(&dev->cmd_lists.cmd_list_lock)
1859         __acquires(&dev->cmd_lists.cmd_list_lock)
1860 {
1861         int state = ucmd->state;
1862         struct scst_user_dev *dev = ucmd->dev;
1863
1864         TRACE_ENTRY();
1865
1866         if (ucmd->this_state_unjammed)
1867                 goto out;
1868
1869         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1870                 state);
1871
1872         ucmd->jammed = 1;
1873         ucmd->this_state_unjammed = 1;
1874         ucmd->sent_to_user = 0;
1875
1876         switch (state) {
1877         case UCMD_STATE_PARSING:
1878         case UCMD_STATE_BUF_ALLOCING:
1879                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1880                         ucmd->aborted = 1;
1881                 else {
1882                         if (busy)
1883                                 scst_set_busy(ucmd->cmd);
1884                         else
1885                                 scst_set_cmd_error(ucmd->cmd,
1886                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
1887                 }
1888                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1889
1890                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1891                 list_add(&ucmd->cmd->cmd_list_entry,
1892                         &ucmd->cmd->cmd_lists->active_cmd_list);
1893                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1894                 break;
1895
1896         case UCMD_STATE_EXECING:
1897                 if (flags != NULL)
1898                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
1899                                                *flags);
1900                 else
1901                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1902
1903                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1904
1905                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1906                         ucmd->aborted = 1;
1907                 else {
1908                         if (busy)
1909                                 scst_set_busy(ucmd->cmd);
1910                         else
1911                                 scst_set_cmd_error(ucmd->cmd,
1912                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
1913                 }
1914
1915                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
1916                         SCST_CONTEXT_DIRECT);
1917                 /* !! At this point cmd and ucmd can be already freed !! */
1918
1919                 if (flags != NULL)
1920                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
1921                                           *flags);
1922                 else
1923                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1924                 break;
1925
1926         case UCMD_STATE_ON_FREEING:
1927         case UCMD_STATE_ON_CACHE_FREEING:
1928         case UCMD_STATE_TM_EXECING:
1929         case UCMD_STATE_ATTACH_SESS:
1930         case UCMD_STATE_DETACH_SESS:
1931         {
1932                 if (flags != NULL)
1933                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
1934                                                *flags);
1935                 else
1936                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1937
1938                 switch (state) {
1939                 case UCMD_STATE_ON_FREEING:
1940                         dev_user_process_reply_on_free(ucmd);
1941                         break;
1942
1943                 case UCMD_STATE_ON_CACHE_FREEING:
1944                         dev_user_process_reply_on_cache_free(ucmd);
1945                         break;
1946
1947                 case UCMD_STATE_TM_EXECING:
1948                         dev_user_process_reply_tm_exec(ucmd,
1949                                                        SCST_MGMT_STATUS_FAILED);
1950                         break;
1951
1952                 case UCMD_STATE_ATTACH_SESS:
1953                 case UCMD_STATE_DETACH_SESS:
1954                         dev_user_process_reply_sess(ucmd, -EFAULT);
1955                         break;
1956                 }
1957
1958                 if (flags != NULL)
1959                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
1960                                           *flags);
1961                 else
1962                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1963                 break;
1964         }
1965
1966         default:
1967                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1968                 sBUG();
1969                 break;
1970         }
1971
1972 out:
1973         TRACE_EXIT();
1974         return;
1975 }
1976
1977 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1978         __releases(&dev->cmd_lists.cmd_list_lock)
1979         __acquires(&dev->cmd_lists.cmd_list_lock)
1980 {
1981         int i;
1982         struct scst_user_cmd *ucmd;
1983
1984         TRACE_ENTRY();
1985
1986         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1987
1988         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1989
1990 repeat:
1991         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1992                 struct list_head *head = &dev->ucmd_hash[i];
1993
1994                 list_for_each_entry(ucmd, head, hash_list_entry) {
1995                         if (!ucmd->sent_to_user)
1996                                 continue;
1997
1998                         if (ucmd_get_check(ucmd))
1999                                 continue;
2000
2001                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2002                                 ucmd->state, ucmd->cmd);
2003
2004                         dev_user_unjam_cmd(ucmd, 0, NULL);
2005
2006                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2007                         ucmd_put(ucmd);
2008                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2009
2010                         goto repeat;
2011                 }
2012         }
2013
2014         if (dev_user_process_scst_commands(dev) != 0)
2015                 goto repeat;
2016
2017         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2018
2019         TRACE_EXIT();
2020         return;
2021 }
2022
2023 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2024         int status)
2025 {
2026         int res = 0;
2027
2028         TRACE_ENTRY();
2029
2030         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2031                 ucmd->user_cmd.tm_cmd.fn, status);
2032
2033         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2034                 /*
2035                  * It is possible that user space seen TM cmd before cmd
2036                  * to abort or will never see it at all, because it was
2037                  * aborted on the way there. So, it is safe to return
2038                  * success instead, because, if there is the TM cmd at this
2039                  * point, then the cmd to abort apparrently does exist.
2040                  */
2041                 status = SCST_MGMT_STATUS_SUCCESS;
2042         }
2043
2044         scst_async_mcmd_completed(ucmd->mcmd, status);
2045
2046         ucmd_put(ucmd);
2047
2048         TRACE_EXIT_RES(res);
2049         return res;
2050 }
2051
2052 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2053 {
2054         struct scst_user_cmd *ucmd;
2055         unsigned long flags;
2056
2057         TRACE_ENTRY();
2058
2059         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2060 again:
2061         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2062                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2063                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2064                         switch (ucmd->state) {
2065                         case UCMD_STATE_PARSING:
2066                         case UCMD_STATE_BUF_ALLOCING:
2067                         case UCMD_STATE_EXECING:
2068                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2069                                 list_del(&ucmd->ready_cmd_list_entry);
2070                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2071                                 goto again;
2072                         }
2073                 }
2074         }
2075
2076         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2077
2078         TRACE_EXIT();
2079         return;
2080 }
2081
2082 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2083         struct scst_tgt_dev *tgt_dev)
2084 {
2085         struct scst_user_cmd *ucmd;
2086         struct scst_user_dev *dev =
2087                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2088         struct scst_user_cmd *ucmd_to_abort = NULL;
2089
2090         TRACE_ENTRY();
2091
2092         /*
2093          * In the used approach we don't do anything with hung devices, which
2094          * stopped responding and/or have stuck commands. We forcedly abort such
2095          * commands only if they not yet sent to the user space or if the device
2096          * is getting unloaded, e.g. if its handler program gets killed. This is
2097          * because it's pretty hard to distinguish between stuck and temporary
2098          * overloaded states of the device. There are several reasons for that:
2099          *
2100          * 1. Some commands need a lot of time to complete (several hours),
2101          *    so for an impatient user such command(s) will always look as
2102          *    stuck.
2103          *
2104          * 2. If we forcedly abort, i.e. abort before it's actually completed
2105          *    in the user space, just one command, we will have to put the whole
2106          *    device offline until we are sure that no more previously aborted
2107          *    commands will get executed. Otherwise, we might have a possibility
2108          *    for data corruption, when aborted and reported as completed
2109          *    command actually gets executed *after* new commands sent
2110          *    after the force abort was done. Many journaling file systems and
2111          *    databases use "provide required commands order via queue draining"
2112          *    approach and not putting the whole device offline after the forced
2113          *    abort will break it. This makes our decision, if a command stuck
2114          *    or not, cost a lot.
2115          *
2116          * So, we leave policy definition if a device stuck or not to
2117          * the user space and simply let all commands live until they are
2118          * completed or their devices get closed/killed. This approach is very
2119          * much OK, but can affect management commands, which need activity
2120          * suspending via scst_suspend_activity() function such as devices or
2121          * targets registration/removal. But during normal life such commands
2122          * should be rare. Plus, when possible, scst_suspend_activity() will
2123          * return after timeout EBUSY status to allow caller to not stuck
2124          * forever as well.
2125          *
2126          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2127          * stuck commands would affect only related devices.
2128          */
2129
2130         dev_user_abort_ready_commands(dev);
2131
2132         /* We can't afford missing TM command due to memory shortage */
2133         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2134
2135         ucmd->user_cmd.cmd_h = ucmd->h;
2136         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2137         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2138         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2139         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2140         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2141
2142         if (mcmd->cmd_to_abort != NULL) {
2143                 ucmd_to_abort =
2144                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2145                 if (ucmd_to_abort != NULL)
2146                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2147         }
2148
2149         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2150                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2151                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2152                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2153
2154         ucmd->mcmd = mcmd;
2155         ucmd->state = UCMD_STATE_TM_EXECING;
2156
2157         scst_prepare_async_mcmd(mcmd);
2158
2159         dev_user_add_to_ready(ucmd);
2160
2161         TRACE_EXIT();
2162         return SCST_DEV_TM_NOT_COMPLETED;
2163 }
2164
2165 static int dev_user_attach(struct scst_device *sdev)
2166 {
2167         int res = 0;
2168         struct scst_user_dev *dev = NULL, *d;
2169
2170         TRACE_ENTRY();
2171
2172         spin_lock(&dev_list_lock);
2173         list_for_each_entry(d, &dev_list, dev_list_entry) {
2174                 if (strcmp(d->name, sdev->virt_name) == 0) {
2175                         dev = d;
2176                         break;
2177                 }
2178         }
2179         spin_unlock(&dev_list_lock);
2180         if (dev == NULL) {
2181                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2182                 res = -EINVAL;
2183                 goto out;
2184         }
2185
2186         sdev->p_cmd_lists = &dev->cmd_lists;
2187         sdev->dh_priv = dev;
2188         sdev->tst = dev->tst;
2189         sdev->queue_alg = dev->queue_alg;
2190         sdev->swp = dev->swp;
2191         sdev->tas = dev->tas;
2192         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2193
2194         dev->sdev = sdev;
2195
2196         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2197                 dev->name);
2198
2199 out:
2200         TRACE_EXIT();
2201         return res;
2202 }
2203
2204 static void dev_user_detach(struct scst_device *sdev)
2205 {
2206         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2207
2208         TRACE_ENTRY();
2209
2210         TRACE_DBG("virt_id %d", sdev->virt_id);
2211
2212         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2213                 dev->name);
2214
2215         /* dev will be freed by the caller */
2216         sdev->dh_priv = NULL;
2217         dev->sdev = NULL;
2218
2219         TRACE_EXIT();
2220         return;
2221 }
2222
2223 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2224 {
2225         int res = 0;
2226         unsigned long flags;
2227
2228         TRACE_ENTRY();
2229
2230         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2231
2232         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2233
2234         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2235                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2236                 ucmd->result = status;
2237         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2238                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2239         } else
2240                 sBUG();
2241
2242         if (ucmd->cmpl != NULL)
2243                 complete_all(ucmd->cmpl);
2244
2245         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2246
2247         ucmd_put(ucmd);
2248
2249         TRACE_EXIT_RES(res);
2250         return res;
2251 }
2252
2253 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2254 {
2255         struct scst_user_dev *dev =
2256                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2257         int res = 0, rc;
2258         struct scst_user_cmd *ucmd;
2259         DECLARE_COMPLETION_ONSTACK(cmpl);
2260
2261         TRACE_ENTRY();
2262
2263         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2264         if (ucmd == NULL)
2265                 goto out_nomem;
2266
2267         ucmd->cmpl = &cmpl;
2268
2269         ucmd->user_cmd.cmd_h = ucmd->h;
2270         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2271         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2272         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2273         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2274         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2275         strncpy(ucmd->user_cmd.sess.initiator_name,
2276                 tgt_dev->sess->initiator_name,
2277                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2278         ucmd->user_cmd.sess.initiator_name[
2279                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2280
2281         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2282                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2283                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2284                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2285                 ucmd->user_cmd.sess.initiator_name);
2286
2287         ucmd->state = UCMD_STATE_ATTACH_SESS;
2288
2289         ucmd_get(ucmd);
2290
2291         dev_user_add_to_ready(ucmd);
2292
2293         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2294         if (rc > 0)
2295                 res = ucmd->result;
2296         else {
2297                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2298                 res = -EFAULT;
2299         }
2300
2301         sBUG_ON(irqs_disabled());
2302
2303         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2304         ucmd->cmpl = NULL;
2305         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2306
2307         ucmd_put(ucmd);
2308
2309 out:
2310         TRACE_EXIT_RES(res);
2311         return res;
2312
2313 out_nomem:
2314         res = -ENOMEM;
2315         goto out;
2316 }
2317
2318 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2319 {
2320         struct scst_user_dev *dev =
2321                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2322         struct scst_user_cmd *ucmd;
2323
2324         TRACE_ENTRY();
2325
2326         /*
2327          * We can't miss TM command due to memory shortage, because it might
2328          * lead to a memory leak in the user space handler.
2329          */
2330         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2331         if (ucmd == NULL)
2332                 goto out;
2333
2334         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2335                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2336
2337         ucmd->user_cmd.cmd_h = ucmd->h;
2338         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2339         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2340
2341         ucmd->state = UCMD_STATE_DETACH_SESS;
2342
2343         dev_user_add_to_ready(ucmd);
2344
2345 out:
2346         TRACE_EXIT();
2347         return;
2348 }
2349
2350 /* No locks are needed, but the activity must be suspended */
2351 static void dev_user_setup_functions(struct scst_user_dev *dev)
2352 {
2353         TRACE_ENTRY();
2354
2355         dev->devtype.parse = dev_user_parse;
2356         dev->devtype.dev_done = NULL;
2357
2358         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2359                 switch (dev->devtype.type) {
2360                 case TYPE_DISK:
2361                         dev->generic_parse = scst_sbc_generic_parse;
2362                         dev->devtype.dev_done = dev_user_disk_done;
2363                         break;
2364
2365                 case TYPE_TAPE:
2366                         dev->generic_parse = scst_tape_generic_parse;
2367                         dev->devtype.dev_done = dev_user_tape_done;
2368                         break;
2369
2370                 case TYPE_MOD:
2371                         dev->generic_parse = scst_modisk_generic_parse;
2372                         dev->devtype.dev_done = dev_user_disk_done;
2373                         break;
2374
2375                 case TYPE_ROM:
2376                         dev->generic_parse = scst_cdrom_generic_parse;
2377                         dev->devtype.dev_done = dev_user_disk_done;
2378                         break;
2379
2380                 case TYPE_MEDIUM_CHANGER:
2381                         dev->generic_parse = scst_changer_generic_parse;
2382                         break;
2383
2384                 case TYPE_PROCESSOR:
2385                         dev->generic_parse = scst_processor_generic_parse;
2386                         break;
2387
2388                 case TYPE_RAID:
2389                         dev->generic_parse = scst_raid_generic_parse;
2390                         break;
2391
2392                 default:
2393                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2394                                 "for it", dev->devtype.type);
2395                         dev->parse_type = SCST_USER_PARSE_CALL;
2396                         break;
2397                 }
2398         } else {
2399                 dev->generic_parse = NULL;
2400                 dev->devtype.dev_done = NULL;
2401         }
2402
2403         TRACE_EXIT();
2404         return;
2405 }
2406
2407 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2408 {
2409         char ver[sizeof(DEV_USER_VERSION)+1];
2410         int res;
2411
2412         res = copy_from_user(ver,
2413                         (void __user *)(unsigned long)dev_desc->version_str,
2414                         sizeof(ver));
2415         if (res < 0) {
2416                 PRINT_ERROR("%s", "Unable to get version string");
2417                 goto out;
2418         }
2419         ver[sizeof(ver)-1] = '\0';
2420
2421         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2422                 /* ->name already 0-terminated in dev_user_ioctl() */
2423                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2424                         dev_desc->name, ver);
2425                 res = -EINVAL;
2426                 goto out;
2427         }
2428
2429 out:
2430         return res;
2431 }
2432
2433 static int dev_user_register_dev(struct file *file,
2434         const struct scst_user_dev_desc *dev_desc)
2435 {
2436         int res = -ENOMEM, i;
2437         struct scst_user_dev *dev, *d;
2438         int block;
2439
2440         TRACE_ENTRY();
2441
2442         res = dev_user_check_version(dev_desc);
2443         if (res != 0)
2444                 goto out;
2445
2446         switch (dev_desc->type) {
2447         case TYPE_DISK:
2448         case TYPE_ROM:
2449         case TYPE_MOD:
2450                 if (dev_desc->block_size == 0) {
2451                         PRINT_ERROR("Wrong block size %d",
2452                                     dev_desc->block_size);
2453                         res = -EINVAL;
2454                         goto out;
2455                 }
2456                 block = scst_calc_block_shift(dev_desc->block_size);
2457                 if (block == -1) {
2458                         res = -EINVAL;
2459                         goto out;
2460                 }
2461                 break;
2462         default:
2463                 block = dev_desc->block_size;
2464                 break;
2465         }
2466
2467         if (!try_module_get(THIS_MODULE)) {
2468                 PRINT_ERROR("%s", "Fail to get module");
2469                 goto out;
2470         }
2471
2472         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2473         if (dev == NULL)
2474                 goto out_put;
2475
2476         init_rwsem(&dev->dev_rwsem);
2477         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2478         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2479         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2480         INIT_LIST_HEAD(&dev->ready_cmd_list);
2481         if (file->f_flags & O_NONBLOCK) {
2482                 TRACE_DBG("%s", "Non-blocking operations");
2483                 dev->blocking = 0;
2484         } else
2485                 dev->blocking = 1;
2486         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2487                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2488
2489         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2490         dev->name[sizeof(dev->name)-1] = '\0';
2491
2492         scst_init_mem_lim(&dev->udev_mem_lim);
2493
2494         /*
2495          * We don't use clustered pool, since it implies pages reordering,
2496          * which isn't possible with user space supplied buffers. Although
2497          * it's still possible to cluster pages by the tail of each other,
2498          * seems it doesn't worth the effort.
2499          */
2500         dev->pool = sgv_pool_create(dev->name, 0);
2501         if (dev->pool == NULL)
2502                 goto out_put;
2503         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2504                 dev_user_free_sg_entries);
2505
2506         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2507                 dev->name);
2508         dev->devtype.type = dev_desc->type;
2509         dev->devtype.threads_num = -1;
2510         dev->devtype.parse_atomic = 1;
2511         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2512         dev->devtype.dev_done_atomic = 1;
2513         dev->devtype.no_proc = 1;
2514         dev->devtype.attach = dev_user_attach;
2515         dev->devtype.detach = dev_user_detach;
2516         dev->devtype.attach_tgt = dev_user_attach_tgt;
2517         dev->devtype.detach_tgt = dev_user_detach_tgt;
2518         dev->devtype.exec = dev_user_exec;
2519         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2520         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2521
2522         init_completion(&dev->cleanup_cmpl);
2523         dev->block = block;
2524         dev->def_block = block;
2525
2526         res = __dev_user_set_opt(dev, &dev_desc->opt);
2527
2528         TRACE_MEM("dev %p, name %s", dev, dev->name);
2529
2530         spin_lock(&dev_list_lock);
2531
2532         list_for_each_entry(d, &dev_list, dev_list_entry) {
2533                 if (strcmp(d->name, dev->name) == 0) {
2534                         PRINT_ERROR("Device %s already exist",
2535                                 dev->name);
2536                         res = -EEXIST;
2537                         spin_unlock(&dev_list_lock);
2538                         goto out_free;
2539                 }
2540         }
2541
2542         list_add_tail(&dev->dev_list_entry, &dev_list);
2543
2544         spin_unlock(&dev_list_lock);
2545
2546         if (res != 0)
2547                 goto out_del_free;
2548
2549         res = scst_register_virtual_dev_driver(&dev->devtype);
2550         if (res < 0)
2551                 goto out_del_free;
2552
2553         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2554         if (dev->virt_id < 0) {
2555                 res = dev->virt_id;
2556                 goto out_unreg_handler;
2557         }
2558
2559         mutex_lock(&dev_priv_mutex);
2560         if (file->private_data != NULL) {
2561                 mutex_unlock(&dev_priv_mutex);
2562                 PRINT_ERROR("%s", "Device already registered");
2563                 res = -EINVAL;
2564                 goto out_unreg_drv;
2565         }
2566         file->private_data = dev;
2567         mutex_unlock(&dev_priv_mutex);
2568
2569 out:
2570         TRACE_EXIT_RES(res);
2571         return res;
2572
2573 out_unreg_drv:
2574         scst_unregister_virtual_device(dev->virt_id);
2575
2576 out_unreg_handler:
2577         scst_unregister_virtual_dev_driver(&dev->devtype);
2578
2579 out_del_free:
2580         spin_lock(&dev_list_lock);
2581         list_del(&dev->dev_list_entry);
2582         spin_unlock(&dev_list_lock);
2583
2584 out_free:
2585         sgv_pool_destroy(dev->pool);
2586         kfree(dev);
2587         goto out_put;
2588
2589 out_put:
2590         module_put(THIS_MODULE);
2591         goto out;
2592 }
2593
2594 static int __dev_user_set_opt(struct scst_user_dev *dev,
2595         const struct scst_user_opt *opt)
2596 {
2597         int res = 0;
2598
2599         TRACE_ENTRY();
2600
2601         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2602                 "memory_reuse_type %x, partial_transfers_type %x, "
2603                 "partial_len %d", dev->name, opt->parse_type,
2604                 opt->on_free_cmd_type, opt->memory_reuse_type,
2605                 opt->partial_transfers_type, opt->partial_len);
2606
2607         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
2608             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
2609             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
2610             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
2611                 PRINT_ERROR("%s", "Invalid option");
2612                 res = -EINVAL;
2613                 goto out;
2614         }
2615
2616         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2617              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2618             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2619              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2620             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2621                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
2622                         " tas %x, has_own_order_mgmt %x)", opt->tst,
2623                         opt->queue_alg, opt->swp, opt->tas,
2624                         opt->has_own_order_mgmt);
2625                 res = -EINVAL;
2626                 goto out;
2627         }
2628
2629         dev->parse_type = opt->parse_type;
2630         dev->on_free_cmd_type = opt->on_free_cmd_type;
2631         dev->memory_reuse_type = opt->memory_reuse_type;
2632         dev->partial_transfers_type = opt->partial_transfers_type;
2633         dev->partial_len = opt->partial_len;
2634
2635         dev->tst = opt->tst;
2636         dev->queue_alg = opt->queue_alg;
2637         dev->swp = opt->swp;
2638         dev->tas = opt->tas;
2639         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2640         if (dev->sdev != NULL) {
2641                 dev->sdev->tst = opt->tst;
2642                 dev->sdev->queue_alg = opt->queue_alg;
2643                 dev->sdev->swp = opt->swp;
2644                 dev->sdev->tas = opt->tas;
2645                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2646         }
2647
2648         dev_user_setup_functions(dev);
2649
2650 out:
2651         TRACE_EXIT_RES(res);
2652         return res;
2653 }
2654
2655 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2656 {
2657         int res = 0;
2658         struct scst_user_dev *dev;
2659
2660         TRACE_ENTRY();
2661
2662         mutex_lock(&dev_priv_mutex);
2663         dev = (struct scst_user_dev *)file->private_data;
2664         res = dev_user_check_reg(dev);
2665         if (res != 0) {
2666                 mutex_unlock(&dev_priv_mutex);
2667                 goto out;
2668         }
2669         down_write(&dev->dev_rwsem);
2670         mutex_unlock(&dev_priv_mutex);
2671
2672         res = scst_suspend_activity(true);
2673         if (res != 0)
2674                 goto out;
2675
2676         res = __dev_user_set_opt(dev, opt);
2677
2678         scst_resume_activity();
2679
2680         up_write(&dev->dev_rwsem);
2681
2682 out:
2683         TRACE_EXIT_RES(res);
2684         return res;
2685 }
2686
2687 static int dev_user_get_opt(struct file *file, void __user *arg)
2688 {
2689         int res = 0;
2690         struct scst_user_dev *dev;
2691         struct scst_user_opt opt;
2692
2693         TRACE_ENTRY();
2694
2695         mutex_lock(&dev_priv_mutex);
2696         dev = (struct scst_user_dev *)file->private_data;
2697         res = dev_user_check_reg(dev);
2698         if (res != 0) {
2699                 mutex_unlock(&dev_priv_mutex);
2700                 goto out;
2701         }
2702         down_read(&dev->dev_rwsem);
2703         mutex_unlock(&dev_priv_mutex);
2704
2705         opt.parse_type = dev->parse_type;
2706         opt.on_free_cmd_type = dev->on_free_cmd_type;
2707         opt.memory_reuse_type = dev->memory_reuse_type;
2708         opt.partial_transfers_type = dev->partial_transfers_type;
2709         opt.partial_len = dev->partial_len;
2710         opt.tst = dev->tst;
2711         opt.queue_alg = dev->queue_alg;
2712         opt.tas = dev->tas;
2713         opt.swp = dev->swp;
2714         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2715
2716         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2717                 "memory_reuse_type %x, partial_transfers_type %x, "
2718                 "partial_len %d", dev->name, opt.parse_type,
2719                 opt.on_free_cmd_type, opt.memory_reuse_type,
2720                 opt.partial_transfers_type, opt.partial_len);
2721
2722         res = copy_to_user(arg, &opt, sizeof(opt));
2723
2724         up_read(&dev->dev_rwsem);
2725 out:
2726         TRACE_EXIT_RES(res);
2727         return res;
2728 }
2729
2730 static int dev_usr_parse(struct scst_cmd *cmd)
2731 {
2732         sBUG();
2733         return SCST_CMD_STATE_DEFAULT;
2734 }
2735
2736 /* Needed only for /proc support */
2737 #define USR_TYPE {                      \
2738         .name =         DEV_USER_NAME,  \
2739         .type =         -1,             \
2740         .parse =        dev_usr_parse,  \
2741 }
2742
2743 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2744
2745 static int dev_user_release(struct inode *inode, struct file *file)
2746 {
2747         int res = 0;
2748         struct scst_user_dev *dev;
2749
2750         TRACE_ENTRY();
2751
2752         mutex_lock(&dev_priv_mutex);
2753         dev = (struct scst_user_dev *)file->private_data;
2754         if (dev == NULL) {
2755                 mutex_unlock(&dev_priv_mutex);
2756                 goto out;
2757         }
2758         file->private_data = NULL;
2759
2760         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2761
2762         spin_lock(&dev_list_lock);
2763         list_del(&dev->dev_list_entry);
2764         spin_unlock(&dev_list_lock);
2765
2766         mutex_unlock(&dev_priv_mutex);
2767
2768         down_write(&dev->dev_rwsem);
2769
2770         spin_lock(&cleanup_lock);
2771         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2772         spin_unlock(&cleanup_lock);
2773
2774         wake_up(&cleanup_list_waitQ);
2775         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2776
2777         scst_unregister_virtual_device(dev->virt_id);
2778         scst_unregister_virtual_dev_driver(&dev->devtype);
2779
2780         sgv_pool_destroy(dev->pool);
2781
2782         TRACE_DBG("Unregistering finished (dev %p)", dev);
2783
2784         dev->cleanup_done = 1;
2785
2786         wake_up(&cleanup_list_waitQ);
2787         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2788
2789         wait_for_completion(&dev->cleanup_cmpl);
2790
2791         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2792
2793         TRACE_DBG("Releasing completed (dev %p)", dev);
2794
2795         kfree(dev);
2796
2797         module_put(THIS_MODULE);
2798
2799 out:
2800         TRACE_EXIT_RES(res);
2801         return res;
2802 }
2803
2804 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2805 {
2806         struct scst_user_cmd *ucmd;
2807         int rc, res = 1;
2808
2809         TRACE_ENTRY();
2810
2811         dev->blocking = 0;
2812
2813         while (1) {
2814                 TRACE_DBG("Cleanuping dev %p", dev);
2815
2816                 dev_user_unjam_dev(dev);
2817
2818                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2819
2820                 rc = dev_user_get_next_cmd(dev, &ucmd);
2821                 if (rc == 0)
2822                         dev_user_unjam_cmd(ucmd, 1, NULL);
2823
2824                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2825
2826                 if (rc == -EAGAIN) {
2827                         if (dev->cleanup_done)
2828                                 break;
2829                         else {
2830                                 TRACE_DBG("No more commands (dev %p)", dev);
2831                                 goto out;
2832                         }
2833                 }
2834         }
2835
2836 #ifdef CONFIG_SCST_EXTRACHECKS
2837 {
2838         int i;
2839         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2840                 struct list_head *head = &dev->ucmd_hash[i];
2841                 struct scst_user_cmd *ucmd2;
2842 again:
2843                 list_for_each_entry(ucmd2, head, hash_list_entry) {
2844                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
2845                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
2846                         ucmd_put(ucmd2);
2847                         goto again;
2848                 }
2849         }
2850 }
2851 #endif
2852
2853         TRACE_DBG("Cleanuping done (dev %p)", dev);
2854         complete_all(&dev->cleanup_cmpl);
2855         res = 0;
2856
2857 out:
2858         TRACE_EXIT_RES(res);
2859         return res;
2860 }
2861
2862 static inline int test_cleanup_list(void)
2863 {
2864         int res = !list_empty(&cleanup_list) ||
2865                   unlikely(kthread_should_stop());
2866         return res;
2867 }
2868
2869 static int dev_user_cleanup_thread(void *arg)
2870 {
2871         TRACE_ENTRY();
2872
2873         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2874
2875         current->flags |= PF_NOFREEZE;
2876
2877         spin_lock(&cleanup_lock);
2878         while (!kthread_should_stop()) {
2879                 wait_queue_t wait;
2880                 init_waitqueue_entry(&wait, current);
2881
2882                 if (!test_cleanup_list()) {
2883                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2884                         for (;;) {
2885                                 set_current_state(TASK_INTERRUPTIBLE);
2886                                 if (test_cleanup_list())
2887                                         break;
2888                                 spin_unlock(&cleanup_lock);
2889                                 schedule();
2890                                 spin_lock(&cleanup_lock);
2891                         }
2892                         set_current_state(TASK_RUNNING);
2893                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2894                 }
2895
2896                 /*
2897                  * We have to poll devices, because commands can go from SCST
2898                  * core on cmd_list_waitQ and we have no practical way to
2899                  * detect them.
2900                  */
2901
2902                 while (1) {
2903                         struct scst_user_dev *dev;
2904                         LIST_HEAD(cl_devs);
2905
2906                         while (!list_empty(&cleanup_list)) {
2907                                 int rc;
2908
2909                                 dev = list_entry(cleanup_list.next,
2910                                         typeof(*dev), cleanup_list_entry);
2911                                 list_del(&dev->cleanup_list_entry);
2912
2913                                 spin_unlock(&cleanup_lock);
2914                                 rc = dev_user_process_cleanup(dev);
2915                                 spin_lock(&cleanup_lock);
2916
2917                                 if (rc != 0)
2918                                         list_add_tail(&dev->cleanup_list_entry,
2919                                                 &cl_devs);
2920                         }
2921
2922                         if (list_empty(&cl_devs))
2923                                 break;
2924
2925                         spin_unlock(&cleanup_lock);
2926                         msleep(100);
2927                         spin_lock(&cleanup_lock);
2928
2929                         while (!list_empty(&cl_devs)) {
2930                                 dev = list_entry(cl_devs.next, typeof(*dev),
2931                                         cleanup_list_entry);
2932                                 list_move_tail(&dev->cleanup_list_entry,
2933                                         &cleanup_list);
2934                         }
2935                 }
2936         }
2937         spin_unlock(&cleanup_lock);
2938
2939         /*
2940          * If kthread_should_stop() is true, we are guaranteed to be
2941          * on the module unload, so cleanup_list must be empty.
2942          */
2943         sBUG_ON(!list_empty(&cleanup_list));
2944
2945         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2946
2947         TRACE_EXIT();
2948         return 0;
2949 }
2950
2951 static int __init init_scst_user(void)
2952 {
2953         int res = 0;
2954         struct max_get_reply {
2955                 union {
2956                         struct scst_user_get_cmd g;
2957                         struct scst_user_reply_cmd r;
2958                 };
2959         };
2960 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
2961         struct class_device *class_member;
2962 #else
2963         struct device *dev;
2964 #endif
2965
2966         TRACE_ENTRY();
2967
2968 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2969         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2970                 "Consider changing VMSPLIT option or use a 64-bit "
2971                 "configuration instead. See README file for details.");
2972         res = -EINVAL;
2973         goto out;
2974 #endif
2975
2976         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2977         if (user_cmd_cachep == NULL) {
2978                 res = -ENOMEM;
2979                 goto out;
2980         }
2981
2982         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
2983         if (user_get_cmd_cachep == NULL) {
2984                 res = -ENOMEM;
2985                 goto out_cache;
2986         }
2987
2988         dev_user_devtype.module = THIS_MODULE;
2989
2990         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2991         if (res < 0)
2992                 goto out_cache1;
2993
2994         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2995         if (res != 0)
2996                 goto out_unreg;
2997
2998         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2999         if (IS_ERR(dev_user_sysfs_class)) {
3000                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
3001                         "space handler");
3002                 res = PTR_ERR(dev_user_sysfs_class);
3003                 goto out_proc;
3004         }
3005
3006         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
3007         if (res) {
3008                 PRINT_ERROR("Unable to get major %d for SCSI tapes",
3009                             DEV_USER_MAJOR);
3010                 goto out_class;
3011         }
3012
3013 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3014         class_member = class_device_create(dev_user_sysfs_class, NULL,
3015                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
3016         if (IS_ERR(class_member)) {
3017                 res = PTR_ERR(class_member);
3018                 goto out_chrdev;
3019         }
3020 #else
3021         dev = device_create(dev_user_sysfs_class, NULL,
3022                             MKDEV(DEV_USER_MAJOR, 0),
3023 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
3024                                 NULL,
3025 #endif
3026                                 DEV_USER_NAME);
3027         if (IS_ERR(dev)) {
3028                 res = PTR_ERR(dev);
3029                 goto out_chrdev;
3030         }
3031 #endif
3032
3033         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
3034                 "scst_usr_cleanupd");
3035         if (IS_ERR(cleanup_thread)) {
3036                 res = PTR_ERR(cleanup_thread);
3037                 PRINT_ERROR("kthread_create() failed: %d", res);
3038                 goto out_dev;
3039         }
3040
3041 out:
3042         TRACE_EXIT_RES(res);
3043         return res;
3044
3045 out_dev:
3046 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3047         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3048 #else
3049         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3050 #endif
3051
3052 out_chrdev:
3053         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3054
3055 out_class:
3056         class_destroy(dev_user_sysfs_class);
3057
3058 out_proc:
3059         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3060
3061 out_unreg:
3062         scst_unregister_dev_driver(&dev_user_devtype);
3063
3064 out_cache1:
3065         kmem_cache_destroy(user_get_cmd_cachep);
3066
3067 out_cache:
3068         kmem_cache_destroy(user_cmd_cachep);
3069         goto out;
3070 }
3071
3072 static void __exit exit_scst_user(void)
3073 {
3074         int rc;
3075
3076         TRACE_ENTRY();
3077
3078         rc = kthread_stop(cleanup_thread);
3079         if (rc < 0)
3080                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3081
3082         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3083 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3084         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3085 #else
3086         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3087 #endif
3088         class_destroy(dev_user_sysfs_class);
3089
3090         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3091         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3092
3093         kmem_cache_destroy(user_get_cmd_cachep);
3094         kmem_cache_destroy(user_cmd_cachep);
3095
3096         TRACE_EXIT();
3097         return;
3098 }
3099
3100 module_init(init_scst_user);
3101 module_exit(exit_scst_user);
3102
3103 MODULE_AUTHOR("Vladislav Bolkhovitin");
3104 MODULE_LICENSE("GPL");
3105 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3106 MODULE_VERSION(SCST_VERSION_STRING);
3107 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);