The first approximation to update to 2.6.27 + minor fixes, particularly remove of...
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23
24 #define LOG_PREFIX              DEV_USER_NAME
25
26 #include "scst.h"
27 #include "scst_user.h"
28 #include "scst_dev_handler.h"
29
30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
31 #warning "HIGHMEM kernel configurations are not supported by this module, \
32         because nowadays it doesn't worth the effort. Consider change \
33         VMSPLIT option or use 64-bit configuration instead. See README file \
34         for details."
35 #endif
36
37 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18))
38 #define COMPLETION_INITIALIZER_ONSTACK(work) \
39         ({ init_completion(&work); work; })
40         
41 /*
42  * Lockdep needs to run a non-constant initializer for on-stack
43  * completions - so we use the _ONSTACK() variant for those that
44  * are on the kernel stack:
45  */
46 #ifdef CONFIG_LOCKDEP
47 # define DECLARE_COMPLETION_ONSTACK(work) \
48         struct completion work = COMPLETION_INITIALIZER_ONSTACK(work)
49 #else
50 # define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work)
51 #endif
52
53 #endif
54
55 #define DEV_USER_MAJOR                  237
56
57 #define DEV_USER_CMD_HASH_ORDER         6
58
59 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
60
61 struct scst_user_dev {
62         struct rw_semaphore dev_rwsem;
63
64         struct scst_cmd_lists cmd_lists;
65
66         /* Protected by cmd_lists.cmd_list_lock */
67         struct list_head ready_cmd_list;
68
69         /* Protected by dev_rwsem or don't need any protection */
70         unsigned int blocking:1;
71         unsigned int cleanup_done:1;
72         unsigned int tst:3;
73         unsigned int queue_alg:4;
74         unsigned int tas:1;
75         unsigned int swp:1;
76         unsigned int has_own_order_mgmt:1;
77
78         int (*generic_parse)(struct scst_cmd *cmd,
79                 int (*get_block)(struct scst_cmd *cmd));
80
81         int block;
82         int def_block;
83
84         struct scst_mem_lim udev_mem_lim;
85         struct sgv_pool *pool;
86
87         uint8_t parse_type;
88         uint8_t on_free_cmd_type;
89         uint8_t memory_reuse_type;
90         uint8_t partial_transfers_type;
91         uint32_t partial_len;
92
93         struct scst_dev_type devtype;
94
95         /* Both protected by cmd_lists.cmd_list_lock */
96         unsigned int handle_counter;
97         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
98
99         struct scst_device *sdev;
100
101         int virt_id;
102         struct list_head dev_list_entry;
103         char name[SCST_MAX_NAME];
104
105         struct list_head cleanup_list_entry;
106         /* ToDo: make it on-stack */
107         struct completion cleanup_cmpl;
108 };
109
110 /* Most fields are unprotected, since only one thread at time can access them */
111 struct scst_user_cmd {
112         struct scst_cmd *cmd;
113         struct scst_user_dev *dev;
114
115         atomic_t ucmd_ref;
116
117         unsigned int buff_cached:1;
118         unsigned int buf_dirty:1;
119         unsigned int background_exec:1;
120         unsigned int aborted:1;
121
122         struct scst_user_cmd *buf_ucmd;
123
124         int cur_data_page;
125         int num_data_pages;
126         int first_page_offset;
127         unsigned long ubuff;
128         struct page **data_pages;
129         struct sgv_pool_obj *sgv;
130
131         /*
132          * Special flags, which can be accessed asynchronously (hence "long").
133          * Protected by cmd_lists.cmd_list_lock.
134          */
135         unsigned long sent_to_user:1;
136         unsigned long jammed:1;
137         unsigned long this_state_unjammed:1;
138         unsigned long seen_by_user:1; /* here only as a small optimization */
139
140         unsigned int state;
141
142         struct list_head ready_cmd_list_entry;
143
144         unsigned int h;
145         struct list_head hash_list_entry;
146
147         struct scst_user_get_cmd user_cmd;
148
149         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
150         union {
151                 struct completion *cmpl;
152                 struct scst_mgmt_cmd *mcmd;
153         };
154         int result;
155 };
156
157 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
158         gfp_t gfp_mask);
159 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
160
161 static int dev_user_parse(struct scst_cmd *cmd);
162 static int dev_user_exec(struct scst_cmd *cmd);
163 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
164 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
165         struct scst_tgt_dev *tgt_dev);
166
167 static int dev_user_disk_done(struct scst_cmd *cmd);
168 static int dev_user_tape_done(struct scst_cmd *cmd);
169
170 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
171         gfp_t gfp_mask, void *priv);
172 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
173                                      void *priv);
174
175 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
176
177 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
178         unsigned long *flags);
179
180 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
181 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
182         int status);
183 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
184 static int dev_user_register_dev(struct file *file,
185         const struct scst_user_dev_desc *dev_desc);
186 static int __dev_user_set_opt(struct scst_user_dev *dev,
187         const struct scst_user_opt *opt);
188 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
189 static int dev_user_get_opt(struct file *file, void __user *arg);
190
191 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
192 static long dev_user_ioctl(struct file *file, unsigned int cmd,
193         unsigned long arg);
194 static int dev_user_release(struct inode *inode, struct file *file);
195
196 /** Data **/
197
198 static struct kmem_cache *user_cmd_cachep;
199 static struct kmem_cache *user_get_cmd_cachep;
200
201 static DEFINE_MUTEX(dev_priv_mutex);
202
203 static struct file_operations dev_user_fops = {
204         .poll           = dev_user_poll,
205         .unlocked_ioctl = dev_user_ioctl,
206 #ifdef CONFIG_COMPAT
207         .compat_ioctl   = dev_user_ioctl,
208 #endif
209         .release        = dev_user_release,
210 };
211
212 static struct class *dev_user_sysfs_class;
213
214 static DEFINE_SPINLOCK(dev_list_lock);
215 static LIST_HEAD(dev_list);
216
217 static DEFINE_SPINLOCK(cleanup_lock);
218 static LIST_HEAD(cleanup_list);
219 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
220 static struct task_struct *cleanup_thread;
221
222 /*
223  * Skip this command if result is not 0. Must be called under
224  * cmd_lists.cmd_list_lock and IRQ off.
225  */
226 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
227 {
228         int r = atomic_inc_return(&ucmd->ucmd_ref);
229         int res;
230         if (unlikely(r == 1)) {
231                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
232                 atomic_dec(&ucmd->ucmd_ref);
233                 res = true;
234                 /*
235                  * Necessary code is serialized by cmd_list_lock in
236                  * cmd_remove_hash()
237                  */
238         } else {
239                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
240                         atomic_read(&ucmd->ucmd_ref));
241                 res = false;
242         }
243         return res;
244 }
245
246 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
247 {
248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
249         atomic_inc(&ucmd->ucmd_ref);
250         if (barrier)
251                 smp_mb__after_atomic_inc();
252 }
253
254 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
255 {
256         __ucmd_get(ucmd, true);
257 }
258
259 static inline void ucmd_get(struct scst_user_cmd *ucmd)
260 {
261         __ucmd_get(ucmd, false);
262 }
263
264 /* Must not be called under cmd_list_lock!! */
265 static inline void ucmd_put(struct scst_user_cmd *ucmd)
266 {
267         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
268
269         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
270
271         if (atomic_dec_and_test(&ucmd->ucmd_ref))
272                 dev_user_free_ucmd(ucmd);
273 }
274
275 static inline int calc_num_pg(unsigned long buf, int len)
276 {
277         len += buf & ~PAGE_MASK;
278         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
279 }
280
281 static inline int is_need_offs_page(unsigned long buf, int len)
282 {
283         return ((buf & ~PAGE_MASK) != 0) &&
284                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
285 }
286
287 static void __dev_user_not_reg(void)
288 {
289         PRINT_ERROR("%s", "Device not registered");
290         return;
291 }
292
293 static inline int dev_user_check_reg(struct scst_user_dev *dev)
294 {
295         if (dev == NULL) {
296                 __dev_user_not_reg();
297                 return -EINVAL;
298         }
299         return 0;
300 }
301
302 static inline int scst_user_cmd_hashfn(int h)
303 {
304         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
305 }
306
307 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
308         unsigned int h)
309 {
310         struct list_head *head;
311         struct scst_user_cmd *ucmd;
312
313         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
314         list_for_each_entry(ucmd, head, hash_list_entry) {
315                 if (ucmd->h == h) {
316                         TRACE_DBG("Found ucmd %p", ucmd);
317                         return ucmd;
318                 }
319         }
320         return NULL;
321 }
322
323 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
324 {
325         struct list_head *head;
326         struct scst_user_dev *dev = ucmd->dev;
327         struct scst_user_cmd *u;
328         unsigned long flags;
329
330         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
331         do {
332                 ucmd->h = dev->handle_counter++;
333                 u = __ucmd_find_hash(dev, ucmd->h);
334         } while (u != NULL);
335         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
336         list_add_tail(&ucmd->hash_list_entry, head);
337         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
338
339         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
340         return;
341 }
342
343 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
344 {
345         unsigned long flags;
346
347         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
348         list_del(&ucmd->hash_list_entry);
349         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
350
351         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
352         return;
353 }
354
355 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
356 {
357         TRACE_ENTRY();
358
359         TRACE_MEM("Freeing ucmd %p", ucmd);
360
361         cmd_remove_hash(ucmd);
362         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
363
364         kmem_cache_free(user_cmd_cachep, ucmd);
365
366         TRACE_EXIT();
367         return;
368 }
369
370 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
371         gfp_t gfp_mask, void *priv)
372 {
373         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
374         int offset = 0;
375
376         TRACE_ENTRY();
377
378         /* *sg supposed to be zeroed */
379
380         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
381                 ucmd->ubuff, ucmd->cur_data_page);
382
383         if (ucmd->cur_data_page == 0) {
384                 TRACE_MEM("ucmd->first_page_offset %d",
385                         ucmd->first_page_offset);
386                 offset = ucmd->first_page_offset;
387                 ucmd_get(ucmd);
388         }
389
390         if (ucmd->cur_data_page >= ucmd->num_data_pages)
391                 goto out;
392
393         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
394                 PAGE_SIZE - offset, offset);
395         ucmd->cur_data_page++;
396
397         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
398                 sg->offset);
399         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
400
401 out:
402         TRACE_EXIT();
403         return sg_page(sg);
404 }
405
406 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
407 {
408         TRACE_ENTRY();
409
410         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
411                 ucmd, ucmd->h, ucmd->ubuff);
412
413         ucmd->user_cmd.cmd_h = ucmd->h;
414         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
415         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
416
417         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
418
419         dev_user_add_to_ready(ucmd);
420
421         TRACE_EXIT();
422         return;
423 }
424
425 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
426 {
427         int i;
428
429         TRACE_ENTRY();
430
431         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
432                 ucmd->ubuff, ucmd->num_data_pages);
433
434         for (i = 0; i < ucmd->num_data_pages; i++) {
435                 struct page *page = ucmd->data_pages[i];
436
437                 if (ucmd->buf_dirty)
438                         SetPageDirty(page);
439
440                 page_cache_release(page);
441         }
442
443         kfree(ucmd->data_pages);
444         ucmd->data_pages = NULL;
445
446         TRACE_EXIT();
447         return;
448 }
449
450 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
451 {
452         TRACE_ENTRY();
453
454         sBUG_ON(ucmd->data_pages == NULL);
455
456         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
457                 ucmd, ucmd->ubuff, ucmd->buff_cached);
458
459         dev_user_unmap_buf(ucmd);
460
461         if (ucmd->buff_cached)
462                 dev_user_on_cached_mem_free(ucmd);
463         else
464                 ucmd_put(ucmd);
465
466         TRACE_EXIT();
467         return;
468 }
469
470 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
471         void *priv)
472 {
473         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
474
475         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
476                 sg_count, ucmd);
477
478         __dev_user_free_sg_entries(ucmd);
479
480         return;
481 }
482
483 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
484 {
485         int mem_reuse_type = ucmd->dev->memory_reuse_type;
486
487         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
488             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
489              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
490             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
491              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
492                 return 1;
493         else
494                 return 0;
495 }
496
497 /*
498  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
499  * Unmaps the buffer, if needed in case of error
500  */
501 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
502 {
503         int res = 0;
504         struct scst_cmd *cmd = ucmd->cmd;
505         struct scst_user_dev *dev = ucmd->dev;
506         gfp_t gfp_mask;
507         int flags = 0;
508         int bufflen = cmd->bufflen;
509         int last_len = 0;
510
511         TRACE_ENTRY();
512
513         sBUG_ON(bufflen == 0);
514
515         gfp_mask = __GFP_NOWARN;
516         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
517
518         if (cached_buff) {
519                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
520                 if (ucmd->ubuff == 0)
521                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
522         } else {
523                 TRACE_MEM("%s", "Not cached buff");
524                 flags |= SCST_POOL_ALLOC_NO_CACHED;
525                 if (ucmd->ubuff == 0) {
526                         res = 1;
527                         goto out;
528                 }
529                 bufflen += ucmd->first_page_offset;
530                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
531                         last_len = bufflen & ~PAGE_MASK;
532                 else
533                         last_len = cmd->bufflen & ~PAGE_MASK;
534                 if (last_len == 0)
535                         last_len = PAGE_SIZE;
536         }
537         ucmd->buff_cached = cached_buff;
538
539         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
540                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
541         if (cmd->sg != NULL) {
542                 struct scst_user_cmd *buf_ucmd =
543                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
544
545                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
546
547                 ucmd->ubuff = buf_ucmd->ubuff;
548                 ucmd->buf_ucmd = buf_ucmd;
549
550                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
551                                    (ucmd != buf_ucmd));
552
553                 if (last_len != 0) {
554                         /* We don't use clustering, so the assignment is safe */
555                         cmd->sg[cmd->sg_cnt-1].length = last_len;
556                 }
557
558                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
559                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
560                         last_len, cmd->sg[cmd->sg_cnt-1].length);
561
562                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
563                         static int ll;
564                         if (ll < 10) {
565                                 PRINT_INFO("Unable to complete command due to "
566                                         "SG IO count limitation (requested %d, "
567                                         "available %d, tgt lim %d)", cmd->sg_cnt,
568                                         cmd->tgt_dev->max_sg_cnt,
569                                         cmd->tgt->sg_tablesize);
570                                 ll++;
571                         }
572                         cmd->sg = NULL;
573                         /* sgv will be freed in dev_user_free_sgv() */
574                         res = -1;
575                 }
576         } else {
577                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
578                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
579                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
580                 if (unlikely(cmd->sg_cnt == 0)) {
581                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
582                         sBUG_ON(ucmd->sgv != NULL);
583                         res = -1;
584                 } else {
585                         switch (ucmd->state) {
586                         case UCMD_STATE_BUF_ALLOCING:
587                                 res = 1;
588                                 break;
589                         case UCMD_STATE_EXECING:
590                                 res = -1;
591                                 break;
592                         default:
593                                 sBUG();
594                                 break;
595                         }
596                 }
597         }
598
599 out:
600         TRACE_EXIT_RES(res);
601         return res;
602 }
603
604 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
605 {
606         int rc, res = SCST_CMD_STATE_DEFAULT;
607         struct scst_cmd *cmd = ucmd->cmd;
608
609         TRACE_ENTRY();
610
611         if (unlikely(ucmd->cmd->data_buf_tgt_alloc)) {
612                 PRINT_ERROR("Target driver %s requested own memory "
613                         "allocation", ucmd->cmd->tgtt->name);
614                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
615                 res = scst_get_cmd_abnormal_done_state(cmd);
616                 goto out;
617         }
618
619         ucmd->state = UCMD_STATE_BUF_ALLOCING;
620         cmd->data_buf_alloced = 1;
621
622         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
623         if (rc == 0)
624                 goto out;
625         else if (rc < 0) {
626                 scst_set_busy(cmd);
627                 res = scst_get_cmd_abnormal_done_state(cmd);
628                 goto out;
629         }
630
631         if ((cmd->data_direction != SCST_DATA_WRITE) &&
632             !scst_is_cmd_local(cmd)) {
633                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
634                 goto out;
635         }
636
637         ucmd->user_cmd.cmd_h = ucmd->h;
638         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
639         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
640         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
641                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
642         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
643         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
644                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
645         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
646         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
647         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
648
649         dev_user_add_to_ready(ucmd);
650
651         res = SCST_CMD_STATE_STOP;
652
653 out:
654         TRACE_EXIT_RES(res);
655         return res;
656 }
657
658 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
659         gfp_t gfp_mask)
660 {
661         struct scst_user_cmd *ucmd = NULL;
662
663         TRACE_ENTRY();
664
665 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
666         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
667         if (ucmd != NULL)
668                 memset(ucmd, 0, sizeof(*ucmd));
669 #else
670         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
671 #endif
672         if (unlikely(ucmd == NULL)) {
673                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
674                         "user cmd (gfp_mask %x)", gfp_mask);
675                 goto out;
676         }
677         ucmd->dev = dev;
678         atomic_set(&ucmd->ucmd_ref, 1);
679
680         cmd_insert_hash(ucmd);
681
682         TRACE_MEM("ucmd %p allocated", ucmd);
683
684 out:
685         TRACE_EXIT_HRES((unsigned long)ucmd);
686         return ucmd;
687 }
688
689 static int dev_user_get_block(struct scst_cmd *cmd)
690 {
691         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
692         /*
693          * No need for locks here, since *_detach() can not be
694          * called, when there are existing commands.
695          */
696         TRACE_EXIT_RES(dev->block);
697         return dev->block;
698 }
699
700 static int dev_user_parse(struct scst_cmd *cmd)
701 {
702         int rc, res = SCST_CMD_STATE_DEFAULT;
703         struct scst_user_cmd *ucmd;
704         int atomic = scst_cmd_atomic(cmd);
705         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
706         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
707
708         TRACE_ENTRY();
709
710         if (cmd->dh_priv == NULL) {
711                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
712                 if (unlikely(ucmd == NULL)) {
713                         if (atomic) {
714                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
715                                 goto out;
716                         } else {
717                                 scst_set_busy(cmd);
718                                 goto out_error;
719                         }
720                 }
721                 ucmd->cmd = cmd;
722                 cmd->dh_priv = ucmd;
723         } else {
724                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
725                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
726         }
727
728         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
729
730         if (ucmd->state != UCMD_STATE_NEW)
731                 goto alloc;
732
733         switch (dev->parse_type) {
734         case SCST_USER_PARSE_STANDARD:
735                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
736                 rc = dev->generic_parse(cmd, dev_user_get_block);
737                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
738                         goto out_invalid;
739                 break;
740
741         case SCST_USER_PARSE_EXCEPTION:
742                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
743                 rc = dev->generic_parse(cmd, dev_user_get_block);
744                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
745                         break;
746                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
747                         TRACE_MEM("Restarting PARSE to thread context "
748                                 "(ucmd %p)", ucmd);
749                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
750                         goto out;
751                 }
752                 /* else go through */
753
754         case SCST_USER_PARSE_CALL:
755                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
756                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
757                 ucmd->user_cmd.cmd_h = ucmd->h;
758                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
759                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
760                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
761                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
762                             sizeof(cmd->cdb)));
763                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
764                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
765                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
766                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
767                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
768                 ucmd->user_cmd.parse_cmd.expected_values_set =
769                                         cmd->expected_values_set;
770                 ucmd->user_cmd.parse_cmd.expected_data_direction =
771                                         cmd->expected_data_direction;
772                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
773                                         cmd->expected_transfer_len;
774                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
775                 ucmd->state = UCMD_STATE_PARSING;
776                 dev_user_add_to_ready(ucmd);
777                 res = SCST_CMD_STATE_STOP;
778                 goto out;
779
780         default:
781                 sBUG();
782                 goto out;
783         }
784
785 alloc:
786         if (cmd->data_direction != SCST_DATA_NONE)
787                 res = dev_user_alloc_space(ucmd);
788
789 out:
790         TRACE_EXIT_RES(res);
791         return res;
792
793 out_invalid:
794         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
795                 cmd->op_flags & SCST_INFO_INVALID);
796         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
797
798 out_error:
799         res = scst_get_cmd_abnormal_done_state(cmd);
800         goto out;
801 }
802
803 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
804 {
805         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
806         unsigned long start = buf_ucmd->ubuff;
807         int i;
808
809         TRACE_ENTRY();
810
811         if (start == 0)
812                 goto out;
813
814         for (i = 0; i < buf_ucmd->num_data_pages; i++) {
815                 struct page *page;
816                 page = buf_ucmd->data_pages[i];
817 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
818                 struct vm_area_struct *vma = find_vma(current->mm, start);
819                 if (vma != NULL)
820                         flush_anon_page(vma, page, start);
821 #endif
822                 flush_dcache_page(page);
823                 start += PAGE_SIZE;
824         }
825
826 out:
827         TRACE_EXIT();
828         return;
829 }
830
831 static int dev_user_exec(struct scst_cmd *cmd)
832 {
833         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
834         int res = SCST_EXEC_COMPLETED;
835
836         TRACE_ENTRY();
837
838 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
839        * context to complete the necessary actions, but all we are going to
840        * do in this function is, in fact, atomic, so let's skip this check.
841        */
842         if (scst_cmd_atomic(cmd)) {
843                 TRACE_DBG("%s", "User exec() can not be called in atomic "
844                         "context, rescheduling to the thread");
845                 res = SCST_EXEC_NEED_THREAD;
846                 goto out;
847         }
848 #endif
849
850         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
851                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
852                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
853
854         if (cmd->data_direction == SCST_DATA_WRITE)
855                 dev_user_flush_dcache(ucmd);
856
857         ucmd->user_cmd.cmd_h = ucmd->h;
858         ucmd->user_cmd.subcode = SCST_USER_EXEC;
859         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
860         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
861                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
862                     sizeof(cmd->cdb)));
863         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
864         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
865         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
866         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
867         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
868                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
869                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
870         }
871         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
872         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
873         ucmd->user_cmd.exec_cmd.partial = 0;
874         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
875         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
876
877         ucmd->state = UCMD_STATE_EXECING;
878
879         dev_user_add_to_ready(ucmd);
880
881         TRACE_EXIT_RES(res);
882         return res;
883 }
884
885 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
886 {
887         if (ucmd->sgv != NULL) {
888                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
889                 ucmd->sgv = NULL;
890         } else if (ucmd->data_pages != NULL) {
891                 /* We mapped pages, but for some reason didn't allocate them */
892                 ucmd_get(ucmd);
893                 __dev_user_free_sg_entries(ucmd);
894         }
895         return;
896 }
897
898 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
899 {
900         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
901
902         TRACE_ENTRY();
903
904         if (unlikely(ucmd == NULL))
905                 goto out;
906
907         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
908                 ucmd->buff_cached, ucmd->ubuff);
909
910         ucmd->cmd = NULL;
911         if ((cmd->data_direction == SCST_DATA_WRITE) && (ucmd->buf_ucmd != NULL))
912                 ucmd->buf_ucmd->buf_dirty = 1;
913
914         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
915                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
916                 /* The state assignment must be before freeing sgv! */
917                 goto out_reply;
918         }
919
920         if (unlikely(!ucmd->seen_by_user)) {
921                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
922                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
923                 goto out_reply;
924         }
925
926         ucmd->user_cmd.cmd_h = ucmd->h;
927         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
928
929         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
930         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
931         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
932         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
933         ucmd->user_cmd.on_free_cmd.status = cmd->status;
934         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
935
936         ucmd->state = UCMD_STATE_ON_FREEING;
937
938         dev_user_add_to_ready(ucmd);
939
940 out:
941         TRACE_EXIT();
942         return;
943
944 out_reply:
945         dev_user_process_reply_on_free(ucmd);
946         goto out;
947 }
948
949 static void dev_user_set_block(struct scst_cmd *cmd, int block)
950 {
951         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
952         /*
953          * No need for locks here, since *_detach() can not be
954          * called, when there are existing commands.
955          */
956         TRACE_DBG("dev %p, new block %d", dev, block);
957         if (block != 0)
958                 dev->block = block;
959         else
960                 dev->block = dev->def_block;
961         return;
962 }
963
964 static int dev_user_disk_done(struct scst_cmd *cmd)
965 {
966         int res = SCST_CMD_STATE_DEFAULT;
967
968         TRACE_ENTRY();
969
970         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
971
972         TRACE_EXIT_RES(res);
973         return res;
974 }
975
976 static int dev_user_tape_done(struct scst_cmd *cmd)
977 {
978         int res = SCST_CMD_STATE_DEFAULT;
979
980         TRACE_ENTRY();
981
982         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
983
984         TRACE_EXIT_RES(res);
985         return res;
986 }
987
988 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
989 {
990         struct scst_user_dev *dev = ucmd->dev;
991         unsigned long flags;
992         int do_wake;
993
994         TRACE_ENTRY();
995
996         do_wake = (in_interrupt() ||
997                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
998         if (ucmd->cmd)
999                 do_wake |= ucmd->cmd->preprocessing_only;
1000
1001         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1002
1003         ucmd->this_state_unjammed = 0;
1004
1005         if ((ucmd->state == UCMD_STATE_PARSING) ||
1006             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1007                 /*
1008                  * If we don't put such commands in the queue head, then under
1009                  * high load we might delay threads, waiting for memory
1010                  * allocations, for too long and start loosing NOPs, which
1011                  * would lead to consider us by remote initiators as
1012                  * unresponsive and stuck => broken connections, etc. If none
1013                  * of our commands completed in NOP timeout to allow the head
1014                  * commands to go, then we are really overloaded and/or stuck.
1015                  */
1016                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1017                         "cmd list", ucmd, ucmd->state);
1018                 list_add(&ucmd->ready_cmd_list_entry,
1019                         &dev->ready_cmd_list);
1020                 do_wake = 1;
1021         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1022                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1023                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1024                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1025                         "ready cmd list", ucmd, ucmd->state);
1026                 list_add(&ucmd->ready_cmd_list_entry,
1027                         &dev->ready_cmd_list);
1028                 do_wake = 1;
1029         } else if ((ucmd->cmd != NULL) &&
1030                    unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1031                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1032                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1033         } else {
1034                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1035                 list_add_tail(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1036         }
1037
1038         if (do_wake) {
1039                 TRACE_DBG("Waking up dev %p", dev);
1040                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1041         }
1042
1043         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1044
1045         TRACE_EXIT();
1046         return;
1047 }
1048
1049 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1050         int num_pg)
1051 {
1052         int res = 0, rc;
1053         int i;
1054
1055         TRACE_ENTRY();
1056
1057         if (unlikely(ubuff == 0))
1058                 goto out_nomem;
1059
1060         sBUG_ON(ucmd->data_pages != NULL);
1061
1062         ucmd->num_data_pages = num_pg;
1063
1064         ucmd->data_pages = kmalloc(sizeof(*ucmd->data_pages)*ucmd->num_data_pages,
1065                 GFP_KERNEL);
1066         if (ucmd->data_pages == NULL) {
1067                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1068                         "(num_data_pages=%d)", ucmd->num_data_pages);
1069                 res = -ENOMEM;
1070                 goto out_nomem;
1071         }
1072
1073         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d, "
1074                 "first_page_offset %d, len %d)", ucmd, ubuff,
1075                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1076                 ucmd->cmd->bufflen);
1077
1078         down_read(&current->mm->mmap_sem);
1079         rc = get_user_pages(current, current->mm, ubuff, ucmd->num_data_pages,
1080                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1081         up_read(&current->mm->mmap_sem);
1082
1083         /* get_user_pages() flushes dcache */
1084
1085         if (rc < ucmd->num_data_pages)
1086                 goto out_unmap;
1087
1088         ucmd->ubuff = ubuff;
1089         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1090
1091 out:
1092         TRACE_EXIT_RES(res);
1093         return res;
1094
1095 out_nomem:
1096         scst_set_busy(ucmd->cmd);
1097         /* go through */
1098
1099 out_err:
1100         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1101         goto out;
1102
1103 out_unmap:
1104         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1105                 ucmd->num_data_pages, rc);
1106         if (rc > 0) {
1107                 for (i = 0; i < rc; i++)
1108                         page_cache_release(ucmd->data_pages[i]);
1109         }
1110         kfree(ucmd->data_pages);
1111         ucmd->data_pages = NULL;
1112         res = -EFAULT;
1113         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1114         goto out_err;
1115 }
1116
1117 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1118         struct scst_user_reply_cmd *reply)
1119 {
1120         int res = 0;
1121         struct scst_cmd *cmd = ucmd->cmd;
1122
1123         TRACE_ENTRY();
1124
1125         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1126
1127         if (likely(reply->alloc_reply.pbuf != 0)) {
1128                 int pages;
1129                 if (ucmd->buff_cached) {
1130                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1131                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1132                                         "page aligned", reply->alloc_reply.pbuf);
1133                                 goto out_hwerr;
1134                         }
1135                         pages = cmd->sg_cnt;
1136                 } else
1137                         pages = calc_num_pg(reply->alloc_reply.pbuf, cmd->bufflen);
1138                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1139         } else {
1140                 scst_set_busy(ucmd->cmd);
1141                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1142         }
1143
1144 out_process:
1145         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1146
1147         TRACE_EXIT_RES(res);
1148         return res;
1149
1150 out_hwerr:
1151         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1152         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1153         res = -EINVAL;
1154         goto out_process;
1155 }
1156
1157 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1158         struct scst_user_reply_cmd *reply)
1159 {
1160         int res = 0;
1161         struct scst_user_scsi_cmd_reply_parse *preply =
1162                 &reply->parse_reply;
1163         struct scst_cmd *cmd = ucmd->cmd;
1164
1165         TRACE_ENTRY();
1166
1167         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1168                 goto out_inval;
1169
1170         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1171                      (preply->data_direction != SCST_DATA_READ) &&
1172                      (preply->data_direction != SCST_DATA_NONE)))
1173                 goto out_inval;
1174
1175         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1176                      (preply->bufflen == 0)))
1177                 goto out_inval;
1178
1179         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1180                 goto out_inval;
1181
1182         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1183                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1184                 preply->data_direction, preply->bufflen, preply->data_len,
1185                 reply->alloc_reply.pbuf);
1186
1187         cmd->queue_type = preply->queue_type;
1188         cmd->data_direction = preply->data_direction;
1189         cmd->bufflen = preply->bufflen;
1190         cmd->data_len = preply->data_len;
1191
1192 out_process:
1193         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1194
1195         TRACE_EXIT_RES(res);
1196         return res;
1197
1198 out_inval:
1199         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1200                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1201         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1202         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1203         scst_set_cmd_abnormal_done_state(cmd);
1204         res = -EINVAL;
1205         goto out_process;
1206 }
1207
1208 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1209 {
1210         int res = 0;
1211
1212         TRACE_ENTRY();
1213
1214         TRACE_DBG("ON FREE ucmd %p", ucmd);
1215
1216         dev_user_free_sgv(ucmd);
1217         ucmd_put(ucmd);
1218
1219         TRACE_EXIT_RES(res);
1220         return res;
1221 }
1222
1223 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1224 {
1225         int res = 0;
1226
1227         TRACE_ENTRY();
1228
1229         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1230
1231         ucmd_put(ucmd);
1232
1233         TRACE_EXIT_RES(res);
1234         return res;
1235 }
1236
1237 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1238         struct scst_user_reply_cmd *reply)
1239 {
1240         int res = 0;
1241         struct scst_user_scsi_cmd_reply_exec *ereply =
1242                 &reply->exec_reply;
1243         struct scst_cmd *cmd = ucmd->cmd;
1244
1245         TRACE_ENTRY();
1246
1247         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1248                 if (ucmd->background_exec) {
1249                         TRACE_DBG("Background ucmd %p finished", ucmd);
1250                         ucmd_put(ucmd);
1251                         goto out;
1252                 }
1253                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1254                         goto out_inval;
1255                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1256                              (ereply->resp_data_len != 0)))
1257                         goto out_inval;
1258         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1259                 if (unlikely(ucmd->background_exec))
1260                         goto out_inval;
1261                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1262                              (cmd->resp_data_len != 0)))
1263                         goto out_inval;
1264                 ucmd_get_ordered(ucmd);
1265                 ucmd->background_exec = 1;
1266                 TRACE_DBG("Background ucmd %p", ucmd);
1267                 goto out_compl;
1268         } else
1269                 goto out_inval;
1270
1271         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1272                 ereply->status, ereply->resp_data_len);
1273
1274          if (ereply->resp_data_len != 0) {
1275                 if (ucmd->ubuff == 0) {
1276                         int pages, rc;
1277                         if (unlikely(ereply->pbuf == 0))
1278                                 goto out_busy;
1279                         if (ucmd->buff_cached) {
1280                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1281                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1282                                                 "page aligned", ereply->pbuf);
1283                                         goto out_hwerr;
1284                                 }
1285                                 pages = cmd->sg_cnt;
1286                         } else
1287                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1288                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1289                         if ((rc != 0) || (ucmd->ubuff == 0))
1290                                 goto out_compl;
1291
1292                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1293                         if (unlikely(rc != 0))
1294                                 goto out_busy;
1295                 } else
1296                         dev_user_flush_dcache(ucmd);
1297                 cmd->may_need_dma_sync = 1;
1298                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1299         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1300                 if (ucmd->ubuff == 0)
1301                         cmd->resp_data_len = ereply->resp_data_len;
1302                 else
1303                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1304         }
1305
1306         cmd->status = ereply->status;
1307         if (ereply->sense_len != 0) {
1308                 res = scst_alloc_sense(cmd, 0);
1309                 if (res != 0)
1310                         goto out_compl;
1311                 res = copy_from_user(cmd->sense,
1312                         (void __user *)(unsigned long)ereply->psense_buffer,
1313                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1314                                 (unsigned int)ereply->sense_len));
1315                 if (res < 0) {
1316                         PRINT_ERROR("%s", "Unable to get sense data");
1317                         goto out_hwerr_res_set;
1318                 }
1319         }
1320
1321 out_compl:
1322         cmd->completed = 1;
1323         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1324         /* !! At this point cmd can be already freed !! */
1325
1326 out:
1327         TRACE_EXIT_RES(res);
1328         return res;
1329
1330 out_inval:
1331         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1332                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1333         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1334
1335 out_hwerr:
1336         res = -EINVAL;
1337
1338 out_hwerr_res_set:
1339         if (ucmd->background_exec) {
1340                 ucmd_put(ucmd);
1341                 goto out;
1342         } else {
1343                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1344                 goto out_compl;
1345         }
1346
1347 out_busy:
1348         scst_set_busy(cmd);
1349         goto out_compl;
1350 }
1351
1352 static int dev_user_process_reply(struct scst_user_dev *dev,
1353         struct scst_user_reply_cmd *reply)
1354 {
1355         int res = 0;
1356         struct scst_user_cmd *ucmd;
1357         int state;
1358
1359         TRACE_ENTRY();
1360
1361         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1362
1363         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1364         if (unlikely(ucmd == NULL)) {
1365                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1366                 res = -ESRCH;
1367                 goto out_unlock;
1368         }
1369
1370         if (unlikely(ucmd_get_check(ucmd))) {
1371                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1372                 res = -ESRCH;
1373                 goto out_unlock;
1374         }
1375
1376         if (ucmd->background_exec) {
1377                 state = UCMD_STATE_EXECING;
1378                 goto unlock_process;
1379         }
1380
1381         if (unlikely(ucmd->this_state_unjammed)) {
1382                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1383                         ucmd);
1384                 goto out_unlock_put;
1385         }
1386
1387         if (unlikely(!ucmd->sent_to_user)) {
1388                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1389                         "state %x", ucmd, ucmd->state);
1390                 res = -EINVAL;
1391                 goto out_unlock_put;
1392         }
1393
1394         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1395                 goto out_wrong_state;
1396
1397         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1398                 goto out_wrong_state;
1399
1400         state = ucmd->state;
1401         ucmd->sent_to_user = 0;
1402
1403 unlock_process:
1404         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1405
1406         switch (state) {
1407         case UCMD_STATE_PARSING:
1408                 res = dev_user_process_reply_parse(ucmd, reply);
1409                 break;
1410
1411         case UCMD_STATE_BUF_ALLOCING:
1412                 res = dev_user_process_reply_alloc(ucmd, reply);
1413                 break;
1414
1415         case UCMD_STATE_EXECING:
1416                 res = dev_user_process_reply_exec(ucmd, reply);
1417                 break;
1418
1419         case UCMD_STATE_ON_FREEING:
1420                 res = dev_user_process_reply_on_free(ucmd);
1421                 break;
1422
1423         case UCMD_STATE_ON_CACHE_FREEING:
1424                 res = dev_user_process_reply_on_cache_free(ucmd);
1425                 break;
1426
1427         case UCMD_STATE_TM_EXECING:
1428                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1429                 break;
1430
1431         case UCMD_STATE_ATTACH_SESS:
1432         case UCMD_STATE_DETACH_SESS:
1433                 res = dev_user_process_reply_sess(ucmd, reply->result);
1434                 break;
1435
1436         default:
1437                 sBUG();
1438                 break;
1439         }
1440
1441 out_put:
1442         ucmd_put(ucmd);
1443
1444 out:
1445         TRACE_EXIT_RES(res);
1446         return res;
1447
1448 out_wrong_state:
1449         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1450                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1451                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1452                 reply->subcode, ucmd->user_cmd.subcode);
1453         res = -EINVAL;
1454         dev_user_unjam_cmd(ucmd, 0, NULL);
1455
1456 out_unlock_put:
1457         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1458         goto out_put;
1459
1460 out_unlock:
1461         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1462         goto out;
1463 }
1464
1465 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1466 {
1467         int res = 0;
1468         struct scst_user_dev *dev;
1469         struct scst_user_reply_cmd reply;
1470
1471         TRACE_ENTRY();
1472
1473         mutex_lock(&dev_priv_mutex);
1474         dev = (struct scst_user_dev *)file->private_data;
1475         res = dev_user_check_reg(dev);
1476         if (res != 0) {
1477                 mutex_unlock(&dev_priv_mutex);
1478                 goto out;
1479         }
1480         down_read(&dev->dev_rwsem);
1481         mutex_unlock(&dev_priv_mutex);
1482
1483         res = copy_from_user(&reply, arg, sizeof(reply));
1484         if (res < 0)
1485                 goto out_up;
1486
1487         TRACE_MGMT_DBG("Reply for dev %s", dev->name);
1488
1489         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1490
1491         res = dev_user_process_reply(dev, &reply);
1492         if (res < 0)
1493                 goto out_up;
1494
1495 out_up:
1496         up_read(&dev->dev_rwsem);
1497
1498 out:
1499         TRACE_EXIT_RES(res);
1500         return res;
1501 }
1502
1503 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1504 {
1505         int res = 0;
1506
1507         TRACE_ENTRY();
1508
1509         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1510                 struct scst_cmd *cmd = list_entry(
1511                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1512                         cmd_list_entry);
1513                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1514                 list_del(&cmd->cmd_list_entry);
1515                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1516                 scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT |
1517                                                  SCST_CONTEXT_PROCESSABLE);
1518                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1519                 res++;
1520         }
1521
1522         TRACE_EXIT_RES(res);
1523         return res;
1524 }
1525
1526 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1527 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1528 {
1529         struct scst_user_cmd *u;
1530
1531 again:
1532         u = NULL;
1533         if (!list_empty(cmd_list)) {
1534                 u = list_entry(cmd_list->next, typeof(*u), ready_cmd_list_entry);
1535
1536                 TRACE_DBG("Found ready ucmd %p", u);
1537                 list_del(&u->ready_cmd_list_entry);
1538
1539                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1540
1541                 if (u->cmd != NULL) {
1542                         if (u->state == UCMD_STATE_EXECING) {
1543                                 struct scst_user_dev *dev = u->dev;
1544                                 int rc;
1545
1546                                 EXTRACHECKS_BUG_ON(u->jammed);
1547
1548                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1549
1550                                 rc = scst_check_local_events(u->cmd);
1551                                 if (unlikely(rc != 0)) {
1552                                         u->cmd->scst_cmd_done(u->cmd,
1553                                                 SCST_CMD_STATE_DEFAULT);
1554                                         /*
1555                                          * !! At this point cmd & u can be !!
1556                                          * !! already freed                !!
1557                                          */
1558                                         spin_lock_irq(
1559                                                 &dev->cmd_lists.cmd_list_lock);
1560                                         goto again;
1561                                 }
1562
1563                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1564                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1565                                         &u->cmd->cmd_flags))) {
1566                                 switch (u->state) {
1567                                 case UCMD_STATE_PARSING:
1568                                 case UCMD_STATE_BUF_ALLOCING:
1569                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1570                                         dev_user_unjam_cmd(u, 0, NULL);
1571                                         goto again;
1572                                 case UCMD_STATE_EXECING:
1573                                         EXTRACHECKS_BUG_ON(1);
1574                                 }
1575                         }
1576                 }
1577                 u->sent_to_user = 1;
1578                 u->seen_by_user = 1;
1579         }
1580         return u;
1581 }
1582
1583 static inline int test_cmd_lists(struct scst_user_dev *dev)
1584 {
1585         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1586                   !list_empty(&dev->ready_cmd_list) ||
1587                   !dev->blocking || dev->cleanup_done ||
1588                   signal_pending(current);
1589         return res;
1590 }
1591
1592 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1593 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1594         struct scst_user_cmd **ucmd)
1595 {
1596         int res = 0;
1597         wait_queue_t wait;
1598
1599         TRACE_ENTRY();
1600
1601         init_waitqueue_entry(&wait, current);
1602
1603         while (1) {
1604                 if (!test_cmd_lists(dev)) {
1605                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1606                                 &wait);
1607                         for (;;) {
1608                                 set_current_state(TASK_INTERRUPTIBLE);
1609                                 if (test_cmd_lists(dev))
1610                                         break;
1611                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1612                                 schedule();
1613                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1614                         }
1615                         set_current_state(TASK_RUNNING);
1616                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1617                                 &wait);
1618                 }
1619
1620                 dev_user_process_scst_commands(dev);
1621
1622                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1623                 if (*ucmd != NULL)
1624                         break;
1625
1626                 if (!dev->blocking || dev->cleanup_done) {
1627                         res = -EAGAIN;
1628                         TRACE_DBG("No ready commands, returning %d", res);
1629                         break;
1630                 }
1631
1632                 if (signal_pending(current)) {
1633                         res = -EINTR;
1634                         TRACE_DBG("Signal pending, returning %d", res);
1635                         break;
1636                 }
1637         }
1638
1639         TRACE_EXIT_RES(res);
1640         return res;
1641 }
1642
1643 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1644 {
1645         int res = 0;
1646         struct scst_user_dev *dev;
1647         struct scst_user_get_cmd *cmd;
1648         struct scst_user_reply_cmd *reply;
1649         struct scst_user_cmd *ucmd;
1650         uint64_t ureply;
1651
1652         TRACE_ENTRY();
1653
1654         mutex_lock(&dev_priv_mutex);
1655         dev = (struct scst_user_dev *)file->private_data;
1656         res = dev_user_check_reg(dev);
1657         if (res != 0) {
1658                 mutex_unlock(&dev_priv_mutex);
1659                 goto out;
1660         }
1661         down_read(&dev->dev_rwsem);
1662         mutex_unlock(&dev_priv_mutex);
1663
1664         res = copy_from_user(&ureply, arg, sizeof(ureply));
1665         if (res < 0)
1666                 goto out_up;
1667
1668         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1669                 dev->name);
1670
1671         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1672         if (cmd == NULL) {
1673                 res = -ENOMEM;
1674                 goto out_up;
1675         }
1676
1677         if (ureply != 0) {
1678                 unsigned long u = (unsigned long)ureply;
1679                 reply = (struct scst_user_reply_cmd *)cmd;
1680                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1681                 if (res < 0)
1682                         goto out_free;
1683
1684                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1685
1686                 res = dev_user_process_reply(dev, reply);
1687                 if (res < 0)
1688                         goto out_free;
1689         }
1690
1691         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1692         res = dev_user_get_next_cmd(dev, &ucmd);
1693         if (res == 0) {
1694                 *cmd = ucmd->user_cmd;
1695                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1696                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1697                 res = copy_to_user(arg, cmd, sizeof(*cmd));
1698         } else
1699                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1700
1701 out_free:
1702         kmem_cache_free(user_get_cmd_cachep, cmd);
1703
1704 out_up:
1705         up_read(&dev->dev_rwsem);
1706
1707 out:
1708         TRACE_EXIT_RES(res);
1709         return res;
1710 }
1711
1712 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1713         unsigned long arg)
1714 {
1715         long res;
1716
1717         TRACE_ENTRY();
1718
1719         switch (cmd) {
1720         case SCST_USER_REPLY_AND_GET_CMD:
1721                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1722                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1723                 break;
1724
1725         case SCST_USER_REPLY_CMD:
1726                 TRACE_DBG("%s", "REPLY_CMD");
1727                 res = dev_user_reply_cmd(file, (void __user *)arg);
1728                 break;
1729
1730         case SCST_USER_REGISTER_DEVICE:
1731         {
1732                 struct scst_user_dev_desc *dev_desc;
1733                 TRACE_DBG("%s", "REGISTER_DEVICE");
1734                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1735                 if (dev_desc == NULL) {
1736                         res = -ENOMEM;
1737                         goto out;
1738                 }
1739                 res = copy_from_user(dev_desc, (void __user *)arg, sizeof(*dev_desc));
1740                 if (res < 0) {
1741                         kfree(dev_desc);
1742                         goto out;
1743                 }
1744                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1745                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1746                 res = dev_user_register_dev(file, dev_desc);
1747                 kfree(dev_desc);
1748                 break;
1749         }
1750
1751         case SCST_USER_SET_OPTIONS:
1752         {
1753                 struct scst_user_opt opt;
1754                 TRACE_DBG("%s", "SET_OPTIONS");
1755                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1756                 if (res < 0)
1757                         goto out;
1758                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1759                 res = dev_user_set_opt(file, &opt);
1760                 break;
1761         }
1762
1763         case SCST_USER_GET_OPTIONS:
1764                 TRACE_DBG("%s", "GET_OPTIONS");
1765                 res = dev_user_get_opt(file, (void __user *)arg);
1766                 break;
1767
1768         default:
1769                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1770                 res = -EINVAL;
1771                 goto out;
1772         }
1773
1774 out:
1775         TRACE_EXIT_RES(res);
1776         return res;
1777 }
1778
1779 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1780 {
1781         int res = 0;
1782         struct scst_user_dev *dev;
1783
1784         TRACE_ENTRY();
1785
1786         mutex_lock(&dev_priv_mutex);
1787         dev = (struct scst_user_dev *)file->private_data;
1788         res = dev_user_check_reg(dev);
1789         if (res != 0) {
1790                 mutex_unlock(&dev_priv_mutex);
1791                 goto out;
1792         }
1793         down_read(&dev->dev_rwsem);
1794         mutex_unlock(&dev_priv_mutex);
1795
1796         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1797
1798         if (!list_empty(&dev->ready_cmd_list) ||
1799             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1800                 res |= POLLIN | POLLRDNORM;
1801                 goto out_unlock;
1802         }
1803
1804         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1805
1806         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
1807         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1808         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
1809
1810         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1811
1812         if (!list_empty(&dev->ready_cmd_list) ||
1813             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1814                 res |= POLLIN | POLLRDNORM;
1815                 goto out_unlock;
1816         }
1817
1818 out_unlock:
1819         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1820
1821         up_read(&dev->dev_rwsem);
1822
1823 out:
1824         TRACE_EXIT_HRES(res);
1825         return res;
1826 }
1827
1828 /*
1829  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reaquire.
1830  */
1831 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1832         unsigned long *flags)
1833 {
1834         int state = ucmd->state;
1835         struct scst_user_dev *dev = ucmd->dev;
1836
1837         TRACE_ENTRY();
1838
1839         if (ucmd->this_state_unjammed)
1840                 goto out;
1841
1842         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1843                 state);
1844
1845         ucmd->jammed = 1;
1846         ucmd->this_state_unjammed = 1;
1847         ucmd->sent_to_user = 0;
1848
1849         switch (state) {
1850         case UCMD_STATE_PARSING:
1851         case UCMD_STATE_BUF_ALLOCING:
1852                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1853                         ucmd->aborted = 1;
1854                 else {
1855                         if (busy)
1856                                 scst_set_busy(ucmd->cmd);
1857                         else
1858                                 scst_set_cmd_error(ucmd->cmd,
1859                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1860                 }
1861                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1862
1863                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1864                 list_add(&ucmd->cmd->cmd_list_entry,
1865                         &ucmd->cmd->cmd_lists->active_cmd_list);
1866                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1867                 break;
1868
1869         case UCMD_STATE_EXECING:
1870                 if (flags != NULL)
1871                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1872                 else
1873                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1874
1875                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1876
1877                 if (test_bit(SCST_CMD_ABORTED,  &ucmd->cmd->cmd_flags))
1878                         ucmd->aborted = 1;
1879                 else {
1880                         if (busy)
1881                                 scst_set_busy(ucmd->cmd);
1882                         else
1883                                 scst_set_cmd_error(ucmd->cmd,
1884                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1885                 }
1886
1887                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT);
1888                 /* !! At this point cmd and ucmd can be already freed !! */
1889
1890                 if (flags != NULL)
1891                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1892                 else
1893                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1894                 break;
1895
1896         case UCMD_STATE_ON_FREEING:
1897         case UCMD_STATE_ON_CACHE_FREEING:
1898         case UCMD_STATE_TM_EXECING:
1899         case UCMD_STATE_ATTACH_SESS:
1900         case UCMD_STATE_DETACH_SESS:
1901         {
1902                 if (flags != NULL)
1903                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1904                 else
1905                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1906
1907                 switch (state) {
1908                 case UCMD_STATE_ON_FREEING:
1909                         dev_user_process_reply_on_free(ucmd);
1910                         break;
1911
1912                 case UCMD_STATE_ON_CACHE_FREEING:
1913                         dev_user_process_reply_on_cache_free(ucmd);
1914                         break;
1915
1916                 case UCMD_STATE_TM_EXECING:
1917                         dev_user_process_reply_tm_exec(ucmd, SCST_MGMT_STATUS_FAILED);
1918                         break;
1919
1920                 case UCMD_STATE_ATTACH_SESS:
1921                 case UCMD_STATE_DETACH_SESS:
1922                         dev_user_process_reply_sess(ucmd, -EFAULT);
1923                         break;
1924                 }
1925
1926                 if (flags != NULL)
1927                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1928                 else
1929                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1930                 break;
1931         }
1932
1933         default:
1934                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1935                 sBUG();
1936                 break;
1937         }
1938
1939 out:
1940         TRACE_EXIT();
1941         return;
1942 }
1943
1944 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1945 {
1946         int i;
1947         struct scst_user_cmd *ucmd;
1948
1949         TRACE_ENTRY();
1950
1951         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1952
1953         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1954
1955 repeat:
1956         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1957                 struct list_head *head = &dev->ucmd_hash[i];
1958
1959                 list_for_each_entry(ucmd, head, hash_list_entry) {
1960                         if (!ucmd->sent_to_user)
1961                                 continue;
1962
1963                         if (ucmd_get_check(ucmd))
1964                                 continue;
1965
1966                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1967                                 ucmd->state, ucmd->cmd);
1968
1969                         dev_user_unjam_cmd(ucmd, 0, NULL);
1970
1971                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1972                         ucmd_put(ucmd);
1973                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1974
1975                         goto repeat;
1976                 }
1977         }
1978
1979         if (dev_user_process_scst_commands(dev) != 0)
1980                 goto repeat;
1981
1982         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1983
1984         TRACE_EXIT();
1985         return;
1986 }
1987
1988 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
1989         int status)
1990 {
1991         int res = 0;
1992
1993         TRACE_ENTRY();
1994
1995         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
1996                 ucmd->user_cmd.tm_cmd.fn, status);
1997
1998         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
1999                 /*
2000                  * It is possible that user space seen TM cmd before cmd
2001                  * to abort or will never see it at all, because it was
2002                  * aborted on the way there. So, it is safe to return
2003                  * success instead, because, if there is the TM cmd at this
2004                  * point, then the cmd to abort apparrently does exist.
2005                  */
2006                 status = SCST_MGMT_STATUS_SUCCESS;
2007         }
2008
2009         scst_async_mcmd_completed(ucmd->mcmd, status);
2010
2011         ucmd_put(ucmd);
2012
2013         TRACE_EXIT_RES(res);
2014         return res;
2015 }
2016
2017 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2018 {
2019         struct scst_user_cmd *ucmd;
2020         unsigned long flags;
2021
2022         TRACE_ENTRY();
2023
2024         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2025 again:
2026         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2027                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2028                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2029                         switch (ucmd->state) {
2030                         case UCMD_STATE_PARSING:
2031                         case UCMD_STATE_BUF_ALLOCING:
2032                         case UCMD_STATE_EXECING:
2033                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2034                                 list_del(&ucmd->ready_cmd_list_entry);
2035                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2036                                 goto again;
2037                         }
2038                 }
2039         }
2040
2041         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2042
2043         TRACE_EXIT();
2044         return;
2045 }
2046
2047 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2048         struct scst_tgt_dev *tgt_dev)
2049 {
2050         struct scst_user_cmd *ucmd;
2051         struct scst_user_dev *dev = (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2052         struct scst_user_cmd *ucmd_to_abort = NULL;
2053
2054         TRACE_ENTRY();
2055
2056         /*
2057          * In the used approach we don't do anything with hung devices, which
2058          * stopped responding and/or have stuck commands. We forcedly abort such
2059          * commands only if they not yet sent to the user space or if the device
2060          * is getting unloaded, e.g. if its handler program gets killed. This is
2061          * because it's pretty hard to distinguish between stuck and temporary
2062          * overloaded states of the device. There are several reasons for that:
2063          *
2064          * 1. Some commands need a lot of time to complete (several hours),
2065          *    so for an impatient user such command(s) will always look as
2066          *    stuck.
2067          *
2068          * 2. If we forcedly abort, i.e. abort before it's actually completed
2069          *    in the user space, just one command, we will have to put the whole
2070          *    device offline until we are sure that no more previously aborted
2071          *    commands will get executed. Otherwise, we might have a possibility
2072          *    for data corruption, when aborted and reported as completed
2073          *    command actually gets executed *after* new commands sent
2074          *    after the force abort was done. Many journaling file systems and
2075          *    databases use "provide required commands order via queue draining"
2076          *    approach and not putting the whole device offline after the forced
2077          *    abort will break it. This makes our decision, if a command stuck
2078          *    or not, cost a lot.
2079          *
2080          * So, we leave policy definition if a device stuck or not to
2081          * the user space and simply let all commands live until they are
2082          * completed or their devices get closed/killed. This approach is very
2083          * much OK, but can affect management commands, which need activity
2084          * suspending via scst_suspend_activity() function such as devices or
2085          * targets registration/removal. But during normal life such commands
2086          * should be rare. Plus, when possible, scst_suspend_activity() will
2087          * return after timeout EBUSY status to allow caller to not stuck
2088          * forever as well.
2089          *
2090          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2091          * stuck commands would affect only related devices.
2092          */
2093
2094         dev_user_abort_ready_commands(dev);
2095
2096         /* We can't afford missing TM command due to memory shortage */
2097         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2098
2099         ucmd->user_cmd.cmd_h = ucmd->h;
2100         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2101         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2102         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2103         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2104         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2105
2106         if (mcmd->cmd_to_abort != NULL) {
2107                 ucmd_to_abort = (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2108                 if (ucmd_to_abort != NULL)
2109                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2110         }
2111
2112         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2113                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2114                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2115                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2116
2117         ucmd->mcmd = mcmd;
2118         ucmd->state = UCMD_STATE_TM_EXECING;
2119
2120         scst_prepare_async_mcmd(mcmd);
2121
2122         dev_user_add_to_ready(ucmd);
2123
2124         TRACE_EXIT();
2125         return SCST_DEV_TM_NOT_COMPLETED;
2126 }
2127
2128 static int dev_user_attach(struct scst_device *sdev)
2129 {
2130         int res = 0;
2131         struct scst_user_dev *dev = NULL, *d;
2132
2133         TRACE_ENTRY();
2134
2135         spin_lock(&dev_list_lock);
2136         list_for_each_entry(d, &dev_list, dev_list_entry) {
2137                 if (strcmp(d->name, sdev->virt_name) == 0) {
2138                         dev = d;
2139                         break;
2140                 }
2141         }
2142         spin_unlock(&dev_list_lock);
2143         if (dev == NULL) {
2144                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2145                 res = -EINVAL;
2146                 goto out;
2147         }
2148
2149         sdev->p_cmd_lists = &dev->cmd_lists;
2150         sdev->dh_priv = dev;
2151         sdev->tst = dev->tst;
2152         sdev->queue_alg = dev->queue_alg;
2153         sdev->swp = dev->swp;
2154         sdev->tas = dev->tas;
2155         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2156
2157         dev->sdev = sdev;
2158
2159         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2160                 dev->name);
2161
2162 out:
2163         TRACE_EXIT();
2164         return res;
2165 }
2166
2167 static void dev_user_detach(struct scst_device *sdev)
2168 {
2169         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2170
2171         TRACE_ENTRY();
2172
2173         TRACE_DBG("virt_id %d", sdev->virt_id);
2174
2175         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2176                 dev->name);
2177
2178         /* dev will be freed by the caller */
2179         sdev->dh_priv = NULL;
2180         dev->sdev = NULL;
2181
2182         TRACE_EXIT();
2183         return;
2184 }
2185
2186 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2187 {
2188         int res = 0;
2189         unsigned long flags;
2190
2191         TRACE_ENTRY();
2192
2193         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2194
2195         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2196
2197         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2198                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2199                 ucmd->result = status;
2200         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2201                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2202         } else
2203                 sBUG();
2204
2205         if (ucmd->cmpl != NULL)
2206                 complete_all(ucmd->cmpl);
2207
2208         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2209
2210         ucmd_put(ucmd);
2211
2212         TRACE_EXIT_RES(res);
2213         return res;
2214 }
2215
2216 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2217 {
2218         struct scst_user_dev *dev =
2219                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2220         int res = 0, rc;
2221         struct scst_user_cmd *ucmd;
2222         DECLARE_COMPLETION_ONSTACK(cmpl);
2223
2224         TRACE_ENTRY();
2225
2226         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2227         if (ucmd == NULL)
2228                 goto out_nomem;
2229
2230         ucmd->cmpl = &cmpl;
2231
2232         ucmd->user_cmd.cmd_h = ucmd->h;
2233         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2234         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2235         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2236         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2237         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2238         strncpy(ucmd->user_cmd.sess.initiator_name,
2239                 tgt_dev->sess->initiator_name,
2240                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2241         ucmd->user_cmd.sess.initiator_name[
2242                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2243
2244         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2245                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2246                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2247                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2248                 ucmd->user_cmd.sess.initiator_name);
2249
2250         ucmd->state = UCMD_STATE_ATTACH_SESS;
2251
2252         ucmd_get(ucmd);
2253
2254         dev_user_add_to_ready(ucmd);
2255
2256         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2257         if (rc > 0)
2258                 res = ucmd->result;
2259         else {
2260                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2261                 res = -EFAULT;
2262         }
2263
2264         sBUG_ON(irqs_disabled());
2265
2266         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2267         ucmd->cmpl = NULL;
2268         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2269
2270         ucmd_put(ucmd);
2271
2272 out:
2273         TRACE_EXIT_RES(res);
2274         return res;
2275
2276 out_nomem:
2277         res = -ENOMEM;
2278         goto out;
2279 }
2280
2281 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2282 {
2283         struct scst_user_dev *dev =
2284                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2285         struct scst_user_cmd *ucmd;
2286
2287         TRACE_ENTRY();
2288
2289         /*
2290          * We can't miss TM command due to memory shortage, because it might
2291          * lead to a memory leak in the user space handler.
2292          */
2293         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2294         if (ucmd == NULL)
2295                 goto out;
2296
2297         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2298                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2299
2300         ucmd->user_cmd.cmd_h = ucmd->h;
2301         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2302         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2303
2304         ucmd->state = UCMD_STATE_DETACH_SESS;
2305
2306         dev_user_add_to_ready(ucmd);
2307
2308 out:
2309         TRACE_EXIT();
2310         return;
2311 }
2312
2313 /* No locks are needed, but the activity must be suspended */
2314 static void dev_user_setup_functions(struct scst_user_dev *dev)
2315 {
2316         TRACE_ENTRY();
2317
2318         dev->devtype.parse = dev_user_parse;
2319         dev->devtype.dev_done = NULL;
2320
2321         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2322                 switch (dev->devtype.type) {
2323                 case TYPE_DISK:
2324                         dev->generic_parse = scst_sbc_generic_parse;
2325                         dev->devtype.dev_done = dev_user_disk_done;
2326                         break;
2327
2328                 case TYPE_TAPE:
2329                         dev->generic_parse = scst_tape_generic_parse;
2330                         dev->devtype.dev_done = dev_user_tape_done;
2331                         break;
2332
2333                 case TYPE_MOD:
2334                         dev->generic_parse = scst_modisk_generic_parse;
2335                         dev->devtype.dev_done = dev_user_disk_done;
2336                         break;
2337
2338                 case TYPE_ROM:
2339                         dev->generic_parse = scst_cdrom_generic_parse;
2340                         dev->devtype.dev_done = dev_user_disk_done;
2341                         break;
2342
2343                 case TYPE_MEDIUM_CHANGER:
2344                         dev->generic_parse = scst_changer_generic_parse;
2345                         break;
2346
2347                 case TYPE_PROCESSOR:
2348                         dev->generic_parse = scst_processor_generic_parse;
2349                         break;
2350
2351                 case TYPE_RAID:
2352                         dev->generic_parse = scst_raid_generic_parse;
2353                         break;
2354
2355                 default:
2356                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2357                                 "for it", dev->devtype.type);
2358                         dev->parse_type = SCST_USER_PARSE_CALL;
2359                         break;
2360                 }
2361         } else {
2362                 dev->generic_parse = NULL;
2363                 dev->devtype.dev_done = NULL;
2364         }
2365
2366         TRACE_EXIT();
2367         return;
2368 }
2369
2370 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2371 {
2372         char ver[sizeof(DEV_USER_VERSION)+1];
2373         int res;
2374
2375         res = copy_from_user(ver, (void __user *)(unsigned long)dev_desc->version_str,
2376                                 sizeof(ver));
2377         if (res < 0) {
2378                 PRINT_ERROR("%s", "Unable to get version string");
2379                 goto out;
2380         }
2381         ver[sizeof(ver)-1] = '\0';
2382
2383         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2384                 /* ->name already 0-terminated in dev_user_ioctl() */
2385                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2386                         dev_desc->name, ver);
2387                 res = -EINVAL;
2388                 goto out;
2389         }
2390
2391 out:
2392         return res;
2393 }
2394
2395 static int dev_user_register_dev(struct file *file,
2396         const struct scst_user_dev_desc *dev_desc)
2397 {
2398         int res = -ENOMEM, i;
2399         struct scst_user_dev *dev, *d;
2400         int block;
2401
2402         TRACE_ENTRY();
2403
2404         res = dev_user_check_version(dev_desc);
2405         if (res != 0)
2406                 goto out;
2407
2408         switch (dev_desc->type) {
2409         case TYPE_DISK:
2410         case TYPE_ROM:
2411         case TYPE_MOD:
2412                 if (dev_desc->block_size == 0) {
2413                         PRINT_ERROR("Wrong block size %d", dev_desc->block_size);
2414                         res = -EINVAL;
2415                         goto out;
2416                 }
2417                 block = scst_calc_block_shift(dev_desc->block_size);
2418                 if (block == -1) {
2419                         res = -EINVAL;
2420                         goto out;
2421                 }
2422                 break;
2423         default:
2424                 block = dev_desc->block_size;
2425                 break;
2426         }
2427
2428         if (!try_module_get(THIS_MODULE)) {
2429                 PRINT_ERROR("%s", "Fail to get module");
2430                 goto out;
2431         }
2432
2433         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2434         if (dev == NULL)
2435                 goto out_put;
2436
2437         init_rwsem(&dev->dev_rwsem);
2438         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2439         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2440         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2441         INIT_LIST_HEAD(&dev->ready_cmd_list);
2442         if (file->f_flags & O_NONBLOCK) {
2443                 TRACE_DBG("%s", "Non-blocking operations");
2444                 dev->blocking = 0;
2445         } else
2446                 dev->blocking = 1;
2447         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2448                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2449
2450         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2451         dev->name[sizeof(dev->name)-1] = '\0';
2452
2453         scst_init_mem_lim(&dev->udev_mem_lim);
2454
2455         /*
2456          * We don't use clustered pool, since it implies pages reordering,
2457          * which isn't possible with user space supplied buffers. Although
2458          * it's still possible to cluster pages by the tail of each other,
2459          * seems it doesn't worth the effort.
2460          */
2461         dev->pool = sgv_pool_create(dev->name, 0);
2462         if (dev->pool == NULL)
2463                 goto out_put;
2464         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2465                 dev_user_free_sg_entries);
2466
2467         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2468                 dev->name);
2469         dev->devtype.type = dev_desc->type;
2470         dev->devtype.threads_num = -1;
2471         dev->devtype.parse_atomic = 1;
2472         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2473         dev->devtype.dev_done_atomic = 1;
2474         dev->devtype.no_proc = 1;
2475         dev->devtype.attach = dev_user_attach;
2476         dev->devtype.detach = dev_user_detach;
2477         dev->devtype.attach_tgt = dev_user_attach_tgt;
2478         dev->devtype.detach_tgt = dev_user_detach_tgt;
2479         dev->devtype.exec = dev_user_exec;
2480         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2481         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2482
2483         init_completion(&dev->cleanup_cmpl);
2484         dev->block = block;
2485         dev->def_block = block;
2486
2487         res = __dev_user_set_opt(dev, &dev_desc->opt);
2488
2489         TRACE_MEM("dev %p, name %s", dev, dev->name);
2490
2491         spin_lock(&dev_list_lock);
2492
2493         list_for_each_entry(d, &dev_list, dev_list_entry) {
2494                 if (strcmp(d->name, dev->name) == 0) {
2495                         PRINT_ERROR("Device %s already exist",
2496                                 dev->name);
2497                         res = -EEXIST;
2498                         spin_unlock(&dev_list_lock);
2499                         goto out_free;
2500                 }
2501         }
2502
2503         list_add_tail(&dev->dev_list_entry, &dev_list);
2504
2505         spin_unlock(&dev_list_lock);
2506
2507         if (res != 0)
2508                 goto out_del_free;
2509
2510         res = scst_register_virtual_dev_driver(&dev->devtype);
2511         if (res < 0)
2512                 goto out_del_free;
2513
2514         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2515         if (dev->virt_id < 0) {
2516                 res = dev->virt_id;
2517                 goto out_unreg_handler;
2518         }
2519
2520         mutex_lock(&dev_priv_mutex);
2521         if (file->private_data != NULL) {
2522                 mutex_unlock(&dev_priv_mutex);
2523                 PRINT_ERROR("%s", "Device already registered");
2524                 res = -EINVAL;
2525                 goto out_unreg_drv;
2526         }
2527         file->private_data = dev;
2528         mutex_unlock(&dev_priv_mutex);
2529
2530 out:
2531         TRACE_EXIT_RES(res);
2532         return res;
2533
2534 out_unreg_drv:
2535         scst_unregister_virtual_device(dev->virt_id);
2536
2537 out_unreg_handler:
2538         scst_unregister_virtual_dev_driver(&dev->devtype);
2539
2540 out_del_free:
2541         spin_lock(&dev_list_lock);
2542         list_del(&dev->dev_list_entry);
2543         spin_unlock(&dev_list_lock);
2544
2545 out_free:
2546         sgv_pool_destroy(dev->pool);
2547         kfree(dev);
2548         goto out_put;
2549
2550 out_put:
2551         module_put(THIS_MODULE);
2552         goto out;
2553 }
2554
2555 static int __dev_user_set_opt(struct scst_user_dev *dev,
2556         const struct scst_user_opt *opt)
2557 {
2558         int res = 0;
2559
2560         TRACE_ENTRY();
2561
2562         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2563                 "memory_reuse_type %x, partial_transfers_type %x, "
2564                 "partial_len %d", dev->name, opt->parse_type,
2565                 opt->on_free_cmd_type, opt->memory_reuse_type,
2566                 opt->partial_transfers_type, opt->partial_len);
2567
2568         if ((opt->parse_type > SCST_USER_MAX_PARSE_OPT) ||
2569             (opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT) ||
2570             (opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT) ||
2571             (opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT)) {
2572                 PRINT_ERROR("%s", "Invalid option");
2573                 res = -EINVAL;
2574                 goto out;
2575         }
2576
2577         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2578              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2579             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2580              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2581             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2582                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x, "
2583                         "tas %x, has_own_order_mgmt %x)", opt->tst,
2584                         opt->queue_alg, opt->swp, opt->tas, opt->has_own_order_mgmt);
2585                 res = -EINVAL;
2586                 goto out;
2587         }
2588
2589         dev->parse_type = opt->parse_type;
2590         dev->on_free_cmd_type = opt->on_free_cmd_type;
2591         dev->memory_reuse_type = opt->memory_reuse_type;
2592         dev->partial_transfers_type = opt->partial_transfers_type;
2593         dev->partial_len = opt->partial_len;
2594
2595         dev->tst = opt->tst;
2596         dev->queue_alg = opt->queue_alg;
2597         dev->swp = opt->swp;
2598         dev->tas = opt->tas;
2599         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2600         if (dev->sdev != NULL) {
2601                 dev->sdev->tst = opt->tst;
2602                 dev->sdev->queue_alg = opt->queue_alg;
2603                 dev->sdev->swp = opt->swp;
2604                 dev->sdev->tas = opt->tas;
2605                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2606         }
2607
2608         dev_user_setup_functions(dev);
2609
2610 out:
2611         TRACE_EXIT_RES(res);
2612         return res;
2613 }
2614
2615 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2616 {
2617         int res = 0;
2618         struct scst_user_dev *dev;
2619
2620         TRACE_ENTRY();
2621
2622         mutex_lock(&dev_priv_mutex);
2623         dev = (struct scst_user_dev *)file->private_data;
2624         res = dev_user_check_reg(dev);
2625         if (res != 0) {
2626                 mutex_unlock(&dev_priv_mutex);
2627                 goto out;
2628         }
2629         down_write(&dev->dev_rwsem);
2630         mutex_unlock(&dev_priv_mutex);
2631
2632         res = scst_suspend_activity(true);
2633         if (res != 0)
2634                 goto out;
2635
2636         res = __dev_user_set_opt(dev, opt);
2637
2638         scst_resume_activity();
2639
2640         up_write(&dev->dev_rwsem);
2641
2642 out:
2643         TRACE_EXIT_RES(res);
2644         return res;
2645 }
2646
2647 static int dev_user_get_opt(struct file *file, void __user *arg)
2648 {
2649         int res = 0;
2650         struct scst_user_dev *dev;
2651         struct scst_user_opt opt;
2652
2653         TRACE_ENTRY();
2654
2655         mutex_lock(&dev_priv_mutex);
2656         dev = (struct scst_user_dev *)file->private_data;
2657         res = dev_user_check_reg(dev);
2658         if (res != 0) {
2659                 mutex_unlock(&dev_priv_mutex);
2660                 goto out;
2661         }
2662         down_read(&dev->dev_rwsem);
2663         mutex_unlock(&dev_priv_mutex);
2664
2665         opt.parse_type = dev->parse_type;
2666         opt.on_free_cmd_type = dev->on_free_cmd_type;
2667         opt.memory_reuse_type = dev->memory_reuse_type;
2668         opt.partial_transfers_type = dev->partial_transfers_type;
2669         opt.partial_len = dev->partial_len;
2670         opt.tst = dev->tst;
2671         opt.queue_alg = dev->queue_alg;
2672         opt.tas = dev->tas;
2673         opt.swp = dev->swp;
2674         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2675
2676         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2677                 "memory_reuse_type %x, partial_transfers_type %x, "
2678                 "partial_len %d", dev->name, opt.parse_type,
2679                 opt.on_free_cmd_type, opt.memory_reuse_type,
2680                 opt.partial_transfers_type, opt.partial_len);
2681
2682         res = copy_to_user(arg, &opt, sizeof(opt));
2683
2684         up_read(&dev->dev_rwsem);
2685 out:
2686         TRACE_EXIT_RES(res);
2687         return res;
2688 }
2689
2690 static int dev_usr_parse(struct scst_cmd *cmd)
2691 {
2692         sBUG();
2693         return SCST_CMD_STATE_DEFAULT;
2694 }
2695
2696 /* Needed only for /proc support */
2697 #define USR_TYPE {                      \
2698         .name =         DEV_USER_NAME,  \
2699         .type =         -1,             \
2700         .parse =        dev_usr_parse,  \
2701 }
2702
2703 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2704
2705 static int dev_user_release(struct inode *inode, struct file *file)
2706 {
2707         int res = 0;
2708         struct scst_user_dev *dev;
2709
2710         TRACE_ENTRY();
2711
2712         mutex_lock(&dev_priv_mutex);
2713         dev = (struct scst_user_dev *)file->private_data;
2714         if (dev == NULL) {
2715                 mutex_unlock(&dev_priv_mutex);
2716                 goto out;
2717         }
2718         file->private_data = NULL;
2719
2720         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2721
2722         spin_lock(&dev_list_lock);
2723         list_del(&dev->dev_list_entry);
2724         spin_unlock(&dev_list_lock);
2725
2726         mutex_unlock(&dev_priv_mutex);
2727
2728         down_write(&dev->dev_rwsem);
2729
2730         spin_lock(&cleanup_lock);
2731         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2732         spin_unlock(&cleanup_lock);
2733
2734         wake_up(&cleanup_list_waitQ);
2735         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2736
2737         scst_unregister_virtual_device(dev->virt_id);
2738         scst_unregister_virtual_dev_driver(&dev->devtype);
2739
2740         sgv_pool_destroy(dev->pool);
2741
2742         TRACE_DBG("Unregistering finished (dev %p)", dev);
2743
2744         dev->cleanup_done = 1;
2745
2746         wake_up(&cleanup_list_waitQ);
2747         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2748
2749         wait_for_completion(&dev->cleanup_cmpl);
2750
2751         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2752
2753         TRACE_DBG("Releasing completed (dev %p)", dev);
2754
2755         kfree(dev);
2756
2757         module_put(THIS_MODULE);
2758
2759 out:
2760         TRACE_EXIT_RES(res);
2761         return res;
2762 }
2763
2764 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2765 {
2766         struct scst_user_cmd *ucmd;
2767         int rc, res = 1;
2768
2769         TRACE_ENTRY();
2770
2771         dev->blocking = 0;
2772
2773         while (1) {
2774                 TRACE_DBG("Cleanuping dev %p", dev);
2775
2776                 dev_user_unjam_dev(dev);
2777
2778                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2779
2780                 rc = dev_user_get_next_cmd(dev, &ucmd);
2781                 if (rc == 0)
2782                         dev_user_unjam_cmd(ucmd, 1, NULL);
2783
2784                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2785
2786                 if (rc == -EAGAIN) {
2787                         if (dev->cleanup_done)
2788                                 break;
2789                         else {
2790                                 TRACE_DBG("No more commands (dev %p)", dev);
2791                                 goto out;
2792                         }
2793                 }
2794         }
2795
2796 #ifdef CONFIG_SCST_EXTRACHECKS
2797 {
2798         int i;
2799         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2800                 struct list_head *head = &dev->ucmd_hash[i];
2801                 struct scst_user_cmd *ucmd;
2802 again:
2803                 list_for_each_entry(ucmd, head, hash_list_entry) {
2804                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd,
2805                                 ucmd->state, atomic_read(&ucmd->ucmd_ref));
2806                         ucmd_put(ucmd);
2807                         goto again;
2808                 }
2809         }
2810 }
2811 #endif
2812
2813         TRACE_DBG("Cleanuping done (dev %p)", dev);
2814         complete_all(&dev->cleanup_cmpl);
2815         res = 0;
2816
2817 out:
2818         TRACE_EXIT_RES(res);
2819         return res;
2820 }
2821
2822 static inline int test_cleanup_list(void)
2823 {
2824         int res = !list_empty(&cleanup_list) ||
2825                   unlikely(kthread_should_stop());
2826         return res;
2827 }
2828
2829 static int dev_user_cleanup_thread(void *arg)
2830 {
2831         TRACE_ENTRY();
2832
2833         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2834
2835         current->flags |= PF_NOFREEZE;
2836
2837         spin_lock(&cleanup_lock);
2838         while (!kthread_should_stop()) {
2839                 wait_queue_t wait;
2840                 init_waitqueue_entry(&wait, current);
2841
2842                 if (!test_cleanup_list()) {
2843                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2844                         for (;;) {
2845                                 set_current_state(TASK_INTERRUPTIBLE);
2846                                 if (test_cleanup_list())
2847                                         break;
2848                                 spin_unlock(&cleanup_lock);
2849                                 schedule();
2850                                 spin_lock(&cleanup_lock);
2851                         }
2852                         set_current_state(TASK_RUNNING);
2853                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2854                 }
2855
2856                 /*
2857                  * We have to poll devices, because commands can go from SCST
2858                  * core on cmd_list_waitQ and we have no practical way to
2859                  * detect them.
2860                  */
2861
2862                 while (1) {
2863                         struct scst_user_dev *dev;
2864                         LIST_HEAD(cl_devs);
2865
2866                         while (!list_empty(&cleanup_list)) {
2867                                 int rc;
2868
2869                                 dev = list_entry(cleanup_list.next,
2870                                         typeof(*dev), cleanup_list_entry);
2871                                 list_del(&dev->cleanup_list_entry);
2872
2873                                 spin_unlock(&cleanup_lock);
2874                                 rc = dev_user_process_cleanup(dev);
2875                                 spin_lock(&cleanup_lock);
2876
2877                                 if (rc != 0)
2878                                         list_add_tail(&dev->cleanup_list_entry,
2879                                                 &cl_devs);
2880                         }
2881
2882                         if (list_empty(&cl_devs))
2883                                 break;
2884
2885                         spin_unlock(&cleanup_lock);
2886                         msleep(100);
2887                         spin_lock(&cleanup_lock);
2888
2889                         while (!list_empty(&cl_devs)) {
2890                                 dev = list_entry(cl_devs.next, typeof(*dev),
2891                                         cleanup_list_entry);
2892                                 list_move_tail(&dev->cleanup_list_entry,
2893                                         &cleanup_list);
2894                         }
2895                 }
2896         }
2897         spin_unlock(&cleanup_lock);
2898
2899         /*
2900          * If kthread_should_stop() is true, we are guaranteed to be
2901          * on the module unload, so cleanup_list must be empty.
2902          */
2903         sBUG_ON(!list_empty(&cleanup_list));
2904
2905         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2906
2907         TRACE_EXIT();
2908         return 0;
2909 }
2910
2911 static int __init init_scst_user(void)
2912 {
2913         int res = 0;
2914         struct max_get_reply {
2915                 union {
2916                         struct scst_user_get_cmd g;
2917                         struct scst_user_reply_cmd r;
2918                 };
2919         };
2920 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
2921         struct class_device *class_member;
2922 #else
2923         struct device *dev;
2924 #endif
2925
2926         TRACE_ENTRY();
2927
2928 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2929         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2930                 "Consider change VMSPLIT option or use 64-bit "
2931                 "configuration instead. See README file for details.");
2932         res = -EINVAL;
2933         goto out;
2934 #endif
2935
2936         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2937         if (user_cmd_cachep == NULL) {
2938                 res = -ENOMEM;
2939                 goto out;
2940         }
2941
2942         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
2943         if (user_get_cmd_cachep == NULL) {
2944                 res = -ENOMEM;
2945                 goto out_cache;
2946         }
2947
2948         dev_user_devtype.module = THIS_MODULE;
2949
2950         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2951         if (res < 0)
2952                 goto out_cache1;
2953
2954         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2955         if (res != 0)
2956                 goto out_unreg;
2957
2958         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2959         if (IS_ERR(dev_user_sysfs_class)) {
2960                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2961                         "space handler");
2962                 res = PTR_ERR(dev_user_sysfs_class);
2963                 goto out_proc;
2964         }
2965
2966         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2967         if (res) {
2968                 PRINT_ERROR("Unable to get major %d for SCSI tapes", DEV_USER_MAJOR);
2969                 goto out_class;
2970         }
2971
2972 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
2973         class_member = class_device_create(dev_user_sysfs_class, NULL,
2974                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
2975         if (IS_ERR(class_member)) {
2976                 res = PTR_ERR(class_member);
2977                 goto out_chrdev;
2978         }
2979 #else
2980         dev = device_create(dev_user_sysfs_class, NULL, MKDEV(DEV_USER_MAJOR, 0),
2981 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
2982                                 NULL,
2983 #endif
2984                                 DEV_USER_NAME);
2985         if (IS_ERR(dev)) {
2986                 res = PTR_ERR(dev);
2987                 goto out_chrdev;
2988         }
2989 #endif
2990
2991         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
2992                 "scst_usr_cleanupd");
2993         if (IS_ERR(cleanup_thread)) {
2994                 res = PTR_ERR(cleanup_thread);
2995                 PRINT_ERROR("kthread_create() failed: %d", res);
2996                 goto out_dev;
2997         }
2998
2999 out:
3000         TRACE_EXIT_RES(res);
3001         return res;
3002
3003 out_dev:
3004 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3005         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3006 #else
3007         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3008 #endif
3009
3010 out_chrdev:
3011         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3012
3013 out_class:
3014         class_destroy(dev_user_sysfs_class);
3015
3016 out_proc:
3017         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3018
3019 out_unreg:
3020         scst_unregister_dev_driver(&dev_user_devtype);
3021
3022 out_cache1:
3023         kmem_cache_destroy(user_get_cmd_cachep);
3024
3025 out_cache:
3026         kmem_cache_destroy(user_cmd_cachep);
3027         goto out;
3028 }
3029
3030 static void __exit exit_scst_user(void)
3031 {
3032         int rc;
3033
3034         TRACE_ENTRY();
3035
3036         rc = kthread_stop(cleanup_thread);
3037         if (rc < 0)
3038                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3039
3040         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3041 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3042         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3043 #else
3044         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3045 #endif
3046         class_destroy(dev_user_sysfs_class);
3047
3048         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3049         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3050
3051         kmem_cache_destroy(user_get_cmd_cachep);
3052         kmem_cache_destroy(user_cmd_cachep);
3053
3054         TRACE_EXIT();
3055         return;
3056 }
3057
3058 module_init(init_scst_user);
3059 module_exit(exit_scst_user);
3060
3061 MODULE_AUTHOR("Vladislav Bolkhovitin");
3062 MODULE_LICENSE("GPL");
3063 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3064 MODULE_VERSION(SCST_VERSION_STRING);
3065 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);