- Fixes incorrect allocation length 0 by scst_user module
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23
24 #define LOG_PREFIX              DEV_USER_NAME
25
26 #include "scst.h"
27 #include "scst_user.h"
28 #include "scst_dev_handler.h"
29
30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
31 #warning "HIGHMEM kernel configurations are not supported by this module, \
32         because nowadays it doesn't worth the effort. Consider change \
33         VMSPLIT option or use 64-bit configuration instead. See README file \
34         for details."
35 #endif
36
37 #define DEV_USER_MAJOR                  237
38
39 #define DEV_USER_CMD_HASH_ORDER         6
40
41 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
42
43 struct scst_user_dev {
44         struct rw_semaphore dev_rwsem;
45
46         struct scst_cmd_lists cmd_lists;
47
48         /* Protected by cmd_lists.cmd_list_lock */
49         struct list_head ready_cmd_list;
50
51         /* Protected by dev_rwsem or don't need any protection */
52         unsigned int blocking:1;
53         unsigned int cleanup_done:1;
54         unsigned int tst:3;
55         unsigned int queue_alg:4;
56         unsigned int tas:1;
57         unsigned int swp:1;
58         unsigned int has_own_order_mgmt:1;
59
60         int (*generic_parse)(struct scst_cmd *cmd,
61                 int (*get_block)(struct scst_cmd *cmd));
62
63         int block;
64         int def_block;
65
66         struct scst_mem_lim udev_mem_lim;
67         struct sgv_pool *pool;
68
69         uint8_t parse_type;
70         uint8_t on_free_cmd_type;
71         uint8_t memory_reuse_type;
72         uint8_t partial_transfers_type;
73         uint32_t partial_len;
74
75         struct scst_dev_type devtype;
76
77         /* Both protected by cmd_lists.cmd_list_lock */
78         unsigned int handle_counter;
79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
80
81         struct scst_device *sdev;
82
83         int virt_id;
84         struct list_head dev_list_entry;
85         char name[SCST_MAX_NAME];
86
87         struct list_head cleanup_list_entry;
88         /* ToDo: make it on-stack */
89         struct completion cleanup_cmpl;
90 };
91
92 /* Most fields are unprotected, since only one thread at time can access them */
93 struct scst_user_cmd {
94         struct scst_cmd *cmd;
95         struct scst_user_dev *dev;
96
97         atomic_t ucmd_ref;
98
99         unsigned int buff_cached:1;
100         unsigned int buf_dirty:1;
101         unsigned int background_exec:1;
102         unsigned int aborted:1;
103
104         struct scst_user_cmd *buf_ucmd;
105
106         int cur_data_page;
107         int num_data_pages;
108         int first_page_offset;
109         unsigned long ubuff;
110         struct page **data_pages;
111         struct sgv_pool_obj *sgv;
112
113         /*
114          * Special flags, which can be accessed asynchronously (hence "long").
115          * Protected by cmd_lists.cmd_list_lock.
116          */
117         unsigned long sent_to_user:1;
118         unsigned long jammed:1;
119         unsigned long this_state_unjammed:1;
120         unsigned long seen_by_user:1; /* here only as a small optimization */
121
122         unsigned int state;
123
124         struct list_head ready_cmd_list_entry;
125
126         unsigned int h;
127         struct list_head hash_list_entry;
128
129         struct scst_user_get_cmd user_cmd;
130
131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
132         union {
133                 struct completion *cmpl;
134                 struct scst_mgmt_cmd *mcmd;
135         };
136         int result;
137 };
138
139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
140         int gfp_mask);
141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
142
143 static int dev_user_parse(struct scst_cmd *cmd);
144 static int dev_user_exec(struct scst_cmd *cmd);
145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
147         struct scst_tgt_dev *tgt_dev);
148
149 static int dev_user_disk_done(struct scst_cmd *cmd);
150 static int dev_user_tape_done(struct scst_cmd *cmd);
151
152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
153         gfp_t gfp_mask, void *priv);
154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
155                                      void *priv);
156
157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
158
159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
160         unsigned long *flags);
161
162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
164         int status);
165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
166 static int dev_user_register_dev(struct file *file,
167         const struct scst_user_dev_desc *dev_desc);
168 static int __dev_user_set_opt(struct scst_user_dev *dev,
169         const struct scst_user_opt *opt);
170 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
171 static int dev_user_get_opt(struct file *file, void *arg);
172
173 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
174 static long dev_user_ioctl(struct file *file, unsigned int cmd,
175         unsigned long arg);
176 static int dev_user_release(struct inode *inode, struct file *file);
177
178 /** Data **/
179
180 static struct kmem_cache *user_cmd_cachep;
181
182 static DEFINE_MUTEX(dev_priv_mutex);
183
184 static struct file_operations dev_user_fops = {
185         .poll           = dev_user_poll,
186         .unlocked_ioctl = dev_user_ioctl,
187 #ifdef CONFIG_COMPAT
188         .compat_ioctl   = dev_user_ioctl,
189 #endif
190         .release        = dev_user_release,
191 };
192
193 static struct class *dev_user_sysfs_class;
194
195 static DEFINE_SPINLOCK(dev_list_lock);
196 static LIST_HEAD(dev_list);
197
198 static DEFINE_SPINLOCK(cleanup_lock);
199 static LIST_HEAD(cleanup_list);
200 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
201 static struct task_struct *cleanup_thread;
202
203 /*
204  * Skip this command if result is not 0. Must be called under
205  * cmd_lists.cmd_list_lock and IRQ off.
206  */
207 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
208 {
209         int r = atomic_inc_return(&ucmd->ucmd_ref);
210         int res;
211         if (unlikely(r == 1)) {
212                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
213                 atomic_dec(&ucmd->ucmd_ref);
214                 res = true;
215                 /*
216                  * Necessary code is serialized by cmd_list_lock in
217                  * cmd_remove_hash()
218                  */
219         } else {
220                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
221                         atomic_read(&ucmd->ucmd_ref));
222                 res = false;
223         }
224         return res;
225 }
226
227 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
228 {
229         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
230         atomic_inc(&ucmd->ucmd_ref);
231         if (barrier)
232                 smp_mb__after_atomic_inc();
233 }
234
235 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
236 {
237         __ucmd_get(ucmd, true);
238 }
239
240 static inline void ucmd_get(struct scst_user_cmd *ucmd)
241 {
242         __ucmd_get(ucmd, false);
243 }
244
245 /* Must not be called under cmd_list_lock!! */
246 static inline void ucmd_put(struct scst_user_cmd *ucmd)
247 {
248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
249
250         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
251
252         if (atomic_dec_and_test(&ucmd->ucmd_ref))
253                 dev_user_free_ucmd(ucmd);
254 }
255
256 static inline int calc_num_pg(unsigned long buf, int len)
257 {
258         len += buf & ~PAGE_MASK;
259         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
260 }
261
262 static inline int is_need_offs_page(unsigned long buf, int len)
263 {
264         return ((buf & ~PAGE_MASK) != 0) &&
265                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
266 }
267
268 static void __dev_user_not_reg(void)
269 {
270         PRINT_ERROR("%s", "Device not registered");
271         return;
272 }
273
274 static inline int dev_user_check_reg(struct scst_user_dev *dev)
275 {
276         if (dev == NULL) {
277                 __dev_user_not_reg();
278                 return -EINVAL;
279         }
280         return 0;
281 }
282
283 static inline int scst_user_cmd_hashfn(int h)
284 {
285         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
286 }
287
288 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
289         unsigned int h)
290 {
291         struct list_head *head;
292         struct scst_user_cmd *ucmd;
293
294         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
295         list_for_each_entry(ucmd, head, hash_list_entry) {
296                 if (ucmd->h == h) {
297                         TRACE_DBG("Found ucmd %p", ucmd);
298                         return ucmd;
299                 }
300         }
301         return NULL;
302 }
303
304 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
305 {
306         struct list_head *head;
307         struct scst_user_dev *dev = ucmd->dev;
308         struct scst_user_cmd *u;
309         unsigned long flags;
310
311         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
312         do {
313                 ucmd->h = dev->handle_counter++;
314                 u = __ucmd_find_hash(dev, ucmd->h);
315         } while (u != NULL);
316         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
317         list_add_tail(&ucmd->hash_list_entry, head);
318         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
319
320         TRACE_DBG("Inserted ucmd %p, h=%d", ucmd, ucmd->h);
321         return;
322 }
323
324 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
325 {
326         unsigned long flags;
327
328         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
329         list_del(&ucmd->hash_list_entry);
330         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
331
332         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
333         return;
334 }
335
336 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
337 {
338         TRACE_ENTRY();
339
340         TRACE_MEM("Freeing ucmd %p", ucmd);
341
342         cmd_remove_hash(ucmd);
343         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
344
345         kmem_cache_free(user_cmd_cachep, ucmd);
346
347         TRACE_EXIT();
348         return;
349 }
350
351 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
352         gfp_t gfp_mask, void *priv)
353 {
354         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
355         int offset = 0;
356
357         TRACE_ENTRY();
358
359         /* *sg supposed to be zeroed */
360
361         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
362                 ucmd->ubuff, ucmd->cur_data_page);
363
364         if (ucmd->cur_data_page == 0) {
365                 TRACE_MEM("ucmd->first_page_offset %d",
366                         ucmd->first_page_offset);
367                 offset = ucmd->first_page_offset;
368                 ucmd_get(ucmd);
369         }
370
371         if (ucmd->cur_data_page >= ucmd->num_data_pages)
372                 goto out;
373
374         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
375                 PAGE_SIZE - offset, offset);
376         ucmd->cur_data_page++;
377
378         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
379                 sg->offset);
380         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
381
382 out:
383         TRACE_EXIT();
384         return sg_page(sg);
385 }
386
387 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
388 {
389         TRACE_ENTRY();
390
391         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
392                 ucmd, ucmd->h, ucmd->ubuff);
393
394         ucmd->user_cmd.cmd_h = ucmd->h;
395         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
396         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
397
398         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
399
400         dev_user_add_to_ready(ucmd);
401
402         TRACE_EXIT();
403         return;
404 }
405
406 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
407 {
408         int i;
409
410         TRACE_ENTRY();
411
412         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
413                 ucmd->ubuff, ucmd->num_data_pages);
414
415         for (i = 0; i < ucmd->num_data_pages; i++) {
416                 struct page *page = ucmd->data_pages[i];
417
418                 if (ucmd->buf_dirty)
419                         SetPageDirty(page);
420
421                 page_cache_release(page);
422         }
423
424         kfree(ucmd->data_pages);
425         ucmd->data_pages = NULL;
426
427         TRACE_EXIT();
428         return;
429 }
430
431 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
432 {
433         TRACE_ENTRY();
434
435         sBUG_ON(ucmd->data_pages == NULL);
436
437         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
438                 ucmd, ucmd->ubuff, ucmd->buff_cached);
439
440         dev_user_unmap_buf(ucmd);
441
442         if (ucmd->buff_cached)
443                 dev_user_on_cached_mem_free(ucmd);
444         else
445                 ucmd_put(ucmd);
446
447         TRACE_EXIT();
448         return;
449 }
450
451 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
452         void *priv)
453 {
454         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
455
456         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
457                 sg_count, ucmd);
458
459         __dev_user_free_sg_entries(ucmd);
460
461         return;
462 }
463
464 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
465 {
466         int mem_reuse_type = ucmd->dev->memory_reuse_type;
467
468         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
469             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
470              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
471             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
472              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
473                 return 1;
474         else
475                 return 0;
476 }
477
478 /*
479  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
480  * Unmaps the buffer, if needed in case of error
481  */
482 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
483 {
484         int res = 0;
485         struct scst_cmd *cmd = ucmd->cmd;
486         struct scst_user_dev *dev = ucmd->dev;
487         int gfp_mask, flags = 0;
488         int bufflen = cmd->bufflen;
489         int last_len = 0;
490
491         TRACE_ENTRY();
492
493         /* User space can return from PARSE bufflen 0 and direction non-NONE */
494         if (unlikely(bufflen == 0)) {
495                 cmd->data_direction = SCST_DATA_NONE;
496                 goto out;
497         }
498
499         gfp_mask = __GFP_NOWARN;
500         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
501
502         if (cached_buff) {
503                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
504                 if (ucmd->ubuff == 0)
505                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
506         } else {
507                 TRACE_MEM("%s", "Not cached buff");
508                 flags |= SCST_POOL_ALLOC_NO_CACHED;
509                 if (ucmd->ubuff == 0) {
510                         res = 1;
511                         goto out;
512                 }
513                 bufflen += ucmd->first_page_offset;
514                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
515                         last_len = bufflen & ~PAGE_MASK;
516                 else
517                         last_len = cmd->bufflen & ~PAGE_MASK;
518                 if (last_len == 0)
519                         last_len = PAGE_SIZE;
520         }
521         ucmd->buff_cached = cached_buff;
522
523         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
524                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
525         if (cmd->sg != NULL) {
526                 struct scst_user_cmd *buf_ucmd =
527                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
528
529                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
530
531                 ucmd->ubuff = buf_ucmd->ubuff;
532                 ucmd->buf_ucmd = buf_ucmd;
533
534                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
535                                    (ucmd != buf_ucmd));
536
537                 if (last_len != 0) {
538                         /* We don't use clustering, so the assignment is safe */
539                         cmd->sg[cmd->sg_cnt-1].length = last_len;
540                 }
541
542                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
543                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
544                         last_len, cmd->sg[cmd->sg_cnt-1].length);
545
546                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
547                         static int ll;
548                         if (ll < 10) {
549                                 PRINT_INFO("Unable to complete command due to "
550                                         "SG IO count limitation (requested %d, "
551                                         "available %d, tgt lim %d)", cmd->sg_cnt,
552                                         cmd->tgt_dev->max_sg_cnt,
553                                         cmd->tgt->sg_tablesize);
554                                 ll++;
555                         }
556                         cmd->sg = NULL;
557                         /* sgv will be freed in dev_user_free_sgv() */
558                         res = -1;
559                 }
560         } else {
561                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
562                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
563                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
564                 if (unlikely(cmd->sg_cnt == 0)) {
565                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
566                         sBUG_ON(ucmd->sgv != NULL);
567                         res = -1;
568                 } else {
569                         switch (ucmd->state) {
570                         case UCMD_STATE_BUF_ALLOCING:
571                                 res = 1;
572                                 break;
573                         case UCMD_STATE_EXECING:
574                                 res = -1;
575                                 break;
576                         default:
577                                 sBUG();
578                                 break;
579                         }
580                 }
581         }
582
583 out:
584         TRACE_EXIT_RES(res);
585         return res;
586 }
587
588 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
589 {
590         int rc, res = SCST_CMD_STATE_DEFAULT;
591         struct scst_cmd *cmd = ucmd->cmd;
592
593         TRACE_ENTRY();
594
595         if (unlikely(ucmd->cmd->data_buf_tgt_alloc)) {
596                 PRINT_ERROR("Target driver %s requested own memory "
597                         "allocation", ucmd->cmd->tgtt->name);
598                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
599                 res = scst_get_cmd_abnormal_done_state(cmd);
600                 goto out;
601         }
602
603         ucmd->state = UCMD_STATE_BUF_ALLOCING;
604         cmd->data_buf_alloced = 1;
605
606         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
607         if (rc == 0)
608                 goto out;
609         else if (rc < 0) {
610                 scst_set_busy(cmd);
611                 res = scst_get_cmd_abnormal_done_state(cmd);
612                 goto out;
613         }
614
615         if ((cmd->data_direction != SCST_DATA_WRITE) &&
616             !scst_is_cmd_local(cmd)) {
617                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
618                 goto out;
619         }
620
621         ucmd->user_cmd.cmd_h = ucmd->h;
622         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
623         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
624         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
625                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
626         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
627         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
628                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
629         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
630         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
631         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
632
633         dev_user_add_to_ready(ucmd);
634
635         res = SCST_CMD_STATE_STOP;
636
637 out:
638         TRACE_EXIT_RES(res);
639         return res;
640 }
641
642 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
643         int gfp_mask)
644 {
645         struct scst_user_cmd *ucmd = NULL;
646
647         TRACE_ENTRY();
648
649 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
650         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
651         if (ucmd != NULL)
652                 memset(ucmd, 0, sizeof(*ucmd));
653 #else
654         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
655 #endif
656         if (unlikely(ucmd == NULL)) {
657                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
658                         "user cmd (gfp_mask %x)", gfp_mask);
659                 goto out;
660         }
661         ucmd->dev = dev;
662         atomic_set(&ucmd->ucmd_ref, 1);
663
664         cmd_insert_hash(ucmd);
665
666         TRACE_MEM("ucmd %p allocated", ucmd);
667
668 out:
669         TRACE_EXIT_HRES((unsigned long)ucmd);
670         return ucmd;
671 }
672
673 static int dev_user_get_block(struct scst_cmd *cmd)
674 {
675         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
676         /*
677          * No need for locks here, since *_detach() can not be
678          * called, when there are existing commands.
679          */
680         TRACE_EXIT_RES(dev->block);
681         return dev->block;
682 }
683
684 static int dev_user_parse(struct scst_cmd *cmd)
685 {
686         int rc, res = SCST_CMD_STATE_DEFAULT;
687         struct scst_user_cmd *ucmd;
688         int atomic = scst_cmd_atomic(cmd);
689         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
690         int gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
691
692         TRACE_ENTRY();
693
694         if (cmd->dh_priv == NULL) {
695                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
696                 if (unlikely(ucmd == NULL)) {
697                         if (atomic) {
698                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
699                                 goto out;
700                         } else {
701                                 scst_set_busy(cmd);
702                                 goto out_error;
703                         }
704                 }
705                 ucmd->cmd = cmd;
706                 cmd->dh_priv = ucmd;
707         } else {
708                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
709                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
710         }
711
712         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
713
714         if (ucmd->state != UCMD_STATE_NEW)
715                 goto alloc;
716
717         switch (dev->parse_type) {
718         case SCST_USER_PARSE_STANDARD:
719                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
720                 rc = dev->generic_parse(cmd, dev_user_get_block);
721                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
722                         goto out_invalid;
723                 break;
724
725         case SCST_USER_PARSE_EXCEPTION:
726                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
727                 rc = dev->generic_parse(cmd, dev_user_get_block);
728                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
729                         break;
730                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
731                         TRACE_MEM("Restarting PARSE to thread context "
732                                 "(ucmd %p)", ucmd);
733                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
734                         goto out;
735                 }
736                 /* else go through */
737
738         case SCST_USER_PARSE_CALL:
739                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
740                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
741                 ucmd->user_cmd.cmd_h = ucmd->h;
742                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
743                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
744                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
745                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
746                             sizeof(cmd->cdb)));
747                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
748                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
749                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
750                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
751                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
752                 ucmd->user_cmd.parse_cmd.expected_values_set =
753                                         cmd->expected_values_set;
754                 ucmd->user_cmd.parse_cmd.expected_data_direction =
755                                         cmd->expected_data_direction;
756                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
757                                         cmd->expected_transfer_len;
758                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
759                 ucmd->state = UCMD_STATE_PARSING;
760                 dev_user_add_to_ready(ucmd);
761                 res = SCST_CMD_STATE_STOP;
762                 goto out;
763
764         default:
765                 sBUG();
766                 goto out;
767         }
768
769 alloc:
770         if (cmd->data_direction != SCST_DATA_NONE)
771                 res = dev_user_alloc_space(ucmd);
772
773 out:
774         TRACE_EXIT_RES(res);
775         return res;
776
777 out_invalid:
778         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
779                 cmd->op_flags & SCST_INFO_INVALID);
780         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
781
782 out_error:
783         res = scst_get_cmd_abnormal_done_state(cmd);
784         goto out;
785 }
786
787 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
788 {
789         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
790         unsigned long start = buf_ucmd->ubuff;
791         int i;
792
793         TRACE_ENTRY();
794
795         if (start == 0)
796                 goto out;
797
798         for (i = 0; i < buf_ucmd->num_data_pages; i++) {
799                 struct page *page;
800                 page = buf_ucmd->data_pages[i];
801 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
802                 struct vm_area_struct *vma = find_vma(current->mm, start);
803                 if (vma != NULL)
804                         flush_anon_page(vma, page, start);
805 #endif
806                 flush_dcache_page(page);
807                 start += PAGE_SIZE;
808         }
809
810 out:
811         TRACE_EXIT();
812         return;
813 }
814
815 static int dev_user_exec(struct scst_cmd *cmd)
816 {
817         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
818         int res = SCST_EXEC_COMPLETED;
819
820         TRACE_ENTRY();
821
822 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
823        * context to complete the necessary actions, but all we are going to
824        * do in this function is, in fact, atomic, so let's skip this check.
825        */
826         if (scst_cmd_atomic(cmd)) {
827                 TRACE_DBG("%s", "User exec() can not be called in atomic "
828                         "context, rescheduling to the thread");
829                 res = SCST_EXEC_NEED_THREAD;
830                 goto out;
831         }
832 #endif
833
834         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
835                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
836                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
837
838         if (cmd->data_direction == SCST_DATA_WRITE)
839                 dev_user_flush_dcache(ucmd);
840
841         ucmd->user_cmd.cmd_h = ucmd->h;
842         ucmd->user_cmd.subcode = SCST_USER_EXEC;
843         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
844         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
845                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
846                     sizeof(cmd->cdb)));
847         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
848         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
849         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
850         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
851         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
852                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
853                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
854         }
855         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
856         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
857         ucmd->user_cmd.exec_cmd.partial = 0;
858         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
859         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
860
861         ucmd->state = UCMD_STATE_EXECING;
862
863         dev_user_add_to_ready(ucmd);
864
865         TRACE_EXIT_RES(res);
866         return res;
867 }
868
869 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
870 {
871         if (ucmd->sgv != NULL) {
872                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
873                 ucmd->sgv = NULL;
874         } else if (ucmd->data_pages != NULL) {
875                 /* We mapped pages, but for some reason didn't allocate them */
876                 ucmd_get(ucmd);
877                 __dev_user_free_sg_entries(ucmd);
878         }
879         return;
880 }
881
882 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
883 {
884         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
885
886         TRACE_ENTRY();
887
888         if (unlikely(ucmd == NULL))
889                 goto out;
890
891         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
892                 ucmd->buff_cached, ucmd->ubuff);
893
894         ucmd->cmd = NULL;
895         if ((cmd->data_direction == SCST_DATA_WRITE) && (ucmd->buf_ucmd != NULL))
896                 ucmd->buf_ucmd->buf_dirty = 1;
897
898         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
899                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
900                 /* The state assignment must be before freeing sgv! */
901                 goto out_reply;
902         }
903
904         if (unlikely(!ucmd->seen_by_user)) {
905                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
906                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
907                 goto out_reply;
908         }
909
910         ucmd->user_cmd.cmd_h = ucmd->h;
911         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
912
913         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
914         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
915         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
916         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
917         ucmd->user_cmd.on_free_cmd.status = cmd->status;
918         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
919
920         ucmd->state = UCMD_STATE_ON_FREEING;
921
922         dev_user_add_to_ready(ucmd);
923
924 out:
925         TRACE_EXIT();
926         return;
927
928 out_reply:
929         dev_user_process_reply_on_free(ucmd);
930         goto out;
931 }
932
933 static void dev_user_set_block(struct scst_cmd *cmd, int block)
934 {
935         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
936         /*
937          * No need for locks here, since *_detach() can not be
938          * called, when there are existing commands.
939          */
940         TRACE_DBG("dev %p, new block %d", dev, block);
941         if (block != 0)
942                 dev->block = block;
943         else
944                 dev->block = dev->def_block;
945         return;
946 }
947
948 static int dev_user_disk_done(struct scst_cmd *cmd)
949 {
950         int res = SCST_CMD_STATE_DEFAULT;
951
952         TRACE_ENTRY();
953
954         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
955
956         TRACE_EXIT_RES(res);
957         return res;
958 }
959
960 static int dev_user_tape_done(struct scst_cmd *cmd)
961 {
962         int res = SCST_CMD_STATE_DEFAULT;
963
964         TRACE_ENTRY();
965
966         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
967
968         TRACE_EXIT_RES(res);
969         return res;
970 }
971
972 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
973 {
974         struct scst_user_dev *dev = ucmd->dev;
975         unsigned long flags;
976         int do_wake;
977
978         TRACE_ENTRY();
979
980         do_wake = (in_interrupt() ||
981                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
982         if (ucmd->cmd)
983                 do_wake |= ucmd->cmd->preprocessing_only;
984
985         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
986
987         ucmd->this_state_unjammed = 0;
988
989         if ((ucmd->state == UCMD_STATE_PARSING) ||
990             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
991                 /*
992                  * If we don't put such commands in the queue head, then under
993                  * high load we might delay threads, waiting for memory
994                  * allocations, for too long and start loosing NOPs, which
995                  * would lead to consider us by remote initiators as
996                  * unresponsive and stuck => broken connections, etc. If none
997                  * of our commands completed in NOP timeout to allow the head
998                  * commands to go, then we are really overloaded and/or stuck.
999                  */
1000                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1001                         "cmd list", ucmd, ucmd->state);
1002                 list_add(&ucmd->ready_cmd_list_entry,
1003                         &dev->ready_cmd_list);
1004                 do_wake = 1;
1005         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1006                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1007                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1008                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1009                         "ready cmd list", ucmd, ucmd->state);
1010                 list_add(&ucmd->ready_cmd_list_entry,
1011                         &dev->ready_cmd_list);
1012                 do_wake = 1;
1013         } else if ((ucmd->cmd != NULL) &&
1014                    unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1015                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1016                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1017         } else {
1018                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1019                 list_add_tail(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1020         }
1021
1022         if (do_wake) {
1023                 TRACE_DBG("Waking up dev %p", dev);
1024                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1025         }
1026
1027         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1028
1029         TRACE_EXIT();
1030         return;
1031 }
1032
1033 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1034         int num_pg)
1035 {
1036         int res = 0, rc;
1037         int i;
1038
1039         TRACE_ENTRY();
1040
1041         if (unlikely(ubuff == 0))
1042                 goto out_nomem;
1043
1044         sBUG_ON(ucmd->data_pages != NULL);
1045
1046         ucmd->num_data_pages = num_pg;
1047
1048         ucmd->data_pages = kzalloc(sizeof(*ucmd->data_pages)*ucmd->num_data_pages,
1049                 GFP_KERNEL);
1050         if (ucmd->data_pages == NULL) {
1051                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1052                         "(num_data_pages=%d)", ucmd->num_data_pages);
1053                 res = -ENOMEM;
1054                 goto out_nomem;
1055         }
1056
1057         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d, "
1058                 "first_page_offset %d, len %d)", ucmd, ubuff,
1059                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1060                 ucmd->cmd->bufflen);
1061
1062         down_read(&current->mm->mmap_sem);
1063         rc = get_user_pages(current, current->mm, ubuff, ucmd->num_data_pages,
1064                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1065         up_read(&current->mm->mmap_sem);
1066
1067         /* get_user_pages() flushes dcache */
1068
1069         if (rc < ucmd->num_data_pages)
1070                 goto out_unmap;
1071
1072         ucmd->ubuff = ubuff;
1073         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1074
1075 out:
1076         TRACE_EXIT_RES(res);
1077         return res;
1078
1079 out_nomem:
1080         scst_set_busy(ucmd->cmd);
1081         /* go through */
1082
1083 out_err:
1084         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1085         goto out;
1086
1087 out_unmap:
1088         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1089                 ucmd->num_data_pages, rc);
1090         if (rc > 0) {
1091                 for (i = 0; i < rc; i++)
1092                         page_cache_release(ucmd->data_pages[i]);
1093         }
1094         kfree(ucmd->data_pages);
1095         ucmd->data_pages = NULL;
1096         res = -EFAULT;
1097         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1098         goto out_err;
1099 }
1100
1101 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1102         struct scst_user_reply_cmd *reply)
1103 {
1104         int res = 0;
1105         struct scst_cmd *cmd = ucmd->cmd;
1106
1107         TRACE_ENTRY();
1108
1109         TRACE_DBG("ucmd %p, pbuf %Lx", ucmd, reply->alloc_reply.pbuf);
1110
1111         if (likely(reply->alloc_reply.pbuf != 0)) {
1112                 int pages;
1113                 if (ucmd->buff_cached) {
1114                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1115                                 PRINT_ERROR("Supplied pbuf %Lx isn't "
1116                                         "page aligned", reply->alloc_reply.pbuf);
1117                                 goto out_hwerr;
1118                         }
1119                         pages = cmd->sg_cnt;
1120                 } else
1121                         pages = calc_num_pg(reply->alloc_reply.pbuf, cmd->bufflen);
1122                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1123         } else {
1124                 scst_set_busy(ucmd->cmd);
1125                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1126         }
1127
1128 out_process:
1129         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1130
1131         TRACE_EXIT_RES(res);
1132         return res;
1133
1134 out_hwerr:
1135         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1136         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1137         res = -EINVAL;
1138         goto out_process;
1139 }
1140
1141 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1142         struct scst_user_reply_cmd *reply)
1143 {
1144         int res = 0;
1145         struct scst_user_scsi_cmd_reply_parse *preply =
1146                 &reply->parse_reply;
1147         struct scst_cmd *cmd = ucmd->cmd;
1148
1149         TRACE_ENTRY();
1150
1151         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1152                 goto out_inval;
1153
1154         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1155                      (preply->data_direction != SCST_DATA_READ) &&
1156                      (preply->data_direction != SCST_DATA_NONE)))
1157                 goto out_inval;
1158
1159         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1160                      (preply->bufflen == 0)))
1161                 goto out_inval;
1162
1163         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1164                 goto out_inval;
1165
1166         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1167                 "data_len %d, pbuf %Lx", ucmd, preply->queue_type,
1168                 preply->data_direction, preply->bufflen, preply->data_len,
1169                 reply->alloc_reply.pbuf);
1170
1171         cmd->queue_type = preply->queue_type;
1172         cmd->data_direction = preply->data_direction;
1173         cmd->bufflen = preply->bufflen;
1174         cmd->data_len = preply->data_len;
1175
1176 out_process:
1177         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1178
1179         TRACE_EXIT_RES(res);
1180         return res;
1181
1182 out_inval:
1183         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1184                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1185         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1186         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1187         scst_set_cmd_abnormal_done_state(cmd);
1188         res = -EINVAL;
1189         goto out_process;
1190 }
1191
1192 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1193 {
1194         int res = 0;
1195
1196         TRACE_ENTRY();
1197
1198         TRACE_DBG("ON FREE ucmd %p", ucmd);
1199
1200         dev_user_free_sgv(ucmd);
1201         ucmd_put(ucmd);
1202
1203         TRACE_EXIT_RES(res);
1204         return res;
1205 }
1206
1207 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1208 {
1209         int res = 0;
1210
1211         TRACE_ENTRY();
1212
1213         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1214
1215         ucmd_put(ucmd);
1216
1217         TRACE_EXIT_RES(res);
1218         return res;
1219 }
1220
1221 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1222         struct scst_user_reply_cmd *reply)
1223 {
1224         int res = 0;
1225         struct scst_user_scsi_cmd_reply_exec *ereply =
1226                 &reply->exec_reply;
1227         struct scst_cmd *cmd = ucmd->cmd;
1228
1229         TRACE_ENTRY();
1230
1231         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1232                 if (ucmd->background_exec) {
1233                         TRACE_DBG("Background ucmd %p finished", ucmd);
1234                         ucmd_put(ucmd);
1235                         goto out;
1236                 }
1237                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1238                         goto out_inval;
1239                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1240                              (ereply->resp_data_len != 0)))
1241                         goto out_inval;
1242         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1243                 if (unlikely(ucmd->background_exec))
1244                         goto out_inval;
1245                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1246                              (cmd->resp_data_len != 0)))
1247                         goto out_inval;
1248                 ucmd_get_ordered(ucmd);
1249                 ucmd->background_exec = 1;
1250                 TRACE_DBG("Background ucmd %p", ucmd);
1251                 goto out_compl;
1252         } else
1253                 goto out_inval;
1254
1255         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1256                 ereply->status, ereply->resp_data_len);
1257
1258          if (ereply->resp_data_len != 0) {
1259                 if (ucmd->ubuff == 0) {
1260                         int pages, rc;
1261                         if (unlikely(ereply->pbuf == 0))
1262                                 goto out_busy;
1263                         if (ucmd->buff_cached) {
1264                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1265                                         PRINT_ERROR("Supplied pbuf %Lx isn't "
1266                                                 "page aligned", ereply->pbuf);
1267                                         goto out_hwerr;
1268                                 }
1269                                 pages = cmd->sg_cnt;
1270                         } else
1271                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1272                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1273                         if ((rc != 0) || (ucmd->ubuff == 0))
1274                                 goto out_compl;
1275
1276                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1277                         if (unlikely(rc != 0))
1278                                 goto out_busy;
1279                 } else
1280                         dev_user_flush_dcache(ucmd);
1281                 cmd->may_need_dma_sync = 1;
1282                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1283         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1284                 if (ucmd->ubuff == 0)
1285                         cmd->resp_data_len = ereply->resp_data_len;
1286                 else
1287                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1288         }
1289
1290         cmd->status = ereply->status;
1291         if (ereply->sense_len != 0) {
1292                 res = scst_alloc_sense(cmd, 0);
1293                 if (res != 0)
1294                         goto out_compl;
1295                 res = copy_from_user(cmd->sense,
1296                         (void *)(unsigned long)ereply->psense_buffer,
1297                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1298                                 (unsigned int)ereply->sense_len));
1299                 if (res < 0) {
1300                         PRINT_ERROR("%s", "Unable to get sense data");
1301                         goto out_hwerr_res_set;
1302                 }
1303         }
1304
1305 out_compl:
1306         cmd->completed = 1;
1307         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1308         /* !! At this point cmd can be already freed !! */
1309
1310 out:
1311         TRACE_EXIT_RES(res);
1312         return res;
1313
1314 out_inval:
1315         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1316                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1317         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1318
1319 out_hwerr:
1320         res = -EINVAL;
1321
1322 out_hwerr_res_set:
1323         if (ucmd->background_exec) {
1324                 ucmd_put(ucmd);
1325                 goto out;
1326         } else {
1327                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1328                 goto out_compl;
1329         }
1330
1331 out_busy:
1332         scst_set_busy(cmd);
1333         goto out_compl;
1334 }
1335
1336 static int dev_user_process_reply(struct scst_user_dev *dev,
1337         struct scst_user_reply_cmd *reply)
1338 {
1339         int res = 0;
1340         struct scst_user_cmd *ucmd;
1341         int state;
1342
1343         TRACE_ENTRY();
1344
1345         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1346
1347         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1348         if (unlikely(ucmd == NULL)) {
1349                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1350                 res = -ESRCH;
1351                 goto out_unlock;
1352         }
1353
1354         if (unlikely(ucmd_get_check(ucmd))) {
1355                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1356                 res = -ESRCH;
1357                 goto out_unlock;
1358         }
1359
1360         if (ucmd->background_exec) {
1361                 state = UCMD_STATE_EXECING;
1362                 goto unlock_process;
1363         }
1364
1365         if (unlikely(ucmd->this_state_unjammed)) {
1366                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1367                         ucmd);
1368                 goto out_unlock_put;
1369         }
1370
1371         if (unlikely(!ucmd->sent_to_user)) {
1372                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1373                         "state %x", ucmd, ucmd->state);
1374                 res = -EINVAL;
1375                 goto out_unlock_put;
1376         }
1377
1378         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1379                 goto out_wrong_state;
1380
1381         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1382                 goto out_wrong_state;
1383
1384         state = ucmd->state;
1385         ucmd->sent_to_user = 0;
1386
1387 unlock_process:
1388         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1389
1390         switch (state) {
1391         case UCMD_STATE_PARSING:
1392                 res = dev_user_process_reply_parse(ucmd, reply);
1393                 break;
1394
1395         case UCMD_STATE_BUF_ALLOCING:
1396                 res = dev_user_process_reply_alloc(ucmd, reply);
1397                 break;
1398
1399         case UCMD_STATE_EXECING:
1400                 res = dev_user_process_reply_exec(ucmd, reply);
1401                 break;
1402
1403         case UCMD_STATE_ON_FREEING:
1404                 res = dev_user_process_reply_on_free(ucmd);
1405                 break;
1406
1407         case UCMD_STATE_ON_CACHE_FREEING:
1408                 res = dev_user_process_reply_on_cache_free(ucmd);
1409                 break;
1410
1411         case UCMD_STATE_TM_EXECING:
1412                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1413                 break;
1414
1415         case UCMD_STATE_ATTACH_SESS:
1416         case UCMD_STATE_DETACH_SESS:
1417                 res = dev_user_process_reply_sess(ucmd, reply->result);
1418                 break;
1419
1420         default:
1421                 sBUG();
1422                 break;
1423         }
1424
1425 out_put:
1426         ucmd_put(ucmd);
1427
1428 out:
1429         TRACE_EXIT_RES(res);
1430         return res;
1431
1432 out_wrong_state:
1433         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1434                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1435                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1436                 reply->subcode, ucmd->user_cmd.subcode);
1437         res = -EINVAL;
1438         dev_user_unjam_cmd(ucmd, 0, NULL);
1439
1440 out_unlock_put:
1441         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1442         goto out_put;
1443
1444 out_unlock:
1445         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1446         goto out;
1447 }
1448
1449 static int dev_user_reply_cmd(struct file *file, unsigned long arg)
1450 {
1451         int res = 0;
1452         struct scst_user_dev *dev;
1453         struct scst_user_reply_cmd *reply;
1454
1455         TRACE_ENTRY();
1456
1457         mutex_lock(&dev_priv_mutex);
1458         dev = (struct scst_user_dev *)file->private_data;
1459         res = dev_user_check_reg(dev);
1460         if (res != 0) {
1461                 mutex_unlock(&dev_priv_mutex);
1462                 goto out;
1463         }
1464         down_read(&dev->dev_rwsem);
1465         mutex_unlock(&dev_priv_mutex);
1466
1467         reply = kzalloc(sizeof(*reply), GFP_KERNEL);
1468         if (reply == NULL) {
1469                 res = -ENOMEM;
1470                 goto out_up;
1471         }
1472
1473         res = copy_from_user(reply, (void *)arg, sizeof(*reply));
1474         if (res < 0)
1475                 goto out_free;
1476
1477         TRACE_BUFFER("Reply", reply, sizeof(*reply));
1478
1479         res = dev_user_process_reply(dev, reply);
1480         if (res < 0)
1481                 goto out_free;
1482
1483 out_free:
1484         kfree(reply);
1485
1486 out_up:
1487         up_read(&dev->dev_rwsem);
1488
1489 out:
1490         TRACE_EXIT_RES(res);
1491         return res;
1492 }
1493
1494 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1495 {
1496         int res = 0;
1497
1498         TRACE_ENTRY();
1499
1500         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1501                 struct scst_cmd *cmd = list_entry(
1502                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1503                         cmd_list_entry);
1504                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1505                 list_del(&cmd->cmd_list_entry);
1506                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1507                 scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT |
1508                                                  SCST_CONTEXT_PROCESSABLE);
1509                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1510                 res++;
1511         }
1512
1513         TRACE_EXIT_RES(res);
1514         return res;
1515 }
1516
1517 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1518 struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1519 {
1520         struct scst_user_cmd *u;
1521
1522 again:
1523         u = NULL;
1524         if (!list_empty(cmd_list)) {
1525                 u = list_entry(cmd_list->next, typeof(*u), ready_cmd_list_entry);
1526
1527                 TRACE_DBG("Found ready ucmd %p", u);
1528                 list_del(&u->ready_cmd_list_entry);
1529
1530                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1531
1532                 if (u->cmd != NULL) {
1533                         if (u->state == UCMD_STATE_EXECING) {
1534                                 struct scst_user_dev *dev = u->dev;
1535                                 int rc;
1536
1537                                 EXTRACHECKS_BUG_ON(u->jammed);
1538
1539                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1540
1541                                 rc = scst_check_local_events(u->cmd);
1542                                 if (unlikely(rc != 0)) {
1543                                         u->cmd->scst_cmd_done(u->cmd,
1544                                                 SCST_CMD_STATE_DEFAULT);
1545                                         /*
1546                                          * !! At this point cmd & u can be !!
1547                                          * !! already freed                !!
1548                                          */
1549                                         spin_lock_irq(
1550                                                 &dev->cmd_lists.cmd_list_lock);
1551                                         goto again;
1552                                 }
1553
1554                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1555                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1556                                         &u->cmd->cmd_flags))) {
1557                                 switch (u->state) {
1558                                 case UCMD_STATE_PARSING:
1559                                 case UCMD_STATE_BUF_ALLOCING:
1560                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1561                                         dev_user_unjam_cmd(u, 0, NULL);
1562                                         goto again;
1563                                 case UCMD_STATE_EXECING:
1564                                         EXTRACHECKS_BUG_ON(1);
1565                                 }
1566                         }
1567                 }
1568                 u->sent_to_user = 1;
1569                 u->seen_by_user = 1;
1570         }
1571         return u;
1572 }
1573
1574 static inline int test_cmd_lists(struct scst_user_dev *dev)
1575 {
1576         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1577                   !list_empty(&dev->ready_cmd_list) ||
1578                   !dev->blocking || dev->cleanup_done ||
1579                   signal_pending(current);
1580         return res;
1581 }
1582
1583 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1584 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1585         struct scst_user_cmd **ucmd)
1586 {
1587         int res = 0;
1588         wait_queue_t wait;
1589
1590         TRACE_ENTRY();
1591
1592         init_waitqueue_entry(&wait, current);
1593
1594         while (1) {
1595                 if (!test_cmd_lists(dev)) {
1596                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1597                                 &wait);
1598                         for (;;) {
1599                                 set_current_state(TASK_INTERRUPTIBLE);
1600                                 if (test_cmd_lists(dev))
1601                                         break;
1602                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1603                                 schedule();
1604                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1605                         }
1606                         set_current_state(TASK_RUNNING);
1607                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1608                                 &wait);
1609                 }
1610
1611                 dev_user_process_scst_commands(dev);
1612
1613                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1614                 if (*ucmd != NULL)
1615                         break;
1616
1617                 if (!dev->blocking || dev->cleanup_done) {
1618                         res = -EAGAIN;
1619                         TRACE_DBG("No ready commands, returning %d", res);
1620                         break;
1621                 }
1622
1623                 if (signal_pending(current)) {
1624                         res = -EINTR;
1625                         TRACE_DBG("Signal pending, returning %d", res);
1626                         break;
1627                 }
1628         }
1629
1630         TRACE_EXIT_RES(res);
1631         return res;
1632 }
1633
1634 static int dev_user_reply_get_cmd(struct file *file, unsigned long arg)
1635 {
1636         int res = 0;
1637         struct scst_user_dev *dev;
1638         struct scst_user_get_cmd *cmd;
1639         struct scst_user_reply_cmd *reply;
1640         struct scst_user_cmd *ucmd;
1641         uint64_t ureply;
1642
1643         TRACE_ENTRY();
1644
1645         mutex_lock(&dev_priv_mutex);
1646         dev = (struct scst_user_dev *)file->private_data;
1647         res = dev_user_check_reg(dev);
1648         if (res != 0) {
1649                 mutex_unlock(&dev_priv_mutex);
1650                 goto out;
1651         }
1652         down_read(&dev->dev_rwsem);
1653         mutex_unlock(&dev_priv_mutex);
1654
1655         res = copy_from_user(&ureply, (void *)arg, sizeof(ureply));
1656         if (res < 0)
1657                 goto out_up;
1658
1659         TRACE_DBG("ureply %Ld", (long long unsigned int)ureply);
1660
1661         cmd = kzalloc(max(sizeof(*cmd), sizeof(*reply)), GFP_KERNEL);
1662         if (cmd == NULL) {
1663                 res = -ENOMEM;
1664                 goto out_up;
1665         }
1666
1667         if (ureply != 0) {
1668                 unsigned long u = (unsigned long)ureply;
1669                 reply = (struct scst_user_reply_cmd *)cmd;
1670                 res = copy_from_user(reply, (void *)u, sizeof(*reply));
1671                 if (res < 0)
1672                         goto out_free;
1673
1674                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1675
1676                 res = dev_user_process_reply(dev, reply);
1677                 if (res < 0)
1678                         goto out_free;
1679         }
1680
1681         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1682         res = dev_user_get_next_cmd(dev, &ucmd);
1683         if (res == 0) {
1684                 *cmd = ucmd->user_cmd;
1685                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1686                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1687                 res = copy_to_user((void *)arg, cmd, sizeof(*cmd));
1688         } else
1689                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1690
1691 out_free:
1692         kfree(cmd);
1693
1694 out_up:
1695         up_read(&dev->dev_rwsem);
1696
1697 out:
1698         TRACE_EXIT_RES(res);
1699         return res;
1700 }
1701
1702 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1703         unsigned long arg)
1704 {
1705         long res;
1706
1707         TRACE_ENTRY();
1708
1709         switch (cmd) {
1710         case SCST_USER_REPLY_AND_GET_CMD:
1711                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1712                 res = dev_user_reply_get_cmd(file, arg);
1713                 break;
1714
1715         case SCST_USER_REPLY_CMD:
1716                 TRACE_DBG("%s", "REPLY_CMD");
1717                 res = dev_user_reply_cmd(file, arg);
1718                 break;
1719
1720         case SCST_USER_REGISTER_DEVICE:
1721         {
1722                 struct scst_user_dev_desc *dev_desc;
1723                 TRACE_DBG("%s", "REGISTER_DEVICE");
1724                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1725                 if (dev_desc == NULL) {
1726                         res = -ENOMEM;
1727                         goto out;
1728                 }
1729                 res = copy_from_user(dev_desc, (void *)arg, sizeof(*dev_desc));
1730                 if (res < 0) {
1731                         kfree(dev_desc);
1732                         goto out;
1733                 }
1734                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1735                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1736                 res = dev_user_register_dev(file, dev_desc);
1737                 kfree(dev_desc);
1738                 break;
1739         }
1740
1741         case SCST_USER_SET_OPTIONS:
1742         {
1743                 struct scst_user_opt opt;
1744                 TRACE_DBG("%s", "SET_OPTIONS");
1745                 res = copy_from_user(&opt, (void *)arg, sizeof(opt));
1746                 if (res < 0)
1747                         goto out;
1748                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1749                 res = dev_user_set_opt(file, &opt);
1750                 break;
1751         }
1752
1753         case SCST_USER_GET_OPTIONS:
1754                 TRACE_DBG("%s", "GET_OPTIONS");
1755                 res = dev_user_get_opt(file, (void *)arg);
1756                 break;
1757
1758         default:
1759                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1760                 res = -EINVAL;
1761                 goto out;
1762         }
1763
1764 out:
1765         TRACE_EXIT_RES(res);
1766         return res;
1767 }
1768
1769 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1770 {
1771         int res = 0;
1772         struct scst_user_dev *dev;
1773
1774         TRACE_ENTRY();
1775
1776         mutex_lock(&dev_priv_mutex);
1777         dev = (struct scst_user_dev *)file->private_data;
1778         res = dev_user_check_reg(dev);
1779         if (res != 0) {
1780                 mutex_unlock(&dev_priv_mutex);
1781                 goto out;
1782         }
1783         down_read(&dev->dev_rwsem);
1784         mutex_unlock(&dev_priv_mutex);
1785
1786         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1787
1788         if (!list_empty(&dev->ready_cmd_list) ||
1789             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1790                 res |= POLLIN | POLLRDNORM;
1791                 goto out_unlock;
1792         }
1793
1794         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1795
1796         TRACE_DBG("Before poll_wait() (dev %p)", dev);
1797         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1798         TRACE_DBG("After poll_wait() (dev %p)", dev);
1799
1800         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1801
1802         if (!list_empty(&dev->ready_cmd_list) ||
1803             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1804                 res |= POLLIN | POLLRDNORM;
1805                 goto out_unlock;
1806         }
1807
1808 out_unlock:
1809         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1810
1811         up_read(&dev->dev_rwsem);
1812
1813 out:
1814         TRACE_EXIT_HRES(res);
1815         return res;
1816 }
1817
1818 /*
1819  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reaquire.
1820  */
1821 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1822         unsigned long *flags)
1823 {
1824         int state = ucmd->state;
1825         struct scst_user_dev *dev = ucmd->dev;
1826
1827         TRACE_ENTRY();
1828
1829         if (ucmd->this_state_unjammed)
1830                 goto out;
1831
1832         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1833                 state);
1834
1835         ucmd->jammed = 1;
1836         ucmd->this_state_unjammed = 1;
1837         ucmd->sent_to_user = 0;
1838
1839         switch (state) {
1840         case UCMD_STATE_PARSING:
1841         case UCMD_STATE_BUF_ALLOCING:
1842                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1843                         ucmd->aborted = 1;
1844                 else {
1845                         if (busy)
1846                                 scst_set_busy(ucmd->cmd);
1847                         else
1848                                 scst_set_cmd_error(ucmd->cmd,
1849                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1850                 }
1851                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1852
1853                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1854                 list_add(&ucmd->cmd->cmd_list_entry,
1855                         &ucmd->cmd->cmd_lists->active_cmd_list);
1856                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1857                 break;
1858
1859         case UCMD_STATE_EXECING:
1860                 if (flags != NULL)
1861                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1862                 else
1863                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1864
1865                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1866
1867                 if (test_bit(SCST_CMD_ABORTED,  &ucmd->cmd->cmd_flags))
1868                         ucmd->aborted = 1;
1869                 else {
1870                         if (busy)
1871                                 scst_set_busy(ucmd->cmd);
1872                         else
1873                                 scst_set_cmd_error(ucmd->cmd,
1874                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1875                 }
1876
1877                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT);
1878                 /* !! At this point cmd and ucmd can be already freed !! */
1879
1880                 if (flags != NULL)
1881                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1882                 else
1883                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1884                 break;
1885
1886         case UCMD_STATE_ON_FREEING:
1887         case UCMD_STATE_ON_CACHE_FREEING:
1888         case UCMD_STATE_TM_EXECING:
1889         case UCMD_STATE_ATTACH_SESS:
1890         case UCMD_STATE_DETACH_SESS:
1891         {
1892                 if (flags != NULL)
1893                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1894                 else
1895                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1896
1897                 switch (state) {
1898                 case UCMD_STATE_ON_FREEING:
1899                         dev_user_process_reply_on_free(ucmd);
1900                         break;
1901
1902                 case UCMD_STATE_ON_CACHE_FREEING:
1903                         dev_user_process_reply_on_cache_free(ucmd);
1904                         break;
1905
1906                 case UCMD_STATE_TM_EXECING:
1907                         dev_user_process_reply_tm_exec(ucmd, SCST_MGMT_STATUS_FAILED);
1908                         break;
1909
1910                 case UCMD_STATE_ATTACH_SESS:
1911                 case UCMD_STATE_DETACH_SESS:
1912                         dev_user_process_reply_sess(ucmd, -EFAULT);
1913                         break;
1914                 }
1915
1916                 if (flags != NULL)
1917                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1918                 else
1919                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1920                 break;
1921         }
1922
1923         default:
1924                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1925                 sBUG();
1926                 break;
1927         }
1928
1929 out:
1930         TRACE_EXIT();
1931         return;
1932 }
1933
1934 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1935 {
1936         int i;
1937         struct scst_user_cmd *ucmd;
1938
1939         TRACE_ENTRY();
1940
1941         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1942
1943         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1944
1945 repeat:
1946         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1947                 struct list_head *head = &dev->ucmd_hash[i];
1948
1949                 list_for_each_entry(ucmd, head, hash_list_entry) {
1950                         if (!ucmd->sent_to_user)
1951                                 continue;
1952
1953                         if (ucmd_get_check(ucmd))
1954                                 continue;
1955
1956                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1957                                 ucmd->state, ucmd->cmd);
1958
1959                         dev_user_unjam_cmd(ucmd, 0, NULL);
1960
1961                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1962                         ucmd_put(ucmd);
1963                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1964
1965                         goto repeat;
1966                 }
1967         }
1968
1969         if (dev_user_process_scst_commands(dev) != 0)
1970                 goto repeat;
1971
1972         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1973
1974         TRACE_EXIT();
1975         return;
1976 }
1977
1978 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
1979         int status)
1980 {
1981         int res = 0;
1982
1983         TRACE_ENTRY();
1984
1985         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
1986                 ucmd->user_cmd.tm_cmd.fn, status);
1987
1988         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
1989                 /*
1990                  * It is possible that user space seen TM cmd before cmd
1991                  * to abort or will never see it at all, because it was
1992                  * aborted on the way there. So, it is safe to return
1993                  * success instead, because, if there is the TM cmd at this
1994                  * point, then the cmd to abort apparrently does exist.
1995                  */
1996                 status = SCST_MGMT_STATUS_SUCCESS;
1997         }
1998
1999         scst_async_mcmd_completed(ucmd->mcmd, status);
2000
2001         ucmd_put(ucmd);
2002
2003         TRACE_EXIT_RES(res);
2004         return res;
2005 }
2006
2007 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2008 {
2009         struct scst_user_cmd *ucmd;
2010         unsigned long flags;
2011
2012         TRACE_ENTRY();
2013
2014         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2015 again:
2016         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2017                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2018                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2019                         switch (ucmd->state) {
2020                         case UCMD_STATE_PARSING:
2021                         case UCMD_STATE_BUF_ALLOCING:
2022                         case UCMD_STATE_EXECING:
2023                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2024                                 list_del(&ucmd->ready_cmd_list_entry);
2025                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2026                                 goto again;
2027                         }
2028                 }
2029         }
2030
2031         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2032
2033         TRACE_EXIT();
2034         return;
2035 }
2036
2037 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2038         struct scst_tgt_dev *tgt_dev)
2039 {
2040         struct scst_user_cmd *ucmd;
2041         struct scst_user_dev *dev = (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2042         struct scst_user_cmd *ucmd_to_abort = NULL;
2043
2044         TRACE_ENTRY();
2045
2046         /*
2047          * In the used approach we don't do anything with hung devices, which
2048          * stopped responding and/or have stuck commands. We forcedly abort such
2049          * commands only if they not yet sent to the user space or if the device
2050          * is getting unloaded, e.g. if its handler program gets killed. This is
2051          * because it's pretty hard to distinguish between stuck and temporary
2052          * overloaded states of the device. There are several reasons for that:
2053          *
2054          * 1. Some commands need a lot of time to complete (several hours),
2055          *    so for an impatient user such command(s) will always look as
2056          *    stuck.
2057          *
2058          * 2. If we forcedly abort, i.e. abort before it's actually completed
2059          *    in the user space, just one command, we will have to put the whole
2060          *    device offline until we are sure that no more previously aborted
2061          *    commands will get executed. Otherwise, we might have a possibility
2062          *    for data corruption, when aborted and reported as completed
2063          *    command actually gets executed *after* new commands sent
2064          *    after the force abort was done. Many journaling file systems and
2065          *    databases use "provide required commands order via queue draining"
2066          *    approach and not putting the whole device offline after the forced
2067          *    abort will break it. This makes our decision, if a command stuck
2068          *    or not, cost a lot.
2069          *
2070          * So, we leave policy definition if a device stuck or not to
2071          * the user space and simply let all commands live until they are
2072          * completed or their devices get closed/killed. This approach is very
2073          * much OK, but can affect management commands, which need activity
2074          * suspending via scst_suspend_activity() function such as devices or
2075          * targets registration/removal. But during normal life such commands
2076          * should be rare. Plus, when possible, scst_suspend_activity() will
2077          * return after timeout EBUSY status to allow caller to not stuck
2078          * forever as well.
2079          *
2080          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2081          * stuck commands would affect only related devices.
2082          */
2083
2084         dev_user_abort_ready_commands(dev);
2085
2086         /* We can't afford missing TM command due to memory shortage */
2087         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2088
2089         ucmd->user_cmd.cmd_h = ucmd->h;
2090         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2091         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2092         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2093         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2094         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2095
2096         if (mcmd->cmd_to_abort != NULL) {
2097                 ucmd_to_abort = (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2098                 if (ucmd_to_abort != NULL)
2099                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2100         }
2101
2102         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2103                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2104                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2105                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2106
2107         ucmd->mcmd = mcmd;
2108         ucmd->state = UCMD_STATE_TM_EXECING;
2109
2110         scst_prepare_async_mcmd(mcmd);
2111
2112         dev_user_add_to_ready(ucmd);
2113
2114         TRACE_EXIT();
2115         return SCST_DEV_TM_NOT_COMPLETED;
2116 }
2117
2118 static int dev_user_attach(struct scst_device *sdev)
2119 {
2120         int res = 0;
2121         struct scst_user_dev *dev = NULL, *d;
2122
2123         TRACE_ENTRY();
2124
2125         spin_lock(&dev_list_lock);
2126         list_for_each_entry(d, &dev_list, dev_list_entry) {
2127                 if (strcmp(d->name, sdev->virt_name) == 0) {
2128                         dev = d;
2129                         break;
2130                 }
2131         }
2132         spin_unlock(&dev_list_lock);
2133         if (dev == NULL) {
2134                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2135                 res = -EINVAL;
2136                 goto out;
2137         }
2138
2139         sdev->p_cmd_lists = &dev->cmd_lists;
2140         sdev->dh_priv = dev;
2141         sdev->tst = dev->tst;
2142         sdev->queue_alg = dev->queue_alg;
2143         sdev->swp = dev->swp;
2144         sdev->tas = dev->tas;
2145         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2146
2147         dev->sdev = sdev;
2148
2149         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2150                 dev->name);
2151
2152 out:
2153         TRACE_EXIT();
2154         return res;
2155 }
2156
2157 static void dev_user_detach(struct scst_device *sdev)
2158 {
2159         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2160
2161         TRACE_ENTRY();
2162
2163         TRACE_DBG("virt_id %d", sdev->virt_id);
2164
2165         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2166                 dev->name);
2167
2168         /* dev will be freed by the caller */
2169         sdev->dh_priv = NULL;
2170         dev->sdev = NULL;
2171
2172         TRACE_EXIT();
2173         return;
2174 }
2175
2176 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2177 {
2178         int res = 0;
2179         unsigned long flags;
2180
2181         TRACE_ENTRY();
2182
2183         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2184
2185         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2186
2187         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2188                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2189                 ucmd->result = status;
2190         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2191                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2192         } else
2193                 sBUG();
2194
2195         if (ucmd->cmpl != NULL)
2196                 complete_all(ucmd->cmpl);
2197
2198         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2199
2200         ucmd_put(ucmd);
2201
2202         TRACE_EXIT_RES(res);
2203         return res;
2204 }
2205
2206 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2207 {
2208         struct scst_user_dev *dev =
2209                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2210         int res = 0, rc;
2211         struct scst_user_cmd *ucmd;
2212
2213         TRACE_ENTRY();
2214
2215         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2216         if (ucmd == NULL)
2217                 goto out_nomem;
2218
2219         ucmd->cmpl = kmalloc(sizeof(*ucmd->cmpl), GFP_KERNEL);
2220         if (ucmd->cmpl == NULL)
2221                 goto out_put_nomem;
2222
2223         init_completion(ucmd->cmpl);
2224
2225         ucmd->user_cmd.cmd_h = ucmd->h;
2226         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2227         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2228         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2229         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2230         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2231         strncpy(ucmd->user_cmd.sess.initiator_name,
2232                 tgt_dev->sess->initiator_name,
2233                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2234         ucmd->user_cmd.sess.initiator_name[
2235                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2236
2237         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %Lx, LUN %Lx, "
2238                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2239                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2240                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2241                 ucmd->user_cmd.sess.initiator_name);
2242
2243         ucmd->state = UCMD_STATE_ATTACH_SESS;
2244
2245         ucmd_get(ucmd);
2246
2247         dev_user_add_to_ready(ucmd);
2248
2249         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2250         if (rc > 0)
2251                 res = ucmd->result;
2252         else {
2253                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2254                 res = -EFAULT;
2255         }
2256
2257         sBUG_ON(irqs_disabled());
2258
2259         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2260         kfree(ucmd->cmpl);
2261         ucmd->cmpl = NULL;
2262         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2263
2264         ucmd_put(ucmd);
2265
2266 out:
2267         TRACE_EXIT_RES(res);
2268         return res;
2269
2270 out_put_nomem:
2271         ucmd_put(ucmd);
2272
2273 out_nomem:
2274         res = -ENOMEM;
2275         goto out;
2276 }
2277
2278 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2279 {
2280         struct scst_user_dev *dev =
2281                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2282         struct scst_user_cmd *ucmd;
2283
2284         TRACE_ENTRY();
2285
2286         /*
2287          * We can't miss TM command due to memory shortage, because it might
2288          * lead to a memory leak in the user space handler.
2289          */
2290         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2291         if (ucmd == NULL)
2292                 goto out;
2293
2294         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %Lx)", ucmd,
2295                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2296
2297         ucmd->user_cmd.cmd_h = ucmd->h;
2298         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2299         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2300
2301         ucmd->state = UCMD_STATE_DETACH_SESS;
2302
2303         dev_user_add_to_ready(ucmd);
2304
2305 out:
2306         TRACE_EXIT();
2307         return;
2308 }
2309
2310 /* No locks are needed, but the activity must be suspended */
2311 static void dev_user_setup_functions(struct scst_user_dev *dev)
2312 {
2313         TRACE_ENTRY();
2314
2315         dev->devtype.parse = dev_user_parse;
2316         dev->devtype.dev_done = NULL;
2317
2318         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2319                 switch (dev->devtype.type) {
2320                 case TYPE_DISK:
2321                         dev->generic_parse = scst_sbc_generic_parse;
2322                         dev->devtype.dev_done = dev_user_disk_done;
2323                         break;
2324
2325                 case TYPE_TAPE:
2326                         dev->generic_parse = scst_tape_generic_parse;
2327                         dev->devtype.dev_done = dev_user_tape_done;
2328                         break;
2329
2330                 case TYPE_MOD:
2331                         dev->generic_parse = scst_modisk_generic_parse;
2332                         dev->devtype.dev_done = dev_user_disk_done;
2333                         break;
2334
2335                 case TYPE_ROM:
2336                         dev->generic_parse = scst_cdrom_generic_parse;
2337                         dev->devtype.dev_done = dev_user_disk_done;
2338                         break;
2339
2340                 case TYPE_MEDIUM_CHANGER:
2341                         dev->generic_parse = scst_changer_generic_parse;
2342                         break;
2343
2344                 case TYPE_PROCESSOR:
2345                         dev->generic_parse = scst_processor_generic_parse;
2346                         break;
2347
2348                 case TYPE_RAID:
2349                         dev->generic_parse = scst_raid_generic_parse;
2350                         break;
2351
2352                 default:
2353                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2354                                 "for it", dev->devtype.type);
2355                         dev->parse_type = SCST_USER_PARSE_CALL;
2356                         break;
2357                 }
2358         } else {
2359                 dev->generic_parse = NULL;
2360                 dev->devtype.dev_done = NULL;
2361         }
2362
2363         TRACE_EXIT();
2364         return;
2365 }
2366
2367 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2368 {
2369         char ver[sizeof(DEV_USER_VERSION)+1];
2370         int res;
2371
2372         res = copy_from_user(ver, (void *)(unsigned long)dev_desc->version_str,
2373                                 sizeof(ver));
2374         if (res < 0) {
2375                 PRINT_ERROR("%s", "Unable to get version string");
2376                 goto out;
2377         }
2378         ver[sizeof(ver)-1] = '\0';
2379
2380         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2381                 /* ->name already 0-terminated in dev_user_ioctl() */
2382                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2383                         dev_desc->name, ver);
2384                 res = -EINVAL;
2385                 goto out;
2386         }
2387
2388 out:
2389         return res;
2390 }
2391
2392 static int dev_user_register_dev(struct file *file,
2393         const struct scst_user_dev_desc *dev_desc)
2394 {
2395         int res = -ENOMEM, i;
2396         struct scst_user_dev *dev, *d;
2397         int block;
2398
2399         TRACE_ENTRY();
2400
2401         res = dev_user_check_version(dev_desc);
2402         if (res != 0)
2403                 goto out;
2404
2405         switch (dev_desc->type) {
2406         case TYPE_DISK:
2407         case TYPE_ROM:
2408         case TYPE_MOD:
2409                 if (dev_desc->block_size == 0) {
2410                         PRINT_ERROR("Wrong block size %d", dev_desc->block_size);
2411                         res = -EINVAL;
2412                         goto out;
2413                 }
2414                 block = scst_calc_block_shift(dev_desc->block_size);
2415                 if (block == -1) {
2416                         res = -EINVAL;
2417                         goto out;
2418                 }
2419                 break;
2420         default:
2421                 block = dev_desc->block_size;
2422                 break;
2423         }
2424
2425         if (!try_module_get(THIS_MODULE)) {
2426                 PRINT_ERROR("%s", "Fail to get module");
2427                 goto out;
2428         }
2429
2430         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2431         if (dev == NULL)
2432                 goto out_put;
2433
2434         init_rwsem(&dev->dev_rwsem);
2435         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2436         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2437         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2438         INIT_LIST_HEAD(&dev->ready_cmd_list);
2439         if (file->f_flags & O_NONBLOCK) {
2440                 TRACE_DBG("%s", "Non-blocking operations");
2441                 dev->blocking = 0;
2442         } else
2443                 dev->blocking = 1;
2444         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2445                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2446
2447         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2448         dev->name[sizeof(dev->name)-1] = '\0';
2449
2450         scst_init_mem_lim(&dev->udev_mem_lim);
2451
2452         /*
2453          * We don't use clustered pool, since it implies pages reordering,
2454          * which isn't possible with user space supplied buffers. Although
2455          * it's still possible to cluster pages by the tail of each other,
2456          * seems it doesn't worth the effort.
2457          */
2458         dev->pool = sgv_pool_create(dev->name, 0);
2459         if (dev->pool == NULL)
2460                 goto out_put;
2461         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2462                 dev_user_free_sg_entries);
2463
2464         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2465                 dev->name);
2466         dev->devtype.type = dev_desc->type;
2467         dev->devtype.threads_num = -1;
2468         dev->devtype.parse_atomic = 1;
2469         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2470         dev->devtype.dev_done_atomic = 1;
2471         dev->devtype.no_proc = 1;
2472         dev->devtype.attach = dev_user_attach;
2473         dev->devtype.detach = dev_user_detach;
2474         dev->devtype.attach_tgt = dev_user_attach_tgt;
2475         dev->devtype.detach_tgt = dev_user_detach_tgt;
2476         dev->devtype.exec = dev_user_exec;
2477         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2478         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2479
2480         init_completion(&dev->cleanup_cmpl);
2481         dev->block = block;
2482         dev->def_block = block;
2483
2484         res = __dev_user_set_opt(dev, &dev_desc->opt);
2485
2486         TRACE_MEM("dev %p, name %s", dev, dev->name);
2487
2488         spin_lock(&dev_list_lock);
2489
2490         list_for_each_entry(d, &dev_list, dev_list_entry) {
2491                 if (strcmp(d->name, dev->name) == 0) {
2492                         PRINT_ERROR("Device %s already exist",
2493                                 dev->name);
2494                         res = -EEXIST;
2495                         spin_unlock(&dev_list_lock);
2496                         goto out_free;
2497                 }
2498         }
2499
2500         list_add_tail(&dev->dev_list_entry, &dev_list);
2501
2502         spin_unlock(&dev_list_lock);
2503
2504         if (res != 0)
2505                 goto out_del_free;
2506
2507         res = scst_register_virtual_dev_driver(&dev->devtype);
2508         if (res < 0)
2509                 goto out_del_free;
2510
2511         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2512         if (dev->virt_id < 0) {
2513                 res = dev->virt_id;
2514                 goto out_unreg_handler;
2515         }
2516
2517         mutex_lock(&dev_priv_mutex);
2518         if (file->private_data != NULL) {
2519                 mutex_unlock(&dev_priv_mutex);
2520                 PRINT_ERROR("%s", "Device already registered");
2521                 res = -EINVAL;
2522                 goto out_unreg_drv;
2523         }
2524         file->private_data = dev;
2525         mutex_unlock(&dev_priv_mutex);
2526
2527 out:
2528         TRACE_EXIT_RES(res);
2529         return res;
2530
2531 out_unreg_drv:
2532         scst_unregister_virtual_device(dev->virt_id);
2533
2534 out_unreg_handler:
2535         scst_unregister_virtual_dev_driver(&dev->devtype);
2536
2537 out_del_free:
2538         spin_lock(&dev_list_lock);
2539         list_del(&dev->dev_list_entry);
2540         spin_unlock(&dev_list_lock);
2541
2542 out_free:
2543         sgv_pool_destroy(dev->pool);
2544         kfree(dev);
2545         goto out_put;
2546
2547 out_put:
2548         module_put(THIS_MODULE);
2549         goto out;
2550 }
2551
2552 static int __dev_user_set_opt(struct scst_user_dev *dev,
2553         const struct scst_user_opt *opt)
2554 {
2555         int res = 0;
2556
2557         TRACE_ENTRY();
2558
2559         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2560                 "partial_transfers_type %x, partial_len %d", opt->parse_type,
2561                 opt->on_free_cmd_type, opt->memory_reuse_type,
2562                 opt->partial_transfers_type, opt->partial_len);
2563
2564         if ((opt->parse_type > SCST_USER_MAX_PARSE_OPT) ||
2565             (opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT) ||
2566             (opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT) ||
2567             (opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT)) {
2568                 PRINT_ERROR("%s", "Invalid option");
2569                 res = -EINVAL;
2570                 goto out;
2571         }
2572
2573         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2574              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2575             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2576              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2577             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2578                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x, "
2579                         "tas %x, has_own_order_mgmt %x)", opt->tst,
2580                         opt->queue_alg, opt->swp, opt->tas, opt->has_own_order_mgmt);
2581                 res = -EINVAL;
2582                 goto out;
2583         }
2584
2585         dev->parse_type = opt->parse_type;
2586         dev->on_free_cmd_type = opt->on_free_cmd_type;
2587         dev->memory_reuse_type = opt->memory_reuse_type;
2588         dev->partial_transfers_type = opt->partial_transfers_type;
2589         dev->partial_len = opt->partial_len;
2590
2591         dev->tst = opt->tst;
2592         dev->queue_alg = opt->queue_alg;
2593         dev->swp = opt->swp;
2594         dev->tas = opt->tas;
2595         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2596         if (dev->sdev != NULL) {
2597                 dev->sdev->tst = opt->tst;
2598                 dev->sdev->queue_alg = opt->queue_alg;
2599                 dev->sdev->swp = opt->swp;
2600                 dev->sdev->tas = opt->tas;
2601                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2602         }
2603
2604         dev_user_setup_functions(dev);
2605
2606 out:
2607         TRACE_EXIT_RES(res);
2608         return res;
2609 }
2610
2611 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2612 {
2613         int res = 0;
2614         struct scst_user_dev *dev;
2615
2616         TRACE_ENTRY();
2617
2618         mutex_lock(&dev_priv_mutex);
2619         dev = (struct scst_user_dev *)file->private_data;
2620         res = dev_user_check_reg(dev);
2621         if (res != 0) {
2622                 mutex_unlock(&dev_priv_mutex);
2623                 goto out;
2624         }
2625         down_write(&dev->dev_rwsem);
2626         mutex_unlock(&dev_priv_mutex);
2627
2628         res = scst_suspend_activity(true);
2629         if (res != 0)
2630                 goto out;
2631
2632         res = __dev_user_set_opt(dev, opt);
2633
2634         scst_resume_activity();
2635
2636         up_write(&dev->dev_rwsem);
2637
2638 out:
2639         TRACE_EXIT_RES(res);
2640         return res;
2641 }
2642
2643 static int dev_user_get_opt(struct file *file, void *arg)
2644 {
2645         int res = 0;
2646         struct scst_user_dev *dev;
2647         struct scst_user_opt opt;
2648
2649         TRACE_ENTRY();
2650
2651         mutex_lock(&dev_priv_mutex);
2652         dev = (struct scst_user_dev *)file->private_data;
2653         res = dev_user_check_reg(dev);
2654         if (res != 0) {
2655                 mutex_unlock(&dev_priv_mutex);
2656                 goto out;
2657         }
2658         down_read(&dev->dev_rwsem);
2659         mutex_unlock(&dev_priv_mutex);
2660
2661         opt.parse_type = dev->parse_type;
2662         opt.on_free_cmd_type = dev->on_free_cmd_type;
2663         opt.memory_reuse_type = dev->memory_reuse_type;
2664         opt.partial_transfers_type = dev->partial_transfers_type;
2665         opt.partial_len = dev->partial_len;
2666         opt.tst = dev->tst;
2667         opt.queue_alg = dev->queue_alg;
2668         opt.tas = dev->tas;
2669         opt.swp = dev->swp;
2670         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2671
2672         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2673                 "partial_transfers_type %x, partial_len %d", opt.parse_type,
2674                 opt.on_free_cmd_type, opt.memory_reuse_type,
2675                 opt.partial_transfers_type, opt.partial_len);
2676
2677         res = copy_to_user(arg, &opt, sizeof(opt));
2678
2679         up_read(&dev->dev_rwsem);
2680 out:
2681         TRACE_EXIT_RES(res);
2682         return res;
2683 }
2684
2685 static int dev_usr_parse(struct scst_cmd *cmd)
2686 {
2687         sBUG();
2688         return SCST_CMD_STATE_DEFAULT;
2689 }
2690
2691 /* Needed only for /proc support */
2692 #define USR_TYPE {                      \
2693         .name =         DEV_USER_NAME,  \
2694         .type =         -1,             \
2695         .parse =        dev_usr_parse,  \
2696 }
2697
2698 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2699
2700 static int dev_user_release(struct inode *inode, struct file *file)
2701 {
2702         int res = 0;
2703         struct scst_user_dev *dev;
2704
2705         TRACE_ENTRY();
2706
2707         mutex_lock(&dev_priv_mutex);
2708         dev = (struct scst_user_dev *)file->private_data;
2709         if (dev == NULL) {
2710                 mutex_unlock(&dev_priv_mutex);
2711                 goto out;
2712         }
2713         file->private_data = NULL;
2714
2715         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2716
2717         spin_lock(&dev_list_lock);
2718         list_del(&dev->dev_list_entry);
2719         spin_unlock(&dev_list_lock);
2720
2721         mutex_unlock(&dev_priv_mutex);
2722
2723         down_write(&dev->dev_rwsem);
2724
2725         spin_lock(&cleanup_lock);
2726         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2727         spin_unlock(&cleanup_lock);
2728
2729         wake_up(&cleanup_list_waitQ);
2730         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2731
2732         scst_unregister_virtual_device(dev->virt_id);
2733         scst_unregister_virtual_dev_driver(&dev->devtype);
2734
2735         sgv_pool_destroy(dev->pool);
2736
2737         TRACE_DBG("Unregistering finished (dev %p)", dev);
2738
2739         dev->cleanup_done = 1;
2740
2741         wake_up(&cleanup_list_waitQ);
2742         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2743
2744         wait_for_completion(&dev->cleanup_cmpl);
2745
2746         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2747
2748         TRACE_DBG("Releasing completed (dev %p)", dev);
2749
2750         kfree(dev);
2751
2752         module_put(THIS_MODULE);
2753
2754 out:
2755         TRACE_EXIT_RES(res);
2756         return res;
2757 }
2758
2759 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2760 {
2761         struct scst_user_cmd *ucmd;
2762         int rc, res = 1;
2763
2764         TRACE_ENTRY();
2765
2766         dev->blocking = 0;
2767
2768         while (1) {
2769                 TRACE_DBG("Cleanuping dev %p", dev);
2770
2771                 dev_user_unjam_dev(dev);
2772
2773                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2774
2775                 rc = dev_user_get_next_cmd(dev, &ucmd);
2776                 if (rc == 0)
2777                         dev_user_unjam_cmd(ucmd, 1, NULL);
2778
2779                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2780
2781                 if (rc == -EAGAIN) {
2782                         if (dev->cleanup_done)
2783                                 break;
2784                         else {
2785                                 TRACE_DBG("No more commands (dev %p)", dev);
2786                                 goto out;
2787                         }
2788                 }
2789         }
2790
2791 #ifdef EXTRACHECKS
2792 {
2793         int i;
2794         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2795                 struct list_head *head = &dev->ucmd_hash[i];
2796                 struct scst_user_cmd *ucmd;
2797 again:
2798                 list_for_each_entry(ucmd, head, hash_list_entry) {
2799                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd,
2800                                 ucmd->state, atomic_read(&ucmd->ucmd_ref));
2801                         ucmd_put(ucmd);
2802                         goto again;
2803                 }
2804         }
2805 }
2806 #endif
2807
2808         TRACE_DBG("Cleanuping done (dev %p)", dev);
2809         complete_all(&dev->cleanup_cmpl);
2810         res = 0;
2811
2812 out:
2813         TRACE_EXIT_RES(res);
2814         return res;
2815 }
2816
2817 static inline int test_cleanup_list(void)
2818 {
2819         int res = !list_empty(&cleanup_list) ||
2820                   unlikely(kthread_should_stop());
2821         return res;
2822 }
2823
2824 static int dev_user_cleanup_thread(void *arg)
2825 {
2826         TRACE_ENTRY();
2827
2828         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2829
2830         current->flags |= PF_NOFREEZE;
2831
2832         spin_lock(&cleanup_lock);
2833         while (!kthread_should_stop()) {
2834                 wait_queue_t wait;
2835                 init_waitqueue_entry(&wait, current);
2836
2837                 if (!test_cleanup_list()) {
2838                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2839                         for (;;) {
2840                                 set_current_state(TASK_INTERRUPTIBLE);
2841                                 if (test_cleanup_list())
2842                                         break;
2843                                 spin_unlock(&cleanup_lock);
2844                                 schedule();
2845                                 spin_lock(&cleanup_lock);
2846                         }
2847                         set_current_state(TASK_RUNNING);
2848                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2849                 }
2850
2851                 /*
2852                  * We have to poll devices, because commands can go from SCST
2853                  * core on cmd_list_waitQ and we have no practical way to
2854                  * detect them.
2855                  */
2856
2857                 while (1) {
2858                         struct scst_user_dev *dev;
2859                         LIST_HEAD(cl_devs);
2860
2861                         while (!list_empty(&cleanup_list)) {
2862                                 int rc;
2863
2864                                 dev = list_entry(cleanup_list.next,
2865                                         typeof(*dev), cleanup_list_entry);
2866                                 list_del(&dev->cleanup_list_entry);
2867
2868                                 spin_unlock(&cleanup_lock);
2869                                 rc = dev_user_process_cleanup(dev);
2870                                 spin_lock(&cleanup_lock);
2871
2872                                 if (rc != 0)
2873                                         list_add_tail(&dev->cleanup_list_entry,
2874                                                 &cl_devs);
2875                         }
2876
2877                         if (list_empty(&cl_devs))
2878                                 break;
2879
2880                         spin_unlock(&cleanup_lock);
2881                         msleep(100);
2882                         spin_lock(&cleanup_lock);
2883
2884                         while (!list_empty(&cl_devs)) {
2885                                 dev = list_entry(cl_devs.next, typeof(*dev),
2886                                         cleanup_list_entry);
2887                                 list_move_tail(&dev->cleanup_list_entry,
2888                                         &cleanup_list);
2889                         }
2890                 }
2891         }
2892         spin_unlock(&cleanup_lock);
2893
2894         /*
2895          * If kthread_should_stop() is true, we are guaranteed to be
2896          * on the module unload, so cleanup_list must be empty.
2897          */
2898         sBUG_ON(!list_empty(&cleanup_list));
2899
2900         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2901
2902         TRACE_EXIT();
2903         return 0;
2904 }
2905
2906 static int __init init_scst_user(void)
2907 {
2908         int res = 0;
2909         struct class_device *class_member;
2910
2911         TRACE_ENTRY();
2912
2913 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2914         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2915                 "Consider change VMSPLIT option or use 64-bit "
2916                 "configuration instead. See README file for details.");
2917         res = -EINVAL;
2918         goto out;
2919 #endif
2920
2921         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2922         if (user_cmd_cachep == NULL) {
2923                 res = -ENOMEM;
2924                 goto out;
2925         }
2926
2927         dev_user_devtype.module = THIS_MODULE;
2928
2929         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2930         if (res < 0)
2931                 goto out_cache;
2932
2933         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2934         if (res != 0)
2935                 goto out_unreg;
2936
2937         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2938         if (IS_ERR(dev_user_sysfs_class)) {
2939                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2940                         "space handler");
2941                 res = PTR_ERR(dev_user_sysfs_class);
2942                 goto out_proc;
2943         }
2944
2945         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2946         if (res) {
2947                 PRINT_ERROR("Unable to get major %d for SCSI tapes", DEV_USER_MAJOR);
2948                 goto out_class;
2949         }
2950
2951         class_member = class_device_create(dev_user_sysfs_class, NULL,
2952                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
2953         if (IS_ERR(class_member)) {
2954                 res = PTR_ERR(class_member);
2955                 goto out_chrdev;
2956         }
2957
2958         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
2959                 "scst_usr_cleanupd");
2960         if (IS_ERR(cleanup_thread)) {
2961                 res = PTR_ERR(cleanup_thread);
2962                 PRINT_ERROR("kthread_create() failed: %d", res);
2963                 goto out_dev;
2964         }
2965
2966 out:
2967         TRACE_EXIT_RES(res);
2968         return res;
2969
2970 out_dev:
2971         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2972
2973 out_chrdev:
2974         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2975
2976 out_class:
2977         class_destroy(dev_user_sysfs_class);
2978
2979 out_proc:
2980         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
2981
2982 out_unreg:
2983         scst_unregister_dev_driver(&dev_user_devtype);
2984
2985 out_cache:
2986         kmem_cache_destroy(user_cmd_cachep);
2987         goto out;
2988 }
2989
2990 static void __exit exit_scst_user(void)
2991 {
2992         int rc;
2993
2994         TRACE_ENTRY();
2995
2996         rc = kthread_stop(cleanup_thread);
2997         if (rc < 0)
2998                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2999
3000         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3001         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3002         class_destroy(dev_user_sysfs_class);
3003
3004         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3005         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3006
3007         kmem_cache_destroy(user_cmd_cachep);
3008
3009         TRACE_EXIT();
3010         return;
3011 }
3012
3013 module_init(init_scst_user);
3014 module_exit(exit_scst_user);
3015
3016 MODULE_AUTHOR("Vladislav Bolkhovitin");
3017 MODULE_LICENSE("GPL");
3018 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3019 MODULE_VERSION(SCST_VERSION_STRING);
3020 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);