- Possibility to operate with single size entries as well as control of the reclamat...
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2009 ID7 Ltd.
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23 #include <linux/stddef.h>
24
25 #define LOG_PREFIX              DEV_USER_NAME
26
27 #include "scst.h"
28 #include "scst_user.h"
29 #include "scst_dev_handler.h"
30
31 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
32 #warning "HIGHMEM kernel configurations are not supported by this module,\
33  because nowadays it doesn't worth the effort. Consider changing\
34  VMSPLIT option or use a 64-bit configuration instead. See README file\
35  for details."
36 #endif
37
38 #define DEV_USER_MAJOR                  237
39 #define DEV_USER_CMD_HASH_ORDER         6
40 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
41
42 struct scst_user_dev {
43         struct rw_semaphore dev_rwsem;
44
45         struct scst_cmd_lists cmd_lists;
46
47         /* Protected by cmd_lists.cmd_list_lock */
48         struct list_head ready_cmd_list;
49
50         /* Protected by dev_rwsem or don't need any protection */
51         unsigned int blocking:1;
52         unsigned int cleanup_done:1;
53         unsigned int tst:3;
54         unsigned int queue_alg:4;
55         unsigned int tas:1;
56         unsigned int swp:1;
57         unsigned int d_sense:1;
58         unsigned int has_own_order_mgmt:1;
59
60         int (*generic_parse)(struct scst_cmd *cmd,
61                 int (*get_block)(struct scst_cmd *cmd));
62
63         int block;
64         int def_block;
65
66         struct scst_mem_lim udev_mem_lim;
67         struct sgv_pool *pool;
68         struct sgv_pool *pool_clust;
69
70         uint8_t parse_type;
71         uint8_t on_free_cmd_type;
72         uint8_t memory_reuse_type;
73         uint8_t partial_transfers_type;
74         uint32_t partial_len;
75
76         struct scst_dev_type devtype;
77
78         /* Both protected by cmd_lists.cmd_list_lock */
79         unsigned int handle_counter;
80         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
81
82         struct scst_device *sdev;
83
84         int virt_id;
85         struct list_head dev_list_entry;
86         char name[SCST_MAX_NAME];
87
88         struct list_head cleanup_list_entry;
89         struct completion cleanup_cmpl;
90 };
91
92 /* Most fields are unprotected, since only one thread at time can access them */
93 struct scst_user_cmd {
94         struct scst_cmd *cmd;
95         struct scst_user_dev *dev;
96
97         atomic_t ucmd_ref;
98
99         unsigned int buff_cached:1;
100         unsigned int buf_dirty:1;
101         unsigned int background_exec:1;
102         unsigned int aborted:1;
103
104         struct scst_user_cmd *buf_ucmd;
105
106         int cur_data_page;
107         int num_data_pages;
108         int first_page_offset;
109         unsigned long ubuff;
110         struct page **data_pages;
111         struct sgv_pool_obj *sgv;
112
113         /*
114          * Special flags, which can be accessed asynchronously (hence "long").
115          * Protected by cmd_lists.cmd_list_lock.
116          */
117         unsigned long sent_to_user:1;
118         unsigned long jammed:1;
119         unsigned long this_state_unjammed:1;
120         unsigned long seen_by_user:1; /* here only as a small optimization */
121
122         unsigned int state;
123
124         struct list_head ready_cmd_list_entry;
125
126         unsigned int h;
127         struct list_head hash_list_entry;
128
129         int user_cmd_payload_len;
130         struct scst_user_get_cmd user_cmd;
131
132         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
133         union {
134                 struct completion *cmpl;
135                 struct scst_mgmt_cmd *mcmd;
136         };
137         int result;
138 };
139
140 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
141         gfp_t gfp_mask);
142 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
143
144 static int dev_user_parse(struct scst_cmd *cmd);
145 static int dev_user_exec(struct scst_cmd *cmd);
146 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
147 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
148         struct scst_tgt_dev *tgt_dev);
149
150 static int dev_user_disk_done(struct scst_cmd *cmd);
151 static int dev_user_tape_done(struct scst_cmd *cmd);
152
153 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
154         gfp_t gfp_mask, void *priv);
155 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
156                                      void *priv);
157
158 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
159
160 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
161         unsigned long *flags);
162
163 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
164 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
165         int status);
166 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
167 static int dev_user_register_dev(struct file *file,
168         const struct scst_user_dev_desc *dev_desc);
169 static int dev_user_unregister_dev(struct file *file);
170 static int dev_user_flush_cache(struct file *file);
171 static int dev_user_capacity_changed(struct file *file);
172 static int __dev_user_set_opt(struct scst_user_dev *dev,
173         const struct scst_user_opt *opt);
174 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
175 static int dev_user_get_opt(struct file *file, void __user *arg);
176
177 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
178 static long dev_user_ioctl(struct file *file, unsigned int cmd,
179         unsigned long arg);
180 static int dev_user_release(struct inode *inode, struct file *file);
181 static int dev_user_exit_dev(struct scst_user_dev *dev);
182 static int dev_user_read_proc(struct seq_file *seq,
183         struct scst_dev_type *dev_type);
184
185 /** Data **/
186
187 static struct kmem_cache *user_cmd_cachep;
188 static struct kmem_cache *user_get_cmd_cachep;
189
190 static DEFINE_MUTEX(dev_priv_mutex);
191
192 static const struct file_operations dev_user_fops = {
193         .poll           = dev_user_poll,
194         .unlocked_ioctl = dev_user_ioctl,
195 #ifdef CONFIG_COMPAT
196         .compat_ioctl   = dev_user_ioctl,
197 #endif
198         .release        = dev_user_release,
199 };
200
201 static struct class *dev_user_sysfs_class;
202
203 static DEFINE_SPINLOCK(dev_list_lock);
204 static LIST_HEAD(dev_list);
205
206 static DEFINE_SPINLOCK(cleanup_lock);
207 static LIST_HEAD(cleanup_list);
208 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
209 static struct task_struct *cleanup_thread;
210
211 /*
212  * Skip this command if result is not 0. Must be called under
213  * cmd_lists.cmd_list_lock and IRQ off.
214  */
215 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
216 {
217         int r = atomic_inc_return(&ucmd->ucmd_ref);
218         int res;
219         if (unlikely(r == 1)) {
220                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
221                 atomic_dec(&ucmd->ucmd_ref);
222                 res = true;
223                 /*
224                  * Necessary code is serialized by cmd_list_lock in
225                  * cmd_remove_hash()
226                  */
227         } else {
228                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
229                         atomic_read(&ucmd->ucmd_ref));
230                 res = false;
231         }
232         return res;
233 }
234
235 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
236 {
237         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
238         atomic_inc(&ucmd->ucmd_ref);
239         if (barrier)
240                 smp_mb__after_atomic_inc();
241 }
242
243 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
244 {
245         __ucmd_get(ucmd, true);
246 }
247
248 static inline void ucmd_get(struct scst_user_cmd *ucmd)
249 {
250         __ucmd_get(ucmd, false);
251 }
252
253 /* Must not be called under cmd_list_lock!! */
254 static inline void ucmd_put(struct scst_user_cmd *ucmd)
255 {
256         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
257
258         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
259
260         if (atomic_dec_and_test(&ucmd->ucmd_ref))
261                 dev_user_free_ucmd(ucmd);
262 }
263
264 static inline int calc_num_pg(unsigned long buf, int len)
265 {
266         len += buf & ~PAGE_MASK;
267         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
268 }
269
270 static void __dev_user_not_reg(void)
271 {
272         TRACE(TRACE_MGMT_MINOR, "%s", "Device not registered");
273         return;
274 }
275
276 static inline int dev_user_check_reg(struct scst_user_dev *dev)
277 {
278         if (dev == NULL) {
279                 __dev_user_not_reg();
280                 return -ENODEV;
281         }
282         return 0;
283 }
284
285 static inline int scst_user_cmd_hashfn(int h)
286 {
287         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
288 }
289
290 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
291         unsigned int h)
292 {
293         struct list_head *head;
294         struct scst_user_cmd *ucmd;
295
296         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
297         list_for_each_entry(ucmd, head, hash_list_entry) {
298                 if (ucmd->h == h) {
299                         TRACE_DBG("Found ucmd %p", ucmd);
300                         return ucmd;
301                 }
302         }
303         return NULL;
304 }
305
306 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
307 {
308         struct list_head *head;
309         struct scst_user_dev *dev = ucmd->dev;
310         struct scst_user_cmd *u;
311         unsigned long flags;
312
313         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
314         do {
315                 ucmd->h = dev->handle_counter++;
316                 u = __ucmd_find_hash(dev, ucmd->h);
317         } while (u != NULL);
318         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
319         list_add_tail(&ucmd->hash_list_entry, head);
320         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
321
322         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
323         return;
324 }
325
326 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
327 {
328         unsigned long flags;
329
330         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
331         list_del(&ucmd->hash_list_entry);
332         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
333
334         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
335         return;
336 }
337
338 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
339 {
340         TRACE_ENTRY();
341
342         TRACE_MEM("Freeing ucmd %p", ucmd);
343
344         cmd_remove_hash(ucmd);
345         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
346
347         kmem_cache_free(user_cmd_cachep, ucmd);
348
349         TRACE_EXIT();
350         return;
351 }
352
353 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
354         gfp_t gfp_mask, void *priv)
355 {
356         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
357         int offset = 0;
358
359         TRACE_ENTRY();
360
361         /* *sg supposed to be zeroed */
362
363         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
364                 ucmd->ubuff, ucmd->cur_data_page);
365
366         if (ucmd->cur_data_page == 0) {
367                 TRACE_MEM("ucmd->first_page_offset %d",
368                         ucmd->first_page_offset);
369                 offset = ucmd->first_page_offset;
370                 ucmd_get(ucmd);
371         }
372
373         if (ucmd->cur_data_page >= ucmd->num_data_pages)
374                 goto out;
375
376         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
377                 PAGE_SIZE - offset, offset);
378         ucmd->cur_data_page++;
379
380         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
381                 sg->offset);
382         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
383
384 out:
385         TRACE_EXIT();
386         return sg_page(sg);
387 }
388
389 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
390 {
391         TRACE_ENTRY();
392
393         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
394                 ucmd, ucmd->h, ucmd->ubuff);
395
396         ucmd->user_cmd_payload_len =
397                 offsetof(struct scst_user_get_cmd, on_cached_mem_free) +
398                 sizeof(ucmd->user_cmd.on_cached_mem_free);
399         ucmd->user_cmd.cmd_h = ucmd->h;
400         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
401         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
402
403         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
404
405         dev_user_add_to_ready(ucmd);
406
407         TRACE_EXIT();
408         return;
409 }
410
411 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
412 {
413         int i;
414
415         TRACE_ENTRY();
416
417         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
418                 ucmd->ubuff, ucmd->num_data_pages);
419
420         for (i = 0; i < ucmd->num_data_pages; i++) {
421                 struct page *page = ucmd->data_pages[i];
422
423                 if (ucmd->buf_dirty)
424                         SetPageDirty(page);
425
426                 page_cache_release(page);
427         }
428
429         kfree(ucmd->data_pages);
430         ucmd->data_pages = NULL;
431
432         TRACE_EXIT();
433         return;
434 }
435
436 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
437 {
438         TRACE_ENTRY();
439
440         sBUG_ON(ucmd->data_pages == NULL);
441
442         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
443                 ucmd, ucmd->ubuff, ucmd->buff_cached);
444
445         dev_user_unmap_buf(ucmd);
446
447         if (ucmd->buff_cached)
448                 dev_user_on_cached_mem_free(ucmd);
449         else
450                 ucmd_put(ucmd);
451
452         TRACE_EXIT();
453         return;
454 }
455
456 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
457         void *priv)
458 {
459         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
460
461         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
462                 sg_count, ucmd);
463
464         __dev_user_free_sg_entries(ucmd);
465
466         return;
467 }
468
469 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
470 {
471         int mem_reuse_type = ucmd->dev->memory_reuse_type;
472
473         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
474             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
475              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
476             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
477              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
478                 return 1;
479         else
480                 return 0;
481 }
482
483 static inline int is_need_offs_page(unsigned long buf, int len)
484 {
485         return ((buf & ~PAGE_MASK) != 0) &&
486                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
487 }
488
489 /*
490  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
491  * Unmaps the buffer, if needed in case of error
492  */
493 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
494 {
495         int res = 0;
496         struct scst_cmd *cmd = ucmd->cmd;
497         struct scst_user_dev *dev = ucmd->dev;
498         struct sgv_pool *pool;
499         gfp_t gfp_mask;
500         int flags = 0;
501         int bufflen, orig_bufflen;
502         int last_len = 0;
503         int out_sg_pages = 0;
504
505         TRACE_ENTRY();
506
507         gfp_mask = __GFP_NOWARN;
508         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
509
510         if (cmd->data_direction != SCST_DATA_BIDI) {
511                 orig_bufflen = cmd->bufflen;
512                 pool = (struct sgv_pool *)cmd->tgt_dev->dh_priv;
513         } else {
514                 /* Make in_sg->offset 0 */
515                 int len = cmd->bufflen + ucmd->first_page_offset;
516                 out_sg_pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
517                 orig_bufflen = (out_sg_pages << PAGE_SHIFT) + cmd->in_bufflen;
518                 pool = dev->pool;
519         }
520         bufflen = orig_bufflen;
521
522         EXTRACHECKS_BUG_ON(bufflen == 0);
523
524         if (cached_buff) {
525                 flags |= SGV_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
526                 if (ucmd->ubuff == 0)
527                         flags |= SGV_POOL_NO_ALLOC_ON_CACHE_MISS;
528         } else {
529                 TRACE_MEM("%s", "Not cached buff");
530                 flags |= SGV_POOL_ALLOC_NO_CACHED;
531                 if (ucmd->ubuff == 0) {
532                         res = 1;
533                         goto out;
534                 }
535                 bufflen += ucmd->first_page_offset;
536                 if (is_need_offs_page(ucmd->ubuff, orig_bufflen))
537                         last_len = bufflen & ~PAGE_MASK;
538                 else
539                         last_len = orig_bufflen & ~PAGE_MASK;
540                 if (last_len == 0)
541                         last_len = PAGE_SIZE;
542         }
543         ucmd->buff_cached = cached_buff;
544
545         cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt,
546                         &ucmd->sgv, &dev->udev_mem_lim, ucmd);
547         if (cmd->sg != NULL) {
548                 struct scst_user_cmd *buf_ucmd =
549                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
550
551                 TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
552                         "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
553                         cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
554
555                 ucmd->ubuff = buf_ucmd->ubuff;
556                 ucmd->buf_ucmd = buf_ucmd;
557
558                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
559                                    (ucmd != buf_ucmd));
560
561                 if (last_len != 0) {
562                         cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
563                         cmd->sg[cmd->sg_cnt-1].length += last_len;
564                 }
565
566                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
567                         "last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
568                         cmd->sg[cmd->sg_cnt-1].length);
569
570                 if (cmd->data_direction == SCST_DATA_BIDI) {
571                         cmd->in_sg = &cmd->sg[out_sg_pages];
572                         cmd->in_sg_cnt = cmd->sg_cnt - out_sg_pages;
573                         cmd->sg_cnt = out_sg_pages;
574                         TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d",
575                                 cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt);
576                 }
577
578                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
579                         static int ll;
580                         if (ll < 10) {
581                                 PRINT_INFO("Unable to complete command due to "
582                                         "SG IO count limitation (requested %d, "
583                                         "available %d, tgt lim %d)",
584                                         cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt,
585                                         cmd->tgt->sg_tablesize);
586                                 ll++;
587                         }
588                         cmd->sg = NULL;
589                         /* sgv will be freed in dev_user_free_sgv() */
590                         res = -1;
591                 }
592         } else {
593                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
594                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
595                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
596                 if (unlikely(cmd->sg_cnt == 0)) {
597                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
598                         sBUG_ON(ucmd->sgv != NULL);
599                         res = -1;
600                 } else {
601                         switch (ucmd->state) {
602                         case UCMD_STATE_BUF_ALLOCING:
603                                 res = 1;
604                                 break;
605                         case UCMD_STATE_EXECING:
606                                 res = -1;
607                                 break;
608                         default:
609                                 sBUG();
610                                 break;
611                         }
612                 }
613         }
614
615 out:
616         TRACE_EXIT_RES(res);
617         return res;
618 }
619
620 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
621 {
622         int rc, res = SCST_CMD_STATE_DEFAULT;
623         struct scst_cmd *cmd = ucmd->cmd;
624
625         TRACE_ENTRY();
626
627         ucmd->state = UCMD_STATE_BUF_ALLOCING;
628         cmd->dh_data_buf_alloced = 1;
629
630         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
631         if (rc == 0)
632                 goto out;
633         else if (rc < 0) {
634                 scst_set_busy(cmd);
635                 res = scst_get_cmd_abnormal_done_state(cmd);
636                 goto out;
637         }
638
639         if ((cmd->data_direction != SCST_DATA_WRITE) &&
640             !scst_is_cmd_local(cmd)) {
641                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
642                 goto out;
643         }
644
645         ucmd->user_cmd_payload_len =
646                 offsetof(struct scst_user_get_cmd, alloc_cmd) +
647                 sizeof(ucmd->user_cmd.alloc_cmd);
648         ucmd->user_cmd.cmd_h = ucmd->h;
649         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
650         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
651         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb, cmd->cdb_len);
652         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
653         ucmd->user_cmd.alloc_cmd.ext_cdb_len = cmd->ext_cdb_len;
654         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
655                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
656         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
657         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
658         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
659
660         dev_user_add_to_ready(ucmd);
661
662         res = SCST_CMD_STATE_STOP;
663
664 out:
665         TRACE_EXIT_RES(res);
666         return res;
667 }
668
669 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
670         gfp_t gfp_mask)
671 {
672         struct scst_user_cmd *ucmd = NULL;
673
674         TRACE_ENTRY();
675
676 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
677         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
678         if (ucmd != NULL)
679                 memset(ucmd, 0, sizeof(*ucmd));
680 #else
681         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
682 #endif
683         if (unlikely(ucmd == NULL)) {
684                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
685                         "user cmd (gfp_mask %x)", gfp_mask);
686                 goto out;
687         }
688         ucmd->dev = dev;
689         atomic_set(&ucmd->ucmd_ref, 1);
690
691         cmd_insert_hash(ucmd);
692
693         TRACE_MEM("ucmd %p allocated", ucmd);
694
695 out:
696         TRACE_EXIT_HRES((unsigned long)ucmd);
697         return ucmd;
698 }
699
700 static int dev_user_get_block(struct scst_cmd *cmd)
701 {
702         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
703         /*
704          * No need for locks here, since *_detach() can not be
705          * called, when there are existing commands.
706          */
707         TRACE_EXIT_RES(dev->block);
708         return dev->block;
709 }
710
711 static int dev_user_parse(struct scst_cmd *cmd)
712 {
713         int rc, res = SCST_CMD_STATE_DEFAULT;
714         struct scst_user_cmd *ucmd;
715         int atomic = scst_cmd_atomic(cmd);
716         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
717         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
718
719         TRACE_ENTRY();
720
721         if (cmd->dh_priv == NULL) {
722                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
723                 if (unlikely(ucmd == NULL)) {
724                         if (atomic) {
725                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
726                                 goto out;
727                         } else {
728                                 scst_set_busy(cmd);
729                                 goto out_error;
730                         }
731                 }
732                 ucmd->cmd = cmd;
733                 cmd->dh_priv = ucmd;
734         } else {
735                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
736                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
737         }
738
739         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
740
741         if (ucmd->state != UCMD_STATE_NEW)
742                 goto alloc;
743
744         switch (dev->parse_type) {
745         case SCST_USER_PARSE_STANDARD:
746                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
747                 rc = dev->generic_parse(cmd, dev_user_get_block);
748                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_NOT_FOUND))
749                         goto out_invalid;
750                 break;
751
752         case SCST_USER_PARSE_EXCEPTION:
753                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
754                 rc = dev->generic_parse(cmd, dev_user_get_block);
755                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_NOT_FOUND)))
756                         break;
757                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
758                         TRACE_MEM("Restarting PARSE to thread context "
759                                 "(ucmd %p)", ucmd);
760                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
761                         goto out;
762                 }
763                 /* else go through */
764
765         case SCST_USER_PARSE_CALL:
766                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
767                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
768                 ucmd->user_cmd_payload_len =
769                         offsetof(struct scst_user_get_cmd, parse_cmd) +
770                         sizeof(ucmd->user_cmd.parse_cmd);
771                 ucmd->user_cmd.cmd_h = ucmd->h;
772                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
773                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
774                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb, cmd->cdb_len);
775                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
776                 ucmd->user_cmd.parse_cmd.ext_cdb_len = cmd->ext_cdb_len;
777                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
778                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
779                 ucmd->user_cmd.parse_cmd.in_bufflen = cmd->in_bufflen;
780                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
781                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
782                 ucmd->user_cmd.parse_cmd.expected_values_set =
783                                         cmd->expected_values_set;
784                 ucmd->user_cmd.parse_cmd.expected_data_direction =
785                                         cmd->expected_data_direction;
786                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
787                                         cmd->expected_transfer_len;
788                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
789                 ucmd->state = UCMD_STATE_PARSING;
790                 dev_user_add_to_ready(ucmd);
791                 res = SCST_CMD_STATE_STOP;
792                 goto out;
793
794         default:
795                 sBUG();
796                 goto out;
797         }
798
799 alloc:
800         if (cmd->bufflen == 0) {
801                 /*
802                  * According to SPC bufflen 0 for data transfer commands isn't
803                  * an error, so we need to fix the transfer direction.
804                  */
805                 cmd->data_direction = SCST_DATA_NONE;
806         }
807
808         if (cmd->data_direction != SCST_DATA_NONE)
809                 res = dev_user_alloc_space(ucmd);
810
811 out:
812         TRACE_EXIT_RES(res);
813         return res;
814
815 out_invalid:
816         PRINT_ERROR("PARSE failed (ucmd %p, rc %d)", ucmd, rc);
817         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
818
819 out_error:
820         res = scst_get_cmd_abnormal_done_state(cmd);
821         goto out;
822 }
823
824 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
825 {
826         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
827         unsigned long start = buf_ucmd->ubuff;
828         int i, bufflen = ucmd->cmd->bufflen;
829
830         TRACE_ENTRY();
831
832         if (start == 0)
833                 goto out;
834
835         /*
836          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
837          * faster, since it should be cache hot, while ucmd->buf_ucmd and
838          * buf_ucmd->data_pages are cache cold. But, from other side,
839          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
840          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
841          * data_pages array can lead to less cache misses. So, real numbers are
842          * needed. ToDo.
843          */
844
845         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
846                 struct page *page;
847                 page = buf_ucmd->data_pages[i];
848 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
849                 struct vm_area_struct *vma = find_vma(current->mm, start);
850                 if (vma != NULL)
851                         flush_anon_page(vma, page, start);
852 #endif
853                 flush_dcache_page(page);
854                 start += PAGE_SIZE;
855                 bufflen -= PAGE_SIZE;
856         }
857
858 out:
859         TRACE_EXIT();
860         return;
861 }
862
863 static int dev_user_exec(struct scst_cmd *cmd)
864 {
865         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
866         int res = SCST_EXEC_COMPLETED;
867
868         TRACE_ENTRY();
869
870 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
871        * context to complete the necessary actions, but all we are going to
872        * do in this function is, in fact, atomic, so let's skip this check.
873        */
874         if (scst_cmd_atomic(cmd)) {
875                 TRACE_DBG("%s", "User exec() can not be called in atomic "
876                         "context, rescheduling to the thread");
877                 res = SCST_EXEC_NEED_THREAD;
878                 goto out;
879         }
880 #endif
881
882         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
883                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
884                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
885
886         if (cmd->data_direction == SCST_DATA_WRITE)
887                 dev_user_flush_dcache(ucmd);
888
889         BUILD_BUG_ON(sizeof(ucmd->user_cmd.exec_cmd.cdb) != sizeof(cmd->cdb));
890
891         ucmd->user_cmd_payload_len =
892                 offsetof(struct scst_user_get_cmd, exec_cmd) +
893                 sizeof(ucmd->user_cmd.exec_cmd);
894         ucmd->user_cmd.cmd_h = ucmd->h;
895         ucmd->user_cmd.subcode = SCST_USER_EXEC;
896         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
897         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb, cmd->cdb_len);
898         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
899         ucmd->user_cmd.exec_cmd.ext_cdb_len = cmd->ext_cdb_len;
900         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
901         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
902         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
903         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
904                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
905                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
906         }
907         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
908         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
909         ucmd->user_cmd.exec_cmd.partial = 0;
910         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
911         ucmd->user_cmd.exec_cmd.p_in_buf = ucmd->ubuff +
912                                                 (cmd->sg_cnt << PAGE_SHIFT);
913         ucmd->user_cmd.exec_cmd.in_bufflen = cmd->in_bufflen;
914         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
915
916         ucmd->state = UCMD_STATE_EXECING;
917
918         dev_user_add_to_ready(ucmd);
919
920         TRACE_EXIT_RES(res);
921         return res;
922 }
923
924 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
925 {
926         if (ucmd->sgv != NULL) {
927                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
928                 ucmd->sgv = NULL;
929         } else if (ucmd->data_pages != NULL) {
930                 /* We mapped pages, but for some reason didn't allocate them */
931                 ucmd_get(ucmd);
932                 __dev_user_free_sg_entries(ucmd);
933         }
934         return;
935 }
936
937 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
938 {
939         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
940
941         TRACE_ENTRY();
942
943         if (unlikely(ucmd == NULL))
944                 goto out;
945
946         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
947                 ucmd->buff_cached, ucmd->ubuff);
948
949         ucmd->cmd = NULL;
950         if (cmd->data_direction == SCST_DATA_WRITE && ucmd->buf_ucmd != NULL)
951                 ucmd->buf_ucmd->buf_dirty = 1;
952
953         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
954                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
955                 /* The state assignment must be before freeing sgv! */
956                 goto out_reply;
957         }
958
959         if (unlikely(!ucmd->seen_by_user)) {
960                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
961                 goto out_reply;
962         }
963
964         ucmd->user_cmd_payload_len =
965                 offsetof(struct scst_user_get_cmd, on_free_cmd) +
966                 sizeof(ucmd->user_cmd.on_free_cmd);
967         ucmd->user_cmd.cmd_h = ucmd->h;
968         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
969         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
970         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
971         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
972         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
973         ucmd->user_cmd.on_free_cmd.status = cmd->status;
974         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
975
976         ucmd->state = UCMD_STATE_ON_FREEING;
977
978         dev_user_add_to_ready(ucmd);
979
980 out:
981         TRACE_EXIT();
982         return;
983
984 out_reply:
985         dev_user_process_reply_on_free(ucmd);
986         goto out;
987 }
988
989 static void dev_user_set_block(struct scst_cmd *cmd, int block)
990 {
991         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
992         /*
993          * No need for locks here, since *_detach() can not be
994          * called, when there are existing commands.
995          */
996         TRACE_DBG("dev %p, new block %d", dev, block);
997         if (block != 0)
998                 dev->block = block;
999         else
1000                 dev->block = dev->def_block;
1001         return;
1002 }
1003
1004 static int dev_user_disk_done(struct scst_cmd *cmd)
1005 {
1006         int res = SCST_CMD_STATE_DEFAULT;
1007
1008         TRACE_ENTRY();
1009
1010         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
1011
1012         TRACE_EXIT_RES(res);
1013         return res;
1014 }
1015
1016 static int dev_user_tape_done(struct scst_cmd *cmd)
1017 {
1018         int res = SCST_CMD_STATE_DEFAULT;
1019
1020         TRACE_ENTRY();
1021
1022         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
1023
1024         TRACE_EXIT_RES(res);
1025         return res;
1026 }
1027
1028 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
1029 {
1030         struct scst_user_dev *dev = ucmd->dev;
1031         unsigned long flags;
1032         int do_wake = in_interrupt();
1033
1034         TRACE_ENTRY();
1035
1036         if (ucmd->cmd)
1037                 do_wake |= ucmd->cmd->preprocessing_only;
1038
1039         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1040
1041         ucmd->this_state_unjammed = 0;
1042
1043         if ((ucmd->state == UCMD_STATE_PARSING) ||
1044             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1045                 /*
1046                  * If we don't put such commands in the queue head, then under
1047                  * high load we might delay threads, waiting for memory
1048                  * allocations, for too long and start loosing NOPs, which
1049                  * would lead to consider us by remote initiators as
1050                  * unresponsive and stuck => broken connections, etc. If none
1051                  * of our commands completed in NOP timeout to allow the head
1052                  * commands to go, then we are really overloaded and/or stuck.
1053                  */
1054                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1055                         "cmd list", ucmd, ucmd->state);
1056                 list_add(&ucmd->ready_cmd_list_entry,
1057                         &dev->ready_cmd_list);
1058         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1059                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1060                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1061                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1062                         "ready cmd list", ucmd, ucmd->state);
1063                 list_add(&ucmd->ready_cmd_list_entry,
1064                         &dev->ready_cmd_list);
1065                 do_wake = 1;
1066         } else {
1067                 if ((ucmd->cmd != NULL) &&
1068                     unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1069                         TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list",
1070                                 ucmd);
1071                         list_add(&ucmd->ready_cmd_list_entry,
1072                                 &dev->ready_cmd_list);
1073                 } else {
1074                         TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1075                         list_add_tail(&ucmd->ready_cmd_list_entry,
1076                                       &dev->ready_cmd_list);
1077                 }
1078                 do_wake |= ((ucmd->state == UCMD_STATE_ON_CACHE_FREEING) ||
1079                             (ucmd->state == UCMD_STATE_ON_FREEING));
1080         }
1081
1082         if (do_wake) {
1083                 TRACE_DBG("Waking up dev %p", dev);
1084                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1085         }
1086
1087         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1088
1089         TRACE_EXIT();
1090         return;
1091 }
1092
1093 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1094         int num_pg)
1095 {
1096         int res = 0, rc;
1097         int i;
1098         struct task_struct *tsk = current;
1099
1100         TRACE_ENTRY();
1101
1102         if (unlikely(ubuff == 0))
1103                 goto out_nomem;
1104
1105         sBUG_ON(ucmd->data_pages != NULL);
1106
1107         ucmd->num_data_pages = num_pg;
1108
1109         ucmd->data_pages =
1110                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1111                           GFP_KERNEL);
1112         if (ucmd->data_pages == NULL) {
1113                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1114                         "(num_data_pages=%d)", ucmd->num_data_pages);
1115                 res = -ENOMEM;
1116                 goto out_nomem;
1117         }
1118
1119         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1120                 " first_page_offset %d, len %d)", ucmd, ubuff,
1121                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1122                 ucmd->cmd->bufflen);
1123
1124         down_read(&tsk->mm->mmap_sem);
1125         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1126                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1127         up_read(&tsk->mm->mmap_sem);
1128
1129         /* get_user_pages() flushes dcache */
1130
1131         if (rc < ucmd->num_data_pages)
1132                 goto out_unmap;
1133
1134         ucmd->ubuff = ubuff;
1135         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1136
1137 out:
1138         TRACE_EXIT_RES(res);
1139         return res;
1140
1141 out_nomem:
1142         scst_set_busy(ucmd->cmd);
1143         /* go through */
1144
1145 out_err:
1146         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1147         goto out;
1148
1149 out_unmap:
1150         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1151                 ucmd->num_data_pages, rc);
1152         if (rc > 0) {
1153                 for (i = 0; i < rc; i++)
1154                         page_cache_release(ucmd->data_pages[i]);
1155         }
1156         kfree(ucmd->data_pages);
1157         ucmd->data_pages = NULL;
1158         res = -EFAULT;
1159         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1160         goto out_err;
1161 }
1162
1163 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1164         struct scst_user_reply_cmd *reply)
1165 {
1166         int res = 0;
1167         struct scst_cmd *cmd = ucmd->cmd;
1168
1169         TRACE_ENTRY();
1170
1171         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1172
1173         if (likely(reply->alloc_reply.pbuf != 0)) {
1174                 int pages;
1175                 if (ucmd->buff_cached) {
1176                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1177                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1178                                         "page aligned",
1179                                         reply->alloc_reply.pbuf);
1180                                 goto out_hwerr;
1181                         }
1182                         pages = cmd->sg_cnt;
1183                 } else
1184                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1185                                             cmd->bufflen);
1186                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1187         } else {
1188                 scst_set_busy(ucmd->cmd);
1189                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1190         }
1191
1192 out_process:
1193         scst_process_active_cmd(cmd, false);
1194
1195         TRACE_EXIT_RES(res);
1196         return res;
1197
1198 out_hwerr:
1199         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1200         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1201         res = -EINVAL;
1202         goto out_process;
1203 }
1204
1205 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1206         struct scst_user_reply_cmd *reply)
1207 {
1208         int res = 0;
1209         struct scst_user_scsi_cmd_reply_parse *preply =
1210                 &reply->parse_reply;
1211         struct scst_cmd *cmd = ucmd->cmd;
1212
1213         TRACE_ENTRY();
1214
1215         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1216                 goto out_inval;
1217
1218         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1219                      (preply->data_direction != SCST_DATA_READ) &&
1220                      (preply->data_direction != SCST_DATA_NONE)))
1221                 goto out_inval;
1222
1223         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1224                      (preply->bufflen == 0)))
1225                 goto out_inval;
1226
1227         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1228                 goto out_inval;
1229
1230         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1231                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1232                 preply->data_direction, preply->bufflen, preply->data_len,
1233                 reply->alloc_reply.pbuf);
1234
1235         cmd->queue_type = preply->queue_type;
1236         cmd->data_direction = preply->data_direction;
1237         cmd->bufflen = preply->bufflen;
1238         cmd->data_len = preply->data_len;
1239         if (preply->write_medium)
1240                 cmd->op_flags |= SCST_WRITE_MEDIUM;
1241
1242 out_process:
1243         scst_process_active_cmd(cmd, false);
1244
1245         TRACE_EXIT_RES(res);
1246         return res;
1247
1248 out_inval:
1249         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1250                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1251         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1252         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1253         scst_set_cmd_abnormal_done_state(cmd);
1254         res = -EINVAL;
1255         goto out_process;
1256 }
1257
1258 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1259 {
1260         int res = 0;
1261
1262         TRACE_ENTRY();
1263
1264         TRACE_DBG("ON FREE ucmd %p", ucmd);
1265
1266         dev_user_free_sgv(ucmd);
1267         ucmd_put(ucmd);
1268
1269         TRACE_EXIT_RES(res);
1270         return res;
1271 }
1272
1273 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1274 {
1275         int res = 0;
1276
1277         TRACE_ENTRY();
1278
1279         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1280
1281         ucmd_put(ucmd);
1282
1283         TRACE_EXIT_RES(res);
1284         return res;
1285 }
1286
1287 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1288         struct scst_user_reply_cmd *reply)
1289 {
1290         int res = 0;
1291         struct scst_user_scsi_cmd_reply_exec *ereply =
1292                 &reply->exec_reply;
1293         struct scst_cmd *cmd = ucmd->cmd;
1294
1295         TRACE_ENTRY();
1296
1297         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1298                 if (ucmd->background_exec) {
1299                         TRACE_DBG("Background ucmd %p finished", ucmd);
1300                         ucmd_put(ucmd);
1301                         goto out;
1302                 }
1303                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1304                         goto out_inval;
1305                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1306                              (ereply->resp_data_len != 0)))
1307                         goto out_inval;
1308         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1309                 if (unlikely(ucmd->background_exec))
1310                         goto out_inval;
1311                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1312                              (cmd->resp_data_len != 0)))
1313                         goto out_inval;
1314                 /*
1315                  * background_exec assignment must be after ucmd get.
1316                  * Otherwise, due to reorder, in dev_user_process_reply()
1317                  * it is possible that ucmd is destroyed before it "got" here.
1318                  */
1319                 ucmd_get_ordered(ucmd);
1320                 ucmd->background_exec = 1;
1321                 TRACE_DBG("Background ucmd %p", ucmd);
1322                 goto out_compl;
1323         } else
1324                 goto out_inval;
1325
1326         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1327                 ereply->status, ereply->resp_data_len);
1328
1329          if (ereply->resp_data_len != 0) {
1330                 if (ucmd->ubuff == 0) {
1331                         int pages, rc;
1332                         if (unlikely(ereply->pbuf == 0))
1333                                 goto out_busy;
1334                         if (ucmd->buff_cached) {
1335                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1336                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1337                                                 "page aligned", ereply->pbuf);
1338                                         goto out_hwerr;
1339                                 }
1340                                 pages = cmd->sg_cnt;
1341                         } else
1342                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1343                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1344                         if ((rc != 0) || (ucmd->ubuff == 0))
1345                                 goto out_compl;
1346
1347                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1348                         if (unlikely(rc != 0))
1349                                 goto out_busy;
1350                 } else
1351                         dev_user_flush_dcache(ucmd);
1352                 cmd->may_need_dma_sync = 1;
1353                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1354         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1355                 if (ucmd->ubuff == 0)
1356                         cmd->resp_data_len = ereply->resp_data_len;
1357                 else
1358                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1359         }
1360
1361         cmd->status = ereply->status;
1362         if (ereply->sense_len != 0) {
1363                 res = scst_alloc_sense(cmd, 0);
1364                 if (res != 0)
1365                         goto out_compl;
1366                 res = copy_from_user(cmd->sense,
1367                         (void __user *)(unsigned long)ereply->psense_buffer,
1368                         min((int)cmd->sense_bufflen, (int)ereply->sense_len));
1369                 if (res < 0) {
1370                         PRINT_ERROR("%s", "Unable to get sense data");
1371                         goto out_hwerr_res_set;
1372                 }
1373         }
1374
1375 out_compl:
1376         cmd->completed = 1;
1377         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1378         /* !! At this point cmd can be already freed !! */
1379
1380 out:
1381         TRACE_EXIT_RES(res);
1382         return res;
1383
1384 out_inval:
1385         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1386                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1387         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1388
1389 out_hwerr:
1390         res = -EINVAL;
1391
1392 out_hwerr_res_set:
1393         if (ucmd->background_exec) {
1394                 ucmd_put(ucmd);
1395                 goto out;
1396         } else {
1397                 scst_set_cmd_error(cmd,
1398                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1399                 goto out_compl;
1400         }
1401
1402 out_busy:
1403         scst_set_busy(cmd);
1404         goto out_compl;
1405 }
1406
1407 static int dev_user_process_reply(struct scst_user_dev *dev,
1408         struct scst_user_reply_cmd *reply)
1409 {
1410         int res = 0;
1411         struct scst_user_cmd *ucmd;
1412         int state;
1413
1414         TRACE_ENTRY();
1415
1416         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1417
1418         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1419         if (unlikely(ucmd == NULL)) {
1420                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1421                 res = -ESRCH;
1422                 goto out_unlock;
1423         }
1424
1425         if (unlikely(ucmd_get_check(ucmd))) {
1426                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1427                 res = -ESRCH;
1428                 goto out_unlock;
1429         }
1430
1431         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1432         smp_mb();
1433         if (ucmd->background_exec) {
1434                 state = UCMD_STATE_EXECING;
1435                 goto unlock_process;
1436         }
1437
1438         if (unlikely(ucmd->this_state_unjammed)) {
1439                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1440                         ucmd);
1441                 goto out_unlock_put;
1442         }
1443
1444         if (unlikely(!ucmd->sent_to_user)) {
1445                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1446                         "state %x", ucmd, ucmd->state);
1447                 res = -EINVAL;
1448                 goto out_unlock_put;
1449         }
1450
1451         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1452                 goto out_wrong_state;
1453
1454         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1455                 goto out_wrong_state;
1456
1457         state = ucmd->state;
1458         ucmd->sent_to_user = 0;
1459
1460 unlock_process:
1461         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1462
1463         switch (state) {
1464         case UCMD_STATE_PARSING:
1465                 res = dev_user_process_reply_parse(ucmd, reply);
1466                 break;
1467
1468         case UCMD_STATE_BUF_ALLOCING:
1469                 res = dev_user_process_reply_alloc(ucmd, reply);
1470                 break;
1471
1472         case UCMD_STATE_EXECING:
1473                 res = dev_user_process_reply_exec(ucmd, reply);
1474                 break;
1475
1476         case UCMD_STATE_ON_FREEING:
1477                 res = dev_user_process_reply_on_free(ucmd);
1478                 break;
1479
1480         case UCMD_STATE_ON_CACHE_FREEING:
1481                 res = dev_user_process_reply_on_cache_free(ucmd);
1482                 break;
1483
1484         case UCMD_STATE_TM_EXECING:
1485                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1486                 break;
1487
1488         case UCMD_STATE_ATTACH_SESS:
1489         case UCMD_STATE_DETACH_SESS:
1490                 res = dev_user_process_reply_sess(ucmd, reply->result);
1491                 break;
1492
1493         default:
1494                 sBUG();
1495                 break;
1496         }
1497
1498 out_put:
1499         ucmd_put(ucmd);
1500
1501 out:
1502         TRACE_EXIT_RES(res);
1503         return res;
1504
1505 out_wrong_state:
1506         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1507                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1508                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1509                 reply->subcode, ucmd->user_cmd.subcode);
1510         res = -EINVAL;
1511         dev_user_unjam_cmd(ucmd, 0, NULL);
1512
1513 out_unlock_put:
1514         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1515         goto out_put;
1516
1517 out_unlock:
1518         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1519         goto out;
1520 }
1521
1522 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1523 {
1524         int res = 0;
1525         struct scst_user_dev *dev;
1526         struct scst_user_reply_cmd reply;
1527
1528         TRACE_ENTRY();
1529
1530         mutex_lock(&dev_priv_mutex);
1531         dev = (struct scst_user_dev *)file->private_data;
1532         res = dev_user_check_reg(dev);
1533         if (unlikely(res != 0)) {
1534                 mutex_unlock(&dev_priv_mutex);
1535                 goto out;
1536         }
1537         down_read(&dev->dev_rwsem);
1538         mutex_unlock(&dev_priv_mutex);
1539
1540         res = copy_from_user(&reply, arg, sizeof(reply));
1541         if (unlikely(res < 0))
1542                 goto out_up;
1543
1544         TRACE_DBG("Reply for dev %s", dev->name);
1545
1546         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1547
1548         res = dev_user_process_reply(dev, &reply);
1549         if (unlikely(res < 0))
1550                 goto out_up;
1551
1552 out_up:
1553         up_read(&dev->dev_rwsem);
1554
1555 out:
1556         TRACE_EXIT_RES(res);
1557         return res;
1558 }
1559
1560 static int dev_user_get_ext_cdb(struct file *file, void __user *arg)
1561 {
1562         int res = 0;
1563         struct scst_user_dev *dev;
1564         struct scst_user_cmd *ucmd;
1565         struct scst_cmd *cmd = NULL;
1566         struct scst_user_get_ext_cdb get;
1567
1568         TRACE_ENTRY();
1569
1570         mutex_lock(&dev_priv_mutex);
1571         dev = (struct scst_user_dev *)file->private_data;
1572         res = dev_user_check_reg(dev);
1573         if (unlikely(res != 0)) {
1574                 mutex_unlock(&dev_priv_mutex);
1575                 goto out;
1576         }
1577         down_read(&dev->dev_rwsem);
1578         mutex_unlock(&dev_priv_mutex);
1579
1580         res = copy_from_user(&get, arg, sizeof(get));
1581         if (unlikely(res < 0))
1582                 goto out_up;
1583
1584         TRACE_MGMT_DBG("Get ext cdb for dev %s", dev->name);
1585
1586         TRACE_BUFFER("Get ext cdb", &get, sizeof(get));
1587
1588         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1589
1590         ucmd = __ucmd_find_hash(dev, get.cmd_h);
1591         if (unlikely(ucmd == NULL)) {
1592                 TRACE_MGMT_DBG("cmd_h %d not found", get.cmd_h);
1593                 res = -ESRCH;
1594                 goto out_unlock;
1595         }
1596
1597         if (unlikely(ucmd_get_check(ucmd))) {
1598                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", get.cmd_h);
1599                 res = -ESRCH;
1600                 goto out_unlock;
1601         }
1602
1603         if ((ucmd->cmd != NULL) && (ucmd->state <= UCMD_STATE_EXECING) &&
1604             (ucmd->sent_to_user || ucmd->background_exec)) {
1605                 cmd = ucmd->cmd;
1606                 scst_cmd_get(cmd);
1607         } else {
1608                 TRACE_MGMT_DBG("Invalid ucmd state %d for cmd_h %d",
1609                         ucmd->state, get.cmd_h);
1610                 res = -EINVAL;
1611                 goto out_unlock;
1612         }
1613
1614         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1615
1616         if (cmd == NULL)
1617                 goto out_put;
1618
1619         if (cmd->ext_cdb == NULL)
1620                 goto out_cmd_put;
1621
1622         TRACE_BUFFER("EXT CDB", cmd->ext_cdb, cmd->ext_cdb_len);
1623         res = copy_to_user((void __user *)(unsigned long)get.ext_cdb_buffer,
1624                 cmd->ext_cdb, cmd->ext_cdb_len);
1625
1626 out_cmd_put:
1627         scst_cmd_put(cmd);
1628
1629 out_put:
1630         ucmd_put(ucmd);
1631
1632 out_up:
1633         up_read(&dev->dev_rwsem);
1634
1635 out:
1636         TRACE_EXIT_RES(res);
1637         return res;
1638
1639 out_unlock:
1640         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1641         goto out_up;
1642 }
1643
1644 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1645         __releases(&dev->cmd_lists.cmd_list_lock)
1646         __acquires(&dev->cmd_lists.cmd_list_lock)
1647 {
1648         int res = 0;
1649
1650         TRACE_ENTRY();
1651
1652         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1653                 struct scst_cmd *cmd = list_entry(
1654                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1655                         cmd_list_entry);
1656                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1657                 list_del(&cmd->cmd_list_entry);
1658                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1659                 scst_process_active_cmd(cmd, false);
1660                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1661                 res++;
1662         }
1663
1664         TRACE_EXIT_RES(res);
1665         return res;
1666 }
1667
1668 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1669 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1670         __releases(&dev->cmd_lists.cmd_list_lock)
1671         __acquires(&dev->cmd_lists.cmd_list_lock)
1672 {
1673         struct scst_user_cmd *u;
1674
1675 again:
1676         u = NULL;
1677         if (!list_empty(cmd_list)) {
1678                 u = list_entry(cmd_list->next, typeof(*u),
1679                                ready_cmd_list_entry);
1680
1681                 TRACE_DBG("Found ready ucmd %p", u);
1682                 list_del(&u->ready_cmd_list_entry);
1683
1684                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1685
1686                 if (u->cmd != NULL) {
1687                         if (u->state == UCMD_STATE_EXECING) {
1688                                 struct scst_user_dev *dev = u->dev;
1689                                 int rc;
1690
1691                                 EXTRACHECKS_BUG_ON(u->jammed);
1692
1693                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1694
1695                                 rc = scst_check_local_events(u->cmd);
1696                                 if (unlikely(rc != 0)) {
1697                                         u->cmd->scst_cmd_done(u->cmd,
1698                                                 SCST_CMD_STATE_DEFAULT,
1699                                                 SCST_CONTEXT_DIRECT);
1700                                         /*
1701                                          * !! At this point cmd & u can be !!
1702                                          * !! already freed                !!
1703                                          */
1704                                         spin_lock_irq(
1705                                                 &dev->cmd_lists.cmd_list_lock);
1706                                         goto again;
1707                                 }
1708
1709                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1710                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1711                                         &u->cmd->cmd_flags))) {
1712                                 switch (u->state) {
1713                                 case UCMD_STATE_PARSING:
1714                                 case UCMD_STATE_BUF_ALLOCING:
1715                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1716                                         dev_user_unjam_cmd(u, 0, NULL);
1717                                         goto again;
1718                                 case UCMD_STATE_EXECING:
1719                                         EXTRACHECKS_BUG_ON(1);
1720                                 }
1721                         }
1722                 }
1723                 u->sent_to_user = 1;
1724                 u->seen_by_user = 1;
1725         }
1726         return u;
1727 }
1728
1729 static inline int test_cmd_lists(struct scst_user_dev *dev)
1730 {
1731         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1732                   !list_empty(&dev->ready_cmd_list) ||
1733                   !dev->blocking || dev->cleanup_done ||
1734                   signal_pending(current);
1735         return res;
1736 }
1737
1738 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1739 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1740         struct scst_user_cmd **ucmd)
1741 {
1742         int res = 0;
1743         wait_queue_t wait;
1744
1745         TRACE_ENTRY();
1746
1747         init_waitqueue_entry(&wait, current);
1748
1749         while (1) {
1750                 if (!test_cmd_lists(dev)) {
1751                         add_wait_queue_exclusive_head(
1752                                 &dev->cmd_lists.cmd_list_waitQ,
1753                                 &wait);
1754                         for (;;) {
1755                                 set_current_state(TASK_INTERRUPTIBLE);
1756                                 if (test_cmd_lists(dev))
1757                                         break;
1758                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1759                                 schedule();
1760                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1761                         }
1762                         set_current_state(TASK_RUNNING);
1763                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1764                                 &wait);
1765                 }
1766
1767                 dev_user_process_scst_commands(dev);
1768
1769                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1770                 if (*ucmd != NULL)
1771                         break;
1772
1773                 if (!dev->blocking || dev->cleanup_done) {
1774                         res = -EAGAIN;
1775                         TRACE_DBG("No ready commands, returning %d", res);
1776                         break;
1777                 }
1778
1779                 if (signal_pending(current)) {
1780                         res = -EINTR;
1781                         TRACE_DBG("Signal pending, returning %d", res);
1782                         break;
1783                 }
1784         }
1785
1786         TRACE_EXIT_RES(res);
1787         return res;
1788 }
1789
1790 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1791 {
1792         int res = 0;
1793         struct scst_user_dev *dev;
1794         struct scst_user_get_cmd *cmd;
1795         struct scst_user_reply_cmd *reply;
1796         struct scst_user_cmd *ucmd;
1797         uint64_t ureply;
1798
1799         TRACE_ENTRY();
1800
1801         mutex_lock(&dev_priv_mutex);
1802         dev = (struct scst_user_dev *)file->private_data;
1803         res = dev_user_check_reg(dev);
1804         if (unlikely(res != 0)) {
1805                 mutex_unlock(&dev_priv_mutex);
1806                 goto out;
1807         }
1808         down_read(&dev->dev_rwsem);
1809         mutex_unlock(&dev_priv_mutex);
1810
1811         /* get_user() can't be used with 64-bit values on x86_32 */
1812         res = copy_from_user(&ureply, (uint64_t __user *)
1813                 &((struct scst_user_get_cmd __user *)arg)->preply,
1814                 sizeof(ureply));
1815         if (unlikely(res < 0))
1816                 goto out_up;
1817
1818         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1819                 dev->name);
1820
1821         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1822         if (unlikely(cmd == NULL)) {
1823                 res = -ENOMEM;
1824                 goto out_up;
1825         }
1826
1827         if (ureply != 0) {
1828                 unsigned long u = (unsigned long)ureply;
1829                 reply = (struct scst_user_reply_cmd *)cmd;
1830                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1831                 if (unlikely(res < 0))
1832                         goto out_free;
1833
1834                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1835
1836                 res = dev_user_process_reply(dev, reply);
1837                 if (unlikely(res < 0))
1838                         goto out_free;
1839         }
1840
1841         kmem_cache_free(user_get_cmd_cachep, cmd);
1842
1843         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1844 again:
1845         res = dev_user_get_next_cmd(dev, &ucmd);
1846         if (res == 0) {
1847                 int len;
1848                 /*
1849                  * A misbehaving user space handler can make ucmd to get dead
1850                  * immediately after we released the lock, which can lead to
1851                  * copy of dead data to the user space, which can lead to a
1852                  * leak of sensitive information.
1853                  */
1854                 if (unlikely(ucmd_get_check(ucmd))) {
1855                         /* Oops, this ucmd is already being destroyed. Retry. */
1856                         goto again;
1857                 }
1858                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1859
1860                 EXTRACHECKS_BUG_ON(ucmd->user_cmd_payload_len == 0);
1861
1862                 len = ucmd->user_cmd_payload_len;
1863                 TRACE_DBG("ucmd %p (user_cmd %p), payload_len %d (len %d)",
1864                         ucmd, &ucmd->user_cmd, ucmd->user_cmd_payload_len, len);
1865                 TRACE_BUFFER("UCMD", &ucmd->user_cmd, len);
1866                 res = copy_to_user(arg, &ucmd->user_cmd, len);
1867                 if (unlikely(res != 0)) {
1868                         /* Requeue ucmd back */
1869                         TRACE_DBG("Copy to user failed (%d), requeuing ucmd %p "
1870                                 "back to head of ready cmd list", res, ucmd);
1871                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1872                         list_add(&ucmd->ready_cmd_list_entry,
1873                                 &dev->ready_cmd_list);
1874                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1875                 }
1876 #ifdef CONFIG_SCST_EXTRACHECKS
1877                 else
1878                         ucmd->user_cmd_payload_len = 0;
1879 #endif
1880                 ucmd_put(ucmd);
1881         } else
1882                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1883
1884 out_up:
1885         up_read(&dev->dev_rwsem);
1886
1887 out:
1888         TRACE_EXIT_RES(res);
1889         return res;
1890
1891 out_free:
1892         kmem_cache_free(user_get_cmd_cachep, cmd);
1893         goto out_up;
1894 }
1895
1896 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1897         unsigned long arg)
1898 {
1899         long res;
1900
1901         TRACE_ENTRY();
1902
1903         switch (cmd) {
1904         case SCST_USER_REPLY_AND_GET_CMD:
1905                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1906                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1907                 break;
1908
1909         case SCST_USER_REPLY_CMD:
1910                 TRACE_DBG("%s", "REPLY_CMD");
1911                 res = dev_user_reply_cmd(file, (void __user *)arg);
1912                 break;
1913
1914         case SCST_USER_GET_EXTENDED_CDB:
1915                 TRACE_DBG("%s", "GET_EXTENDED_CDB");
1916                 res = dev_user_get_ext_cdb(file, (void __user *)arg);
1917                 break;
1918
1919         case SCST_USER_REGISTER_DEVICE:
1920         {
1921                 struct scst_user_dev_desc *dev_desc;
1922                 TRACE_DBG("%s", "REGISTER_DEVICE");
1923                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1924                 if (dev_desc == NULL) {
1925                         res = -ENOMEM;
1926                         goto out;
1927                 }
1928                 res = copy_from_user(dev_desc, (void __user *)arg,
1929                                      sizeof(*dev_desc));
1930                 if (res < 0) {
1931                         kfree(dev_desc);
1932                         goto out;
1933                 }
1934                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1935                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1936                 dev_desc->sgv_name[sizeof(dev_desc->sgv_name)-1] = '\0';
1937                 res = dev_user_register_dev(file, dev_desc);
1938                 kfree(dev_desc);
1939                 break;
1940         }
1941
1942         case SCST_USER_UNREGISTER_DEVICE:
1943                 TRACE_DBG("%s", "UNREGISTER_DEVICE");
1944                 res = dev_user_unregister_dev(file);
1945                 break;
1946
1947         case SCST_USER_FLUSH_CACHE:
1948                 TRACE_DBG("%s", "FLUSH_CACHE");
1949                 res = dev_user_flush_cache(file);
1950                 break;
1951
1952         case SCST_USER_SET_OPTIONS:
1953         {
1954                 struct scst_user_opt opt;
1955                 TRACE_DBG("%s", "SET_OPTIONS");
1956                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1957                 if (res < 0)
1958                         goto out;
1959                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1960                 res = dev_user_set_opt(file, &opt);
1961                 break;
1962         }
1963
1964         case SCST_USER_GET_OPTIONS:
1965                 TRACE_DBG("%s", "GET_OPTIONS");
1966                 res = dev_user_get_opt(file, (void __user *)arg);
1967                 break;
1968
1969         case SCST_USER_DEVICE_CAPACITY_CHANGED:
1970                 TRACE_DBG("%s", "CAPACITY_CHANGED");
1971                 res = dev_user_capacity_changed(file);
1972                 break;
1973
1974         default:
1975                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1976                 res = -EINVAL;
1977                 goto out;
1978         }
1979
1980 out:
1981         TRACE_EXIT_RES(res);
1982         return res;
1983 }
1984
1985 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1986 {
1987         int res = 0;
1988         struct scst_user_dev *dev;
1989
1990         TRACE_ENTRY();
1991
1992         mutex_lock(&dev_priv_mutex);
1993         dev = (struct scst_user_dev *)file->private_data;
1994         res = dev_user_check_reg(dev);
1995         if (unlikely(res != 0)) {
1996                 mutex_unlock(&dev_priv_mutex);
1997                 goto out;
1998         }
1999         down_read(&dev->dev_rwsem);
2000         mutex_unlock(&dev_priv_mutex);
2001
2002         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2003
2004         if (!list_empty(&dev->ready_cmd_list) ||
2005             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2006                 res |= POLLIN | POLLRDNORM;
2007                 goto out_unlock;
2008         }
2009
2010         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2011
2012         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
2013         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
2014         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
2015
2016         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2017
2018         if (!list_empty(&dev->ready_cmd_list) ||
2019             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2020                 res |= POLLIN | POLLRDNORM;
2021                 goto out_unlock;
2022         }
2023
2024 out_unlock:
2025         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2026
2027         up_read(&dev->dev_rwsem);
2028
2029 out:
2030         TRACE_EXIT_HRES(res);
2031         return res;
2032 }
2033
2034 /*
2035  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
2036  */
2037 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
2038         unsigned long *flags)
2039         __releases(&dev->cmd_lists.cmd_list_lock)
2040         __acquires(&dev->cmd_lists.cmd_list_lock)
2041 {
2042         int state = ucmd->state;
2043         struct scst_user_dev *dev = ucmd->dev;
2044
2045         TRACE_ENTRY();
2046
2047         if (ucmd->this_state_unjammed)
2048                 goto out;
2049
2050         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
2051                 state);
2052
2053         ucmd->jammed = 1;
2054         ucmd->this_state_unjammed = 1;
2055         ucmd->sent_to_user = 0;
2056
2057         switch (state) {
2058         case UCMD_STATE_PARSING:
2059         case UCMD_STATE_BUF_ALLOCING:
2060                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2061                         ucmd->aborted = 1;
2062                 else {
2063                         if (busy)
2064                                 scst_set_busy(ucmd->cmd);
2065                         else
2066                                 scst_set_cmd_error(ucmd->cmd,
2067                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2068                 }
2069                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
2070
2071                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
2072                 list_add(&ucmd->cmd->cmd_list_entry,
2073                         &ucmd->cmd->cmd_lists->active_cmd_list);
2074                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
2075                 break;
2076
2077         case UCMD_STATE_EXECING:
2078                 if (flags != NULL)
2079                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2080                                                *flags);
2081                 else
2082                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2083
2084                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
2085
2086                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2087                         ucmd->aborted = 1;
2088                 else {
2089                         if (busy)
2090                                 scst_set_busy(ucmd->cmd);
2091                         else
2092                                 scst_set_cmd_error(ucmd->cmd,
2093                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2094                 }
2095
2096                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
2097                         SCST_CONTEXT_DIRECT);
2098                 /* !! At this point cmd and ucmd can be already freed !! */
2099
2100                 if (flags != NULL)
2101                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2102                                           *flags);
2103                 else
2104                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2105                 break;
2106
2107         case UCMD_STATE_ON_FREEING:
2108         case UCMD_STATE_ON_CACHE_FREEING:
2109         case UCMD_STATE_TM_EXECING:
2110         case UCMD_STATE_ATTACH_SESS:
2111         case UCMD_STATE_DETACH_SESS:
2112                 if (flags != NULL)
2113                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2114                                                *flags);
2115                 else
2116                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2117
2118                 switch (state) {
2119                 case UCMD_STATE_ON_FREEING:
2120                         dev_user_process_reply_on_free(ucmd);
2121                         break;
2122
2123                 case UCMD_STATE_ON_CACHE_FREEING:
2124                         dev_user_process_reply_on_cache_free(ucmd);
2125                         break;
2126
2127                 case UCMD_STATE_TM_EXECING:
2128                         dev_user_process_reply_tm_exec(ucmd,
2129                                                        SCST_MGMT_STATUS_FAILED);
2130                         break;
2131
2132                 case UCMD_STATE_ATTACH_SESS:
2133                 case UCMD_STATE_DETACH_SESS:
2134                         dev_user_process_reply_sess(ucmd, -EFAULT);
2135                         break;
2136                 }
2137
2138                 if (flags != NULL)
2139                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2140                                           *flags);
2141                 else
2142                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2143                 break;
2144
2145         default:
2146                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
2147                 sBUG();
2148                 break;
2149         }
2150
2151 out:
2152         TRACE_EXIT();
2153         return;
2154 }
2155
2156 static int dev_user_unjam_dev(struct scst_user_dev *dev)
2157         __releases(&dev->cmd_lists.cmd_list_lock)
2158         __acquires(&dev->cmd_lists.cmd_list_lock)
2159 {
2160         int i, res = 0;
2161         struct scst_user_cmd *ucmd;
2162
2163         TRACE_ENTRY();
2164
2165         TRACE_MGMT_DBG("Unjamming dev %p", dev);
2166
2167         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2168
2169 repeat:
2170         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2171                 struct list_head *head = &dev->ucmd_hash[i];
2172
2173                 list_for_each_entry(ucmd, head, hash_list_entry) {
2174                         res++;
2175
2176                         if (!ucmd->sent_to_user)
2177                                 continue;
2178
2179                         if (ucmd_get_check(ucmd))
2180                                 continue;
2181
2182                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2183                                 ucmd->state, ucmd->cmd);
2184
2185                         dev_user_unjam_cmd(ucmd, 0, NULL);
2186
2187                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2188                         ucmd_put(ucmd);
2189                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2190
2191                         goto repeat;
2192                 }
2193         }
2194
2195         if (dev_user_process_scst_commands(dev) != 0)
2196                 goto repeat;
2197
2198         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2199
2200         TRACE_EXIT_RES(res);
2201         return res;
2202 }
2203
2204 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2205         int status)
2206 {
2207         int res = 0;
2208
2209         TRACE_ENTRY();
2210
2211         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2212                 ucmd->user_cmd.tm_cmd.fn, status);
2213
2214         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2215                 /*
2216                  * It is possible that user space seen TM cmd before cmd
2217                  * to abort or will never see it at all, because it was
2218                  * aborted on the way there. So, it is safe to return
2219                  * success instead, because, if there is the TM cmd at this
2220                  * point, then the cmd to abort apparrently does exist.
2221                  */
2222                 status = SCST_MGMT_STATUS_SUCCESS;
2223         }
2224
2225         scst_async_mcmd_completed(ucmd->mcmd, status);
2226
2227         ucmd_put(ucmd);
2228
2229         TRACE_EXIT_RES(res);
2230         return res;
2231 }
2232
2233 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2234 {
2235         struct scst_user_cmd *ucmd;
2236         unsigned long flags;
2237
2238         TRACE_ENTRY();
2239
2240         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2241 again:
2242         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2243                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2244                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2245                         switch (ucmd->state) {
2246                         case UCMD_STATE_PARSING:
2247                         case UCMD_STATE_BUF_ALLOCING:
2248                         case UCMD_STATE_EXECING:
2249                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2250                                 list_del(&ucmd->ready_cmd_list_entry);
2251                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2252                                 goto again;
2253                         }
2254                 }
2255         }
2256
2257         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2258
2259         TRACE_EXIT();
2260         return;
2261 }
2262
2263 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2264         struct scst_tgt_dev *tgt_dev)
2265 {
2266         struct scst_user_cmd *ucmd;
2267         struct scst_user_dev *dev =
2268                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2269         struct scst_user_cmd *ucmd_to_abort = NULL;
2270
2271         TRACE_ENTRY();
2272
2273         /*
2274          * In the used approach we don't do anything with hung devices, which
2275          * stopped responding and/or have stuck commands. We forcedly abort such
2276          * commands only if they not yet sent to the user space or if the device
2277          * is getting unloaded, e.g. if its handler program gets killed. This is
2278          * because it's pretty hard to distinguish between stuck and temporary
2279          * overloaded states of the device. There are several reasons for that:
2280          *
2281          * 1. Some commands need a lot of time to complete (several hours),
2282          *    so for an impatient user such command(s) will always look as
2283          *    stuck.
2284          *
2285          * 2. If we forcedly abort, i.e. abort before it's actually completed
2286          *    in the user space, just one command, we will have to put the whole
2287          *    device offline until we are sure that no more previously aborted
2288          *    commands will get executed. Otherwise, we might have a possibility
2289          *    for data corruption, when aborted and reported as completed
2290          *    command actually gets executed *after* new commands sent
2291          *    after the force abort was done. Many journaling file systems and
2292          *    databases use "provide required commands order via queue draining"
2293          *    approach and not putting the whole device offline after the forced
2294          *    abort will break it. This makes our decision, if a command stuck
2295          *    or not, cost a lot.
2296          *
2297          * So, we leave policy definition if a device stuck or not to
2298          * the user space and simply let all commands live until they are
2299          * completed or their devices get closed/killed. This approach is very
2300          * much OK, but can affect management commands, which need activity
2301          * suspending via scst_suspend_activity() function such as devices or
2302          * targets registration/removal. But during normal life such commands
2303          * should be rare. Plus, when possible, scst_suspend_activity() will
2304          * return after timeout EBUSY status to allow caller to not stuck
2305          * forever as well.
2306          *
2307          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2308          * stuck commands would affect only related devices.
2309          */
2310
2311         dev_user_abort_ready_commands(dev);
2312
2313         /* We can't afford missing TM command due to memory shortage */
2314         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2315
2316         ucmd->user_cmd_payload_len =
2317                 offsetof(struct scst_user_get_cmd, tm_cmd) +
2318                 sizeof(ucmd->user_cmd.tm_cmd);
2319         ucmd->user_cmd.cmd_h = ucmd->h;
2320         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2321         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2322         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2323         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2324         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2325
2326         if (mcmd->cmd_to_abort != NULL) {
2327                 ucmd_to_abort =
2328                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2329                 if (ucmd_to_abort != NULL)
2330                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2331         }
2332
2333         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2334                 "ucmd_to_abort %p, cmd_h_to_abort %d, mcmd %p)", ucmd, ucmd->h,
2335                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2336                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort, mcmd);
2337
2338         ucmd->mcmd = mcmd;
2339         ucmd->state = UCMD_STATE_TM_EXECING;
2340
2341         scst_prepare_async_mcmd(mcmd);
2342
2343         dev_user_add_to_ready(ucmd);
2344
2345         TRACE_EXIT();
2346         return SCST_DEV_TM_NOT_COMPLETED;
2347 }
2348
2349 static int dev_user_attach(struct scst_device *sdev)
2350 {
2351         int res = 0;
2352         struct scst_user_dev *dev = NULL, *d;
2353
2354         TRACE_ENTRY();
2355
2356         spin_lock(&dev_list_lock);
2357         list_for_each_entry(d, &dev_list, dev_list_entry) {
2358                 if (strcmp(d->name, sdev->virt_name) == 0) {
2359                         dev = d;
2360                         break;
2361                 }
2362         }
2363         spin_unlock(&dev_list_lock);
2364         if (dev == NULL) {
2365                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2366                 res = -EINVAL;
2367                 goto out;
2368         }
2369
2370         sdev->p_cmd_lists = &dev->cmd_lists;
2371         sdev->dh_priv = dev;
2372         sdev->tst = dev->tst;
2373         sdev->queue_alg = dev->queue_alg;
2374         sdev->swp = dev->swp;
2375         sdev->tas = dev->tas;
2376         sdev->d_sense = dev->d_sense;
2377         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2378
2379         dev->sdev = sdev;
2380
2381         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2382                 dev->name);
2383
2384 out:
2385         TRACE_EXIT();
2386         return res;
2387 }
2388
2389 static void dev_user_detach(struct scst_device *sdev)
2390 {
2391         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2392
2393         TRACE_ENTRY();
2394
2395         TRACE_DBG("virt_id %d", sdev->virt_id);
2396
2397         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2398                 dev->name);
2399
2400         /* dev will be freed by the caller */
2401         sdev->dh_priv = NULL;
2402         dev->sdev = NULL;
2403
2404         TRACE_EXIT();
2405         return;
2406 }
2407
2408 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2409 {
2410         int res = 0;
2411         unsigned long flags;
2412
2413         TRACE_ENTRY();
2414
2415         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2416
2417         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2418
2419         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2420                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2421                 ucmd->result = status;
2422         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2423                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2424         } else
2425                 sBUG();
2426
2427         if (ucmd->cmpl != NULL)
2428                 complete_all(ucmd->cmpl);
2429
2430         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2431
2432         ucmd_put(ucmd);
2433
2434         TRACE_EXIT_RES(res);
2435         return res;
2436 }
2437
2438 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2439 {
2440         struct scst_user_dev *dev =
2441                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2442         int res = 0, rc;
2443         struct scst_user_cmd *ucmd;
2444         DECLARE_COMPLETION_ONSTACK(cmpl);
2445
2446         TRACE_ENTRY();
2447
2448         /*
2449          * We can't replace tgt_dev->pool, because it can be used to allocate
2450          * memory for SCST local commands, like REPORT LUNS, where there is no
2451          * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
2452          */
2453         if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
2454                 tgt_dev->dh_priv = dev->pool_clust;
2455         else
2456                 tgt_dev->dh_priv = dev->pool;
2457
2458         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2459         if (ucmd == NULL)
2460                 goto out_nomem;
2461
2462         ucmd->cmpl = &cmpl;
2463
2464         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2465                 sizeof(ucmd->user_cmd.sess);
2466         ucmd->user_cmd.cmd_h = ucmd->h;
2467         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2468         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2469         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2470         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2471         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only;
2472         strncpy(ucmd->user_cmd.sess.initiator_name,
2473                 tgt_dev->sess->initiator_name,
2474                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2475         ucmd->user_cmd.sess.initiator_name[
2476                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2477         if (tgt_dev->sess->tgt->default_group_name != NULL) {
2478                 strncpy(ucmd->user_cmd.sess.target_name,
2479                         &tgt_dev->sess->tgt->default_group_name[sizeof(SCST_DEFAULT_ACG_NAME)],
2480                         sizeof(ucmd->user_cmd.sess.target_name)-1);
2481                 ucmd->user_cmd.sess.target_name[
2482                         sizeof(ucmd->user_cmd.sess.target_name)-1] = '\0';
2483         }
2484
2485         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2486                 "threads_num %d, rd_only %d, initiator %s, target %s)",
2487                 ucmd, ucmd->h, ucmd->user_cmd.sess.sess_h,
2488                 ucmd->user_cmd.sess.lun, ucmd->user_cmd.sess.threads_num,
2489                 ucmd->user_cmd.sess.rd_only, ucmd->user_cmd.sess.initiator_name,
2490                 ucmd->user_cmd.sess.target_name);
2491
2492         ucmd->state = UCMD_STATE_ATTACH_SESS;
2493
2494         ucmd_get(ucmd);
2495
2496         dev_user_add_to_ready(ucmd);
2497
2498         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2499         if (rc > 0)
2500                 res = ucmd->result;
2501         else {
2502                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2503                 res = -EFAULT;
2504         }
2505
2506         sBUG_ON(irqs_disabled());
2507
2508         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2509         ucmd->cmpl = NULL;
2510         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2511
2512         ucmd_put(ucmd);
2513
2514 out:
2515         TRACE_EXIT_RES(res);
2516         return res;
2517
2518 out_nomem:
2519         res = -ENOMEM;
2520         goto out;
2521 }
2522
2523 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2524 {
2525         struct scst_user_dev *dev =
2526                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2527         struct scst_user_cmd *ucmd;
2528
2529         TRACE_ENTRY();
2530
2531         /*
2532          * We can't miss TM command due to memory shortage, because it might
2533          * lead to a memory leak in the user space handler.
2534          */
2535         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2536         if (ucmd == NULL)
2537                 goto out;
2538
2539         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2540                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2541
2542         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2543                 sizeof(ucmd->user_cmd.sess);
2544         ucmd->user_cmd.cmd_h = ucmd->h;
2545         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2546         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2547
2548         ucmd->state = UCMD_STATE_DETACH_SESS;
2549
2550         dev_user_add_to_ready(ucmd);
2551
2552 out:
2553         TRACE_EXIT();
2554         return;
2555 }
2556
2557 /* No locks are needed, but the activity must be suspended */
2558 static void dev_user_setup_functions(struct scst_user_dev *dev)
2559 {
2560         TRACE_ENTRY();
2561
2562         dev->devtype.parse = dev_user_parse;
2563         dev->devtype.dev_done = NULL;
2564
2565         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2566                 switch (dev->devtype.type) {
2567                 case TYPE_DISK:
2568                         dev->generic_parse = scst_sbc_generic_parse;
2569                         dev->devtype.dev_done = dev_user_disk_done;
2570                         break;
2571
2572                 case TYPE_TAPE:
2573                         dev->generic_parse = scst_tape_generic_parse;
2574                         dev->devtype.dev_done = dev_user_tape_done;
2575                         break;
2576
2577                 case TYPE_MOD:
2578                         dev->generic_parse = scst_modisk_generic_parse;
2579                         dev->devtype.dev_done = dev_user_disk_done;
2580                         break;
2581
2582                 case TYPE_ROM:
2583                         dev->generic_parse = scst_cdrom_generic_parse;
2584                         dev->devtype.dev_done = dev_user_disk_done;
2585                         break;
2586
2587                 case TYPE_MEDIUM_CHANGER:
2588                         dev->generic_parse = scst_changer_generic_parse;
2589                         break;
2590
2591                 case TYPE_PROCESSOR:
2592                         dev->generic_parse = scst_processor_generic_parse;
2593                         break;
2594
2595                 case TYPE_RAID:
2596                         dev->generic_parse = scst_raid_generic_parse;
2597                         break;
2598
2599                 default:
2600                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2601                                 "for it", dev->devtype.type);
2602                         dev->parse_type = SCST_USER_PARSE_CALL;
2603                         break;
2604                 }
2605         } else {
2606                 dev->generic_parse = NULL;
2607                 dev->devtype.dev_done = NULL;
2608         }
2609
2610         TRACE_EXIT();
2611         return;
2612 }
2613
2614 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2615 {
2616         char ver[sizeof(DEV_USER_VERSION)+1];
2617         int res;
2618
2619         res = copy_from_user(ver,
2620                         (void __user *)(unsigned long)dev_desc->version_str,
2621                         sizeof(ver));
2622         if (res < 0) {
2623                 PRINT_ERROR("%s", "Unable to get version string");
2624                 goto out;
2625         }
2626         ver[sizeof(ver)-1] = '\0';
2627
2628         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2629                 /* ->name already 0-terminated in dev_user_ioctl() */
2630                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2631                         dev_desc->name, ver);
2632                 res = -EINVAL;
2633                 goto out;
2634         }
2635
2636 out:
2637         return res;
2638 }
2639
2640 static int dev_user_register_dev(struct file *file,
2641         const struct scst_user_dev_desc *dev_desc)
2642 {
2643         int res, i;
2644         struct scst_user_dev *dev, *d;
2645         int block;
2646
2647         TRACE_ENTRY();
2648
2649         res = dev_user_check_version(dev_desc);
2650         if (res != 0)
2651                 goto out;
2652
2653         switch (dev_desc->type) {
2654         case TYPE_DISK:
2655         case TYPE_ROM:
2656         case TYPE_MOD:
2657                 if (dev_desc->block_size == 0) {
2658                         PRINT_ERROR("Wrong block size %d",
2659                                     dev_desc->block_size);
2660                         res = -EINVAL;
2661                         goto out;
2662                 }
2663                 block = scst_calc_block_shift(dev_desc->block_size);
2664                 if (block == -1) {
2665                         res = -EINVAL;
2666                         goto out;
2667                 }
2668                 break;
2669         default:
2670                 block = dev_desc->block_size;
2671                 break;
2672         }
2673
2674         if (!try_module_get(THIS_MODULE)) {
2675                 PRINT_ERROR("%s", "Fail to get module");
2676                 res = -ETXTBSY;
2677                 goto out;
2678         }
2679
2680         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2681         if (dev == NULL) {
2682                 res = -ENOMEM;
2683                 goto out_put;
2684         }
2685
2686         init_rwsem(&dev->dev_rwsem);
2687         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2688         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2689         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2690         INIT_LIST_HEAD(&dev->ready_cmd_list);
2691         if (file->f_flags & O_NONBLOCK) {
2692                 TRACE_DBG("%s", "Non-blocking operations");
2693                 dev->blocking = 0;
2694         } else
2695                 dev->blocking = 1;
2696         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2697                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2698
2699         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2700         dev->name[sizeof(dev->name)-1] = '\0';
2701
2702         scst_init_mem_lim(&dev->udev_mem_lim);
2703
2704         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2705                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2706                                                   dev_desc->sgv_name);
2707         dev->pool = sgv_pool_create(dev->devtype.name, sgv_no_clustering,
2708                                         dev_desc->sgv_single_alloc_pages,
2709                                         dev_desc->sgv_shared,
2710                                         dev_desc->sgv_purge_interval);
2711         if (dev->pool == NULL) {
2712                 res = -ENOMEM;
2713                 goto out_free_dev;
2714         }
2715         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2716                 dev_user_free_sg_entries);
2717
2718         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust",
2719                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2720                                                   dev_desc->sgv_name);
2721         dev->pool_clust = sgv_pool_create(dev->devtype.name,
2722                                 sgv_tail_clustering,
2723                                 dev_desc->sgv_single_alloc_pages,
2724                                 dev_desc->sgv_shared,
2725                                 dev_desc->sgv_purge_interval);
2726         if (dev->pool_clust == NULL) {
2727                 res = -ENOMEM;
2728                 goto out_free0;
2729         }
2730         sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
2731                 dev_user_free_sg_entries);
2732
2733         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2734                 dev->name);
2735         dev->devtype.type = dev_desc->type;
2736         dev->devtype.threads_num = -1;
2737         dev->devtype.parse_atomic = 1;
2738         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2739         dev->devtype.dev_done_atomic = 1;
2740         dev->devtype.no_proc = 1;
2741         dev->devtype.attach = dev_user_attach;
2742         dev->devtype.detach = dev_user_detach;
2743         dev->devtype.attach_tgt = dev_user_attach_tgt;
2744         dev->devtype.detach_tgt = dev_user_detach_tgt;
2745         dev->devtype.exec = dev_user_exec;
2746         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2747         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2748
2749         init_completion(&dev->cleanup_cmpl);
2750         dev->block = block;
2751         dev->def_block = block;
2752
2753         res = __dev_user_set_opt(dev, &dev_desc->opt);
2754         if (res != 0)
2755                 goto out_free;
2756
2757         TRACE_MEM("dev %p, name %s", dev, dev->name);
2758
2759         spin_lock(&dev_list_lock);
2760
2761         list_for_each_entry(d, &dev_list, dev_list_entry) {
2762                 if (strcmp(d->name, dev->name) == 0) {
2763                         PRINT_ERROR("Device %s already exist",
2764                                 dev->name);
2765                         res = -EEXIST;
2766                         spin_unlock(&dev_list_lock);
2767                         goto out_free;
2768                 }
2769         }
2770
2771         list_add_tail(&dev->dev_list_entry, &dev_list);
2772
2773         spin_unlock(&dev_list_lock);
2774
2775         res = scst_register_virtual_dev_driver(&dev->devtype);
2776         if (res < 0)
2777                 goto out_del_free;
2778
2779         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2780         if (dev->virt_id < 0) {
2781                 res = dev->virt_id;
2782                 goto out_unreg_handler;
2783         }
2784
2785         mutex_lock(&dev_priv_mutex);
2786         if (file->private_data != NULL) {
2787                 mutex_unlock(&dev_priv_mutex);
2788                 PRINT_ERROR("%s", "Device already registered");
2789                 res = -EINVAL;
2790                 goto out_unreg_drv;
2791         }
2792         file->private_data = dev;
2793         mutex_unlock(&dev_priv_mutex);
2794
2795 out:
2796         TRACE_EXIT_RES(res);
2797         return res;
2798
2799 out_unreg_drv:
2800         scst_unregister_virtual_device(dev->virt_id);
2801
2802 out_unreg_handler:
2803         scst_unregister_virtual_dev_driver(&dev->devtype);
2804
2805 out_del_free:
2806         spin_lock(&dev_list_lock);
2807         list_del(&dev->dev_list_entry);
2808         spin_unlock(&dev_list_lock);
2809
2810 out_free:
2811         sgv_pool_del(dev->pool_clust);
2812
2813 out_free0:
2814         sgv_pool_del(dev->pool);
2815
2816 out_free_dev:
2817         kfree(dev);
2818
2819 out_put:
2820         module_put(THIS_MODULE);
2821         goto out;
2822 }
2823
2824 static int dev_user_unregister_dev(struct file *file)
2825 {
2826         int res;
2827         struct scst_user_dev *dev;
2828
2829         TRACE_ENTRY();
2830
2831         mutex_lock(&dev_priv_mutex);
2832         dev = (struct scst_user_dev *)file->private_data;
2833         res = dev_user_check_reg(dev);
2834         if (res != 0) {
2835                 mutex_unlock(&dev_priv_mutex);
2836                 goto out;
2837         }
2838         down_read(&dev->dev_rwsem);
2839         mutex_unlock(&dev_priv_mutex);
2840
2841         res = scst_suspend_activity(true);
2842         if (res != 0)
2843                 goto out_up;
2844
2845         up_read(&dev->dev_rwsem);
2846
2847         mutex_lock(&dev_priv_mutex);
2848         dev = (struct scst_user_dev *)file->private_data;
2849         if (dev == NULL) {
2850                 mutex_unlock(&dev_priv_mutex);
2851                 goto out;
2852         }
2853
2854         dev->blocking = 0;
2855         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
2856
2857         down_write(&dev->dev_rwsem);
2858         file->private_data = NULL;
2859         mutex_unlock(&dev_priv_mutex);
2860
2861         dev_user_exit_dev(dev);
2862
2863         up_write(&dev->dev_rwsem); /* to make lockdep happy */
2864
2865         kfree(dev);
2866
2867         scst_resume_activity();
2868
2869 out:
2870         TRACE_EXIT_RES(res);
2871         return res;
2872
2873 out_up:
2874         up_read(&dev->dev_rwsem);
2875         goto out;
2876 }
2877
2878 static int dev_user_flush_cache(struct file *file)
2879 {
2880         int res;
2881         struct scst_user_dev *dev;
2882
2883         TRACE_ENTRY();
2884
2885         mutex_lock(&dev_priv_mutex);
2886         dev = (struct scst_user_dev *)file->private_data;
2887         res = dev_user_check_reg(dev);
2888         if (res != 0) {
2889                 mutex_unlock(&dev_priv_mutex);
2890                 goto out;
2891         }
2892         down_read(&dev->dev_rwsem);
2893         mutex_unlock(&dev_priv_mutex);
2894
2895         res = scst_suspend_activity(true);
2896         if (res != 0)
2897                 goto out_up;
2898
2899         sgv_pool_flush(dev->pool);
2900         sgv_pool_flush(dev->pool_clust);
2901
2902         scst_resume_activity();
2903
2904 out_up:
2905         up_read(&dev->dev_rwsem);
2906
2907 out:
2908         TRACE_EXIT_RES(res);
2909         return res;
2910 }
2911
2912 static int dev_user_capacity_changed(struct file *file)
2913 {
2914         int res;
2915         struct scst_user_dev *dev;
2916
2917         TRACE_ENTRY();
2918
2919         mutex_lock(&dev_priv_mutex);
2920         dev = (struct scst_user_dev *)file->private_data;
2921         res = dev_user_check_reg(dev);
2922         if (res != 0) {
2923                 mutex_unlock(&dev_priv_mutex);
2924                 goto out;
2925         }
2926         down_read(&dev->dev_rwsem);
2927         mutex_unlock(&dev_priv_mutex);
2928
2929         scst_capacity_data_changed(dev->sdev);
2930
2931         up_read(&dev->dev_rwsem);
2932
2933 out:
2934         TRACE_EXIT_RES(res);
2935         return res;
2936 }
2937
2938
2939 static int __dev_user_set_opt(struct scst_user_dev *dev,
2940         const struct scst_user_opt *opt)
2941 {
2942         int res = 0;
2943
2944         TRACE_ENTRY();
2945
2946         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2947                 "memory_reuse_type %x, partial_transfers_type %x, "
2948                 "partial_len %d", dev->name, opt->parse_type,
2949                 opt->on_free_cmd_type, opt->memory_reuse_type,
2950                 opt->partial_transfers_type, opt->partial_len);
2951
2952         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
2953             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
2954             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
2955             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
2956                 PRINT_ERROR("%s", "Invalid option");
2957                 res = -EINVAL;
2958                 goto out;
2959         }
2960
2961         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2962              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2963             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2964              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2965             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) ||
2966             (opt->d_sense > 1)) {
2967                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
2968                         " tas %x, d_sense %d, has_own_order_mgmt %x)", opt->tst,
2969                         opt->queue_alg, opt->swp, opt->tas, opt->d_sense,
2970                         opt->has_own_order_mgmt);
2971                 res = -EINVAL;
2972                 goto out;
2973         }
2974
2975         dev->parse_type = opt->parse_type;
2976         dev->on_free_cmd_type = opt->on_free_cmd_type;
2977         dev->memory_reuse_type = opt->memory_reuse_type;
2978         dev->partial_transfers_type = opt->partial_transfers_type;
2979         dev->partial_len = opt->partial_len;
2980
2981         dev->tst = opt->tst;
2982         dev->queue_alg = opt->queue_alg;
2983         dev->swp = opt->swp;
2984         dev->tas = opt->tas;
2985         dev->tst = opt->tst;
2986         dev->d_sense = opt->d_sense;
2987         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2988         if (dev->sdev != NULL) {
2989                 dev->sdev->tst = opt->tst;
2990                 dev->sdev->queue_alg = opt->queue_alg;
2991                 dev->sdev->swp = opt->swp;
2992                 dev->sdev->tas = opt->tas;
2993                 dev->sdev->d_sense = opt->d_sense;
2994                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2995         }
2996
2997         dev_user_setup_functions(dev);
2998
2999 out:
3000         TRACE_EXIT_RES(res);
3001         return res;
3002 }
3003
3004 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
3005 {
3006         int res;
3007         struct scst_user_dev *dev;
3008
3009         TRACE_ENTRY();
3010
3011         mutex_lock(&dev_priv_mutex);
3012         dev = (struct scst_user_dev *)file->private_data;
3013         res = dev_user_check_reg(dev);
3014         if (res != 0) {
3015                 mutex_unlock(&dev_priv_mutex);
3016                 goto out;
3017         }
3018         down_read(&dev->dev_rwsem);
3019         mutex_unlock(&dev_priv_mutex);
3020
3021         res = scst_suspend_activity(true);
3022         if (res != 0)
3023                 goto out_up;
3024
3025         res = __dev_user_set_opt(dev, opt);
3026
3027         scst_resume_activity();
3028
3029 out_up:
3030         up_read(&dev->dev_rwsem);
3031
3032 out:
3033         TRACE_EXIT_RES(res);
3034         return res;
3035 }
3036
3037 static int dev_user_get_opt(struct file *file, void __user *arg)
3038 {
3039         int res;
3040         struct scst_user_dev *dev;
3041         struct scst_user_opt opt;
3042
3043         TRACE_ENTRY();
3044
3045         mutex_lock(&dev_priv_mutex);
3046         dev = (struct scst_user_dev *)file->private_data;
3047         res = dev_user_check_reg(dev);
3048         if (res != 0) {
3049                 mutex_unlock(&dev_priv_mutex);
3050                 goto out;
3051         }
3052         down_read(&dev->dev_rwsem);
3053         mutex_unlock(&dev_priv_mutex);
3054
3055         opt.parse_type = dev->parse_type;
3056         opt.on_free_cmd_type = dev->on_free_cmd_type;
3057         opt.memory_reuse_type = dev->memory_reuse_type;
3058         opt.partial_transfers_type = dev->partial_transfers_type;
3059         opt.partial_len = dev->partial_len;
3060         opt.tst = dev->tst;
3061         opt.queue_alg = dev->queue_alg;
3062         opt.tas = dev->tas;
3063         opt.swp = dev->swp;
3064         opt.d_sense = dev->d_sense;
3065         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
3066
3067         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3068                 "memory_reuse_type %x, partial_transfers_type %x, "
3069                 "partial_len %d", dev->name, opt.parse_type,
3070                 opt.on_free_cmd_type, opt.memory_reuse_type,
3071                 opt.partial_transfers_type, opt.partial_len);
3072
3073         res = copy_to_user(arg, &opt, sizeof(opt));
3074
3075         up_read(&dev->dev_rwsem);
3076 out:
3077         TRACE_EXIT_RES(res);
3078         return res;
3079 }
3080
3081 static int dev_usr_parse(struct scst_cmd *cmd)
3082 {
3083         sBUG();
3084         return SCST_CMD_STATE_DEFAULT;
3085 }
3086
3087 /* Needed only for /proc support */
3088 #define USR_TYPE {                      \
3089         .name =         DEV_USER_NAME,  \
3090         .type =         -1,             \
3091         .parse =        dev_usr_parse,  \
3092         .read_proc =    dev_user_read_proc, \
3093 }
3094
3095 static struct scst_dev_type dev_user_devtype = USR_TYPE;
3096
3097 static int dev_user_exit_dev(struct scst_user_dev *dev)
3098 {
3099         TRACE_ENTRY();
3100
3101         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
3102
3103         spin_lock(&dev_list_lock);
3104         list_del(&dev->dev_list_entry);
3105         spin_unlock(&dev_list_lock);
3106
3107         dev->blocking = 0;
3108         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
3109
3110         spin_lock(&cleanup_lock);
3111         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
3112         spin_unlock(&cleanup_lock);
3113
3114         wake_up(&cleanup_list_waitQ);
3115
3116         scst_unregister_virtual_device(dev->virt_id);
3117         scst_unregister_virtual_dev_driver(&dev->devtype);
3118
3119         sgv_pool_flush(dev->pool_clust);
3120         sgv_pool_flush(dev->pool);
3121
3122         TRACE_MGMT_DBG("Unregistering finished (dev %p)", dev);
3123
3124         dev->cleanup_done = 1;
3125
3126         wake_up(&cleanup_list_waitQ);
3127         wake_up(&dev->cmd_lists.cmd_list_waitQ);
3128
3129         wait_for_completion(&dev->cleanup_cmpl);
3130
3131         sgv_pool_del(dev->pool_clust);
3132         sgv_pool_del(dev->pool);
3133
3134         TRACE_MGMT_DBG("Releasing completed (dev %p)", dev);
3135
3136         module_put(THIS_MODULE);
3137
3138         TRACE_EXIT();
3139         return 0;
3140 }
3141
3142 static int __dev_user_release(void *arg)
3143 {
3144         struct scst_user_dev *dev = (struct scst_user_dev *)arg;
3145         dev_user_exit_dev(dev);
3146         kfree(dev);
3147         return 0;
3148 }
3149
3150 static int dev_user_release(struct inode *inode, struct file *file)
3151 {
3152         struct scst_user_dev *dev;
3153         struct task_struct *t;
3154
3155         TRACE_ENTRY();
3156
3157         dev = (struct scst_user_dev *)file->private_data;
3158         if (dev == NULL) {
3159                 mutex_unlock(&dev_priv_mutex);
3160                 goto out;
3161         }
3162         file->private_data = NULL;
3163
3164         TRACE_MGMT_DBG("Going to release dev %s", dev->name);
3165
3166         t = kthread_run(__dev_user_release, dev, "scst_usr_released");
3167         if (IS_ERR(t)) {
3168                 PRINT_CRIT_ERROR("kthread_run() failed (%ld), releasing device "
3169                         "%p directly. If you have several devices under load "
3170                         "it might deadlock!", PTR_ERR(t), dev);
3171                 __dev_user_release(dev);
3172         }
3173
3174 out:
3175         TRACE_EXIT();
3176         return 0;
3177 }
3178
3179 static int dev_user_process_cleanup(struct scst_user_dev *dev)
3180 {
3181         struct scst_user_cmd *ucmd;
3182         int rc = 0, res = 1;
3183
3184         TRACE_ENTRY();
3185
3186         sBUG_ON(dev->blocking);
3187         wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
3188
3189         while (1) {
3190                 int rc1;
3191
3192                 TRACE_DBG("Cleanuping dev %p", dev);
3193
3194                 rc1 = dev_user_unjam_dev(dev);
3195                 if ((rc1 == 0) && (rc == -EAGAIN) && dev->cleanup_done)
3196                         break;
3197
3198                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
3199
3200                 rc = dev_user_get_next_cmd(dev, &ucmd);
3201                 if (rc == 0)
3202                         dev_user_unjam_cmd(ucmd, 1, NULL);
3203
3204                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
3205
3206                 if (rc == -EAGAIN) {
3207                         if (!dev->cleanup_done) {
3208                                 TRACE_DBG("No more commands (dev %p)", dev);
3209                                 goto out;
3210                         }
3211                 }
3212         }
3213
3214 #ifdef CONFIG_SCST_EXTRACHECKS
3215 {
3216         int i;
3217         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3218                 struct list_head *head = &dev->ucmd_hash[i];
3219                 struct scst_user_cmd *ucmd2;
3220 again:
3221                 list_for_each_entry(ucmd2, head, hash_list_entry) {
3222                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
3223                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
3224                         ucmd_put(ucmd2);
3225                         goto again;
3226                 }
3227         }
3228 }
3229 #endif
3230
3231         TRACE_DBG("Cleanuping done (dev %p)", dev);
3232         complete_all(&dev->cleanup_cmpl);
3233         res = 0;
3234
3235 out:
3236         TRACE_EXIT_RES(res);
3237         return res;
3238 }
3239
3240
3241 /*
3242  * Called when a file in the /proc/scsi_tgt/scst_user is read
3243  */
3244 static int dev_user_read_proc(struct seq_file *seq, struct scst_dev_type *dev_type)
3245 {
3246         int res = 0;
3247         struct scst_user_dev *dev;
3248         unsigned long flags;
3249
3250         TRACE_ENTRY();
3251
3252         spin_lock(&dev_list_lock);
3253
3254         list_for_each_entry(dev, &dev_list, dev_list_entry) {
3255                 int i;
3256                 seq_printf(seq, "Device %s commands:\n", dev->name);
3257                 spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
3258                 for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3259                         struct list_head *head = &dev->ucmd_hash[i];
3260                         struct scst_user_cmd *ucmd;
3261                         list_for_each_entry(ucmd, head, hash_list_entry) {
3262                                 seq_printf(seq, "ucmd %p (state %x, ref %d), "
3263                                         "sent_to_user %d, seen_by_user %d, "
3264                                         "aborted %d, jammed %d, scst_cmd %p\n",
3265                                         ucmd, ucmd->state,
3266                                         atomic_read(&ucmd->ucmd_ref),
3267                                         ucmd->sent_to_user, ucmd->seen_by_user,
3268                                         ucmd->aborted, ucmd->jammed, ucmd->cmd);
3269                         }
3270                 }
3271                 spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
3272         }
3273         spin_unlock(&dev_list_lock);
3274
3275         TRACE_EXIT_RES(res);
3276         return res;
3277 }
3278
3279 static inline int test_cleanup_list(void)
3280 {
3281         int res = !list_empty(&cleanup_list) ||
3282                   unlikely(kthread_should_stop());
3283         return res;
3284 }
3285
3286 static int dev_user_cleanup_thread(void *arg)
3287 {
3288         TRACE_ENTRY();
3289
3290         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
3291
3292         current->flags |= PF_NOFREEZE;
3293
3294         spin_lock(&cleanup_lock);
3295         while (!kthread_should_stop()) {
3296                 wait_queue_t wait;
3297                 init_waitqueue_entry(&wait, current);
3298
3299                 if (!test_cleanup_list()) {
3300                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
3301                         for (;;) {
3302                                 set_current_state(TASK_INTERRUPTIBLE);
3303                                 if (test_cleanup_list())
3304                                         break;
3305                                 spin_unlock(&cleanup_lock);
3306                                 schedule();
3307                                 spin_lock(&cleanup_lock);
3308                         }
3309                         set_current_state(TASK_RUNNING);
3310                         remove_wait_queue(&cleanup_list_waitQ, &wait);
3311                 }
3312
3313                 /*
3314                  * We have to poll devices, because commands can go from SCST
3315                  * core on cmd_list_waitQ and we have no practical way to
3316                  * detect them.
3317                  */
3318
3319                 while (1) {
3320                         struct scst_user_dev *dev;
3321                         LIST_HEAD(cl_devs);
3322
3323                         while (!list_empty(&cleanup_list)) {
3324                                 int rc;
3325
3326                                 dev = list_entry(cleanup_list.next,
3327                                         typeof(*dev), cleanup_list_entry);
3328                                 list_del(&dev->cleanup_list_entry);
3329
3330                                 spin_unlock(&cleanup_lock);
3331                                 rc = dev_user_process_cleanup(dev);
3332                                 spin_lock(&cleanup_lock);
3333
3334                                 if (rc != 0)
3335                                         list_add_tail(&dev->cleanup_list_entry,
3336                                                 &cl_devs);
3337                         }
3338
3339                         if (list_empty(&cl_devs))
3340                                 break;
3341
3342                         spin_unlock(&cleanup_lock);
3343                         msleep(100);
3344                         spin_lock(&cleanup_lock);
3345
3346                         while (!list_empty(&cl_devs)) {
3347                                 dev = list_entry(cl_devs.next, typeof(*dev),
3348                                         cleanup_list_entry);
3349                                 list_move_tail(&dev->cleanup_list_entry,
3350                                         &cleanup_list);
3351                         }
3352                 }
3353         }
3354         spin_unlock(&cleanup_lock);
3355
3356         /*
3357          * If kthread_should_stop() is true, we are guaranteed to be
3358          * on the module unload, so cleanup_list must be empty.
3359          */
3360         sBUG_ON(!list_empty(&cleanup_list));
3361
3362         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
3363
3364         TRACE_EXIT();
3365         return 0;
3366 }
3367
3368 static int __init init_scst_user(void)
3369 {
3370         int res = 0;
3371         struct max_get_reply {
3372                 union {
3373                         struct scst_user_get_cmd g;
3374                         struct scst_user_reply_cmd r;
3375                 };
3376         };
3377 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3378         struct class_device *class_member;
3379 #else
3380         struct device *dev;
3381 #endif
3382
3383         TRACE_ENTRY();
3384
3385 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
3386         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
3387                 "Consider changing VMSPLIT option or use a 64-bit "
3388                 "configuration instead. See README file for details.");
3389         res = -EINVAL;
3390         goto out;
3391 #endif
3392
3393         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
3394         if (user_cmd_cachep == NULL) {
3395                 res = -ENOMEM;
3396                 goto out;
3397         }
3398
3399         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
3400         if (user_get_cmd_cachep == NULL) {
3401                 res = -ENOMEM;
3402                 goto out_cache;
3403         }
3404
3405         dev_user_devtype.module = THIS_MODULE;
3406
3407         res = scst_register_virtual_dev_driver(&dev_user_devtype);
3408         if (res < 0)
3409                 goto out_cache1;
3410
3411         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
3412         if (res != 0)
3413                 goto out_unreg;
3414
3415         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
3416         if (IS_ERR(dev_user_sysfs_class)) {
3417                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
3418                         "space handler");
3419                 res = PTR_ERR(dev_user_sysfs_class);
3420                 goto out_proc;
3421         }
3422
3423         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
3424         if (res) {
3425                 PRINT_ERROR("Unable to get major %d for SCSI tapes",
3426                             DEV_USER_MAJOR);
3427                 goto out_class;
3428         }
3429
3430 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3431         class_member = class_device_create(dev_user_sysfs_class, NULL,
3432                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
3433         if (IS_ERR(class_member)) {
3434                 res = PTR_ERR(class_member);
3435                 goto out_chrdev;
3436         }
3437 #else
3438         dev = device_create(dev_user_sysfs_class, NULL,
3439                             MKDEV(DEV_USER_MAJOR, 0),
3440 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
3441                                 NULL,
3442 #endif
3443                                 DEV_USER_NAME);
3444         if (IS_ERR(dev)) {
3445                 res = PTR_ERR(dev);
3446                 goto out_chrdev;
3447         }
3448 #endif
3449
3450         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
3451                 "scst_usr_cleanupd");
3452         if (IS_ERR(cleanup_thread)) {
3453                 res = PTR_ERR(cleanup_thread);
3454                 PRINT_ERROR("kthread_create() failed: %d", res);
3455                 goto out_dev;
3456         }
3457
3458 out:
3459         TRACE_EXIT_RES(res);
3460         return res;
3461
3462 out_dev:
3463 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3464         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3465 #else
3466         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3467 #endif
3468
3469 out_chrdev:
3470         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3471
3472 out_class:
3473         class_destroy(dev_user_sysfs_class);
3474
3475 out_proc:
3476         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3477
3478 out_unreg:
3479         scst_unregister_dev_driver(&dev_user_devtype);
3480
3481 out_cache1:
3482         kmem_cache_destroy(user_get_cmd_cachep);
3483
3484 out_cache:
3485         kmem_cache_destroy(user_cmd_cachep);
3486         goto out;
3487 }
3488
3489 static void __exit exit_scst_user(void)
3490 {
3491         int rc;
3492
3493         TRACE_ENTRY();
3494
3495         rc = kthread_stop(cleanup_thread);
3496         if (rc < 0)
3497                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3498
3499         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3500 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3501         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3502 #else
3503         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3504 #endif
3505         class_destroy(dev_user_sysfs_class);
3506
3507         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3508         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3509
3510         kmem_cache_destroy(user_get_cmd_cachep);
3511         kmem_cache_destroy(user_cmd_cachep);
3512
3513         TRACE_EXIT();
3514         return;
3515 }
3516
3517 module_init(init_scst_user);
3518 module_exit(exit_scst_user);
3519
3520 MODULE_AUTHOR("Vladislav Bolkhovitin");
3521 MODULE_LICENSE("GPL");
3522 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3523 MODULE_VERSION(SCST_VERSION_STRING);
3524 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);