Fixes incorrect errors reporting on devices unregistration.
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2009 ID7 Ltd.
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23 #include <linux/stddef.h>
24
25 #define LOG_PREFIX              DEV_USER_NAME
26
27 #include "scst.h"
28 #include "scst_user.h"
29 #include "scst_dev_handler.h"
30
31 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
32 #warning "HIGHMEM kernel configurations are not supported by this module,\
33  because nowadays it doesn't worth the effort. Consider changing\
34  VMSPLIT option or use a 64-bit configuration instead. See README file\
35  for details."
36 #endif
37
38 #define DEV_USER_MAJOR                  237
39 #define DEV_USER_CMD_HASH_ORDER         6
40 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
41
42 struct scst_user_dev {
43         struct rw_semaphore dev_rwsem;
44
45         struct scst_cmd_lists cmd_lists;
46
47         /* Protected by cmd_lists.cmd_list_lock */
48         struct list_head ready_cmd_list;
49
50         /* Protected by dev_rwsem or don't need any protection */
51         unsigned int blocking:1;
52         unsigned int cleanup_done:1;
53         unsigned int tst:3;
54         unsigned int queue_alg:4;
55         unsigned int tas:1;
56         unsigned int swp:1;
57         unsigned int d_sense:1;
58         unsigned int has_own_order_mgmt:1;
59
60         int (*generic_parse)(struct scst_cmd *cmd,
61                 int (*get_block)(struct scst_cmd *cmd));
62
63         int block;
64         int def_block;
65
66         struct scst_mem_lim udev_mem_lim;
67         struct sgv_pool *pool;
68         struct sgv_pool *pool_clust;
69
70         uint8_t parse_type;
71         uint8_t on_free_cmd_type;
72         uint8_t memory_reuse_type;
73         uint8_t partial_transfers_type;
74         uint32_t partial_len;
75
76         struct scst_dev_type devtype;
77
78         /* Both protected by cmd_lists.cmd_list_lock */
79         unsigned int handle_counter;
80         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
81
82         struct scst_device *sdev;
83
84         int virt_id;
85         struct list_head dev_list_entry;
86         char name[SCST_MAX_NAME];
87
88         struct list_head cleanup_list_entry;
89         struct completion cleanup_cmpl;
90 };
91
92 /* Most fields are unprotected, since only one thread at time can access them */
93 struct scst_user_cmd {
94         struct scst_cmd *cmd;
95         struct scst_user_dev *dev;
96
97         atomic_t ucmd_ref;
98
99         unsigned int buff_cached:1;
100         unsigned int buf_dirty:1;
101         unsigned int background_exec:1;
102         unsigned int aborted:1;
103
104         struct scst_user_cmd *buf_ucmd;
105
106         int cur_data_page;
107         int num_data_pages;
108         int first_page_offset;
109         unsigned long ubuff;
110         struct page **data_pages;
111         struct sgv_pool_obj *sgv;
112
113         /*
114          * Special flags, which can be accessed asynchronously (hence "long").
115          * Protected by cmd_lists.cmd_list_lock.
116          */
117         unsigned long sent_to_user:1;
118         unsigned long jammed:1;
119         unsigned long this_state_unjammed:1;
120         unsigned long seen_by_user:1; /* here only as a small optimization */
121
122         unsigned int state;
123
124         struct list_head ready_cmd_list_entry;
125
126         unsigned int h;
127         struct list_head hash_list_entry;
128
129         int user_cmd_payload_len;
130         struct scst_user_get_cmd user_cmd;
131
132         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
133         union {
134                 struct completion *cmpl;
135                 struct scst_mgmt_cmd *mcmd;
136         };
137         int result;
138 };
139
140 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
141         gfp_t gfp_mask);
142 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
143
144 static int dev_user_parse(struct scst_cmd *cmd);
145 static int dev_user_exec(struct scst_cmd *cmd);
146 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
147 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
148         struct scst_tgt_dev *tgt_dev);
149
150 static int dev_user_disk_done(struct scst_cmd *cmd);
151 static int dev_user_tape_done(struct scst_cmd *cmd);
152
153 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
154         gfp_t gfp_mask, void *priv);
155 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
156                                      void *priv);
157
158 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
159
160 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
161         unsigned long *flags);
162
163 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
164 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
165         int status);
166 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
167 static int dev_user_register_dev(struct file *file,
168         const struct scst_user_dev_desc *dev_desc);
169 static int dev_user_unregister_dev(struct file *file);
170 static int dev_user_flush_cache(struct file *file);
171 static int dev_user_capacity_changed(struct file *file);
172 static int __dev_user_set_opt(struct scst_user_dev *dev,
173         const struct scst_user_opt *opt);
174 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
175 static int dev_user_get_opt(struct file *file, void __user *arg);
176
177 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
178 static long dev_user_ioctl(struct file *file, unsigned int cmd,
179         unsigned long arg);
180 static int dev_user_release(struct inode *inode, struct file *file);
181
182 /** Data **/
183
184 static struct kmem_cache *user_cmd_cachep;
185 static struct kmem_cache *user_get_cmd_cachep;
186
187 static DEFINE_MUTEX(dev_priv_mutex);
188
189 static const struct file_operations dev_user_fops = {
190         .poll           = dev_user_poll,
191         .unlocked_ioctl = dev_user_ioctl,
192 #ifdef CONFIG_COMPAT
193         .compat_ioctl   = dev_user_ioctl,
194 #endif
195         .release        = dev_user_release,
196 };
197
198 static struct class *dev_user_sysfs_class;
199
200 static DEFINE_SPINLOCK(dev_list_lock);
201 static LIST_HEAD(dev_list);
202
203 static DEFINE_SPINLOCK(cleanup_lock);
204 static LIST_HEAD(cleanup_list);
205 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
206 static struct task_struct *cleanup_thread;
207
208 /*
209  * Skip this command if result is not 0. Must be called under
210  * cmd_lists.cmd_list_lock and IRQ off.
211  */
212 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
213 {
214         int r = atomic_inc_return(&ucmd->ucmd_ref);
215         int res;
216         if (unlikely(r == 1)) {
217                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
218                 atomic_dec(&ucmd->ucmd_ref);
219                 res = true;
220                 /*
221                  * Necessary code is serialized by cmd_list_lock in
222                  * cmd_remove_hash()
223                  */
224         } else {
225                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
226                         atomic_read(&ucmd->ucmd_ref));
227                 res = false;
228         }
229         return res;
230 }
231
232 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
233 {
234         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
235         atomic_inc(&ucmd->ucmd_ref);
236         if (barrier)
237                 smp_mb__after_atomic_inc();
238 }
239
240 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
241 {
242         __ucmd_get(ucmd, true);
243 }
244
245 static inline void ucmd_get(struct scst_user_cmd *ucmd)
246 {
247         __ucmd_get(ucmd, false);
248 }
249
250 /* Must not be called under cmd_list_lock!! */
251 static inline void ucmd_put(struct scst_user_cmd *ucmd)
252 {
253         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
254
255         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
256
257         if (atomic_dec_and_test(&ucmd->ucmd_ref))
258                 dev_user_free_ucmd(ucmd);
259 }
260
261 static inline int calc_num_pg(unsigned long buf, int len)
262 {
263         len += buf & ~PAGE_MASK;
264         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
265 }
266
267 static void __dev_user_not_reg(void)
268 {
269         TRACE(TRACE_MGMT_MINOR, "%s", "Device not registered");
270         return;
271 }
272
273 static inline int dev_user_check_reg(struct scst_user_dev *dev)
274 {
275         if (dev == NULL) {
276                 __dev_user_not_reg();
277                 return -ENODEV;
278         }
279         return 0;
280 }
281
282 static inline int scst_user_cmd_hashfn(int h)
283 {
284         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
285 }
286
287 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
288         unsigned int h)
289 {
290         struct list_head *head;
291         struct scst_user_cmd *ucmd;
292
293         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
294         list_for_each_entry(ucmd, head, hash_list_entry) {
295                 if (ucmd->h == h) {
296                         TRACE_DBG("Found ucmd %p", ucmd);
297                         return ucmd;
298                 }
299         }
300         return NULL;
301 }
302
303 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
304 {
305         struct list_head *head;
306         struct scst_user_dev *dev = ucmd->dev;
307         struct scst_user_cmd *u;
308         unsigned long flags;
309
310         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
311         do {
312                 ucmd->h = dev->handle_counter++;
313                 u = __ucmd_find_hash(dev, ucmd->h);
314         } while (u != NULL);
315         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
316         list_add_tail(&ucmd->hash_list_entry, head);
317         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
318
319         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
320         return;
321 }
322
323 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
324 {
325         unsigned long flags;
326
327         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
328         list_del(&ucmd->hash_list_entry);
329         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
330
331         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
332         return;
333 }
334
335 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
336 {
337         TRACE_ENTRY();
338
339         TRACE_MEM("Freeing ucmd %p", ucmd);
340
341         cmd_remove_hash(ucmd);
342         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
343
344         kmem_cache_free(user_cmd_cachep, ucmd);
345
346         TRACE_EXIT();
347         return;
348 }
349
350 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
351         gfp_t gfp_mask, void *priv)
352 {
353         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
354         int offset = 0;
355
356         TRACE_ENTRY();
357
358         /* *sg supposed to be zeroed */
359
360         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
361                 ucmd->ubuff, ucmd->cur_data_page);
362
363         if (ucmd->cur_data_page == 0) {
364                 TRACE_MEM("ucmd->first_page_offset %d",
365                         ucmd->first_page_offset);
366                 offset = ucmd->first_page_offset;
367                 ucmd_get(ucmd);
368         }
369
370         if (ucmd->cur_data_page >= ucmd->num_data_pages)
371                 goto out;
372
373         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
374                 PAGE_SIZE - offset, offset);
375         ucmd->cur_data_page++;
376
377         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
378                 sg->offset);
379         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
380
381 out:
382         TRACE_EXIT();
383         return sg_page(sg);
384 }
385
386 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
387 {
388         TRACE_ENTRY();
389
390         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
391                 ucmd, ucmd->h, ucmd->ubuff);
392
393         ucmd->user_cmd_payload_len =
394                 offsetof(struct scst_user_get_cmd, on_cached_mem_free) +
395                 sizeof(ucmd->user_cmd.on_cached_mem_free);
396         ucmd->user_cmd.cmd_h = ucmd->h;
397         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
398         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
399
400         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
401
402         dev_user_add_to_ready(ucmd);
403
404         TRACE_EXIT();
405         return;
406 }
407
408 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
409 {
410         int i;
411
412         TRACE_ENTRY();
413
414         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
415                 ucmd->ubuff, ucmd->num_data_pages);
416
417         for (i = 0; i < ucmd->num_data_pages; i++) {
418                 struct page *page = ucmd->data_pages[i];
419
420                 if (ucmd->buf_dirty)
421                         SetPageDirty(page);
422
423                 page_cache_release(page);
424         }
425
426         kfree(ucmd->data_pages);
427         ucmd->data_pages = NULL;
428
429         TRACE_EXIT();
430         return;
431 }
432
433 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
434 {
435         TRACE_ENTRY();
436
437         sBUG_ON(ucmd->data_pages == NULL);
438
439         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
440                 ucmd, ucmd->ubuff, ucmd->buff_cached);
441
442         dev_user_unmap_buf(ucmd);
443
444         if (ucmd->buff_cached)
445                 dev_user_on_cached_mem_free(ucmd);
446         else
447                 ucmd_put(ucmd);
448
449         TRACE_EXIT();
450         return;
451 }
452
453 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
454         void *priv)
455 {
456         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
457
458         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
459                 sg_count, ucmd);
460
461         __dev_user_free_sg_entries(ucmd);
462
463         return;
464 }
465
466 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
467 {
468         int mem_reuse_type = ucmd->dev->memory_reuse_type;
469
470         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
471             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
472              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
473             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
474              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
475                 return 1;
476         else
477                 return 0;
478 }
479
480 static inline int is_need_offs_page(unsigned long buf, int len)
481 {
482         return ((buf & ~PAGE_MASK) != 0) &&
483                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
484 }
485
486 /*
487  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
488  * Unmaps the buffer, if needed in case of error
489  */
490 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
491 {
492         int res = 0;
493         struct scst_cmd *cmd = ucmd->cmd;
494         struct scst_user_dev *dev = ucmd->dev;
495         struct sgv_pool *pool;
496         gfp_t gfp_mask;
497         int flags = 0;
498         int bufflen, orig_bufflen;
499         int last_len = 0;
500         int out_sg_pages = 0;
501
502         TRACE_ENTRY();
503
504         gfp_mask = __GFP_NOWARN;
505         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
506
507         if (cmd->data_direction != SCST_DATA_BIDI) {
508                 orig_bufflen = cmd->bufflen;
509                 pool = (struct sgv_pool *)cmd->tgt_dev->dh_priv;
510         } else {
511                 /* Make in_sg->offset 0 */
512                 int len = cmd->bufflen + ucmd->first_page_offset;
513                 out_sg_pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
514                 orig_bufflen = (out_sg_pages << PAGE_SHIFT) + cmd->in_bufflen;
515                 pool = dev->pool;
516         }
517         bufflen = orig_bufflen;
518
519         EXTRACHECKS_BUG_ON(bufflen == 0);
520
521         if (cached_buff) {
522                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
523                 if (ucmd->ubuff == 0)
524                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
525         } else {
526                 TRACE_MEM("%s", "Not cached buff");
527                 flags |= SCST_POOL_ALLOC_NO_CACHED;
528                 if (ucmd->ubuff == 0) {
529                         res = 1;
530                         goto out;
531                 }
532                 bufflen += ucmd->first_page_offset;
533                 if (is_need_offs_page(ucmd->ubuff, orig_bufflen))
534                         last_len = bufflen & ~PAGE_MASK;
535                 else
536                         last_len = orig_bufflen & ~PAGE_MASK;
537                 if (last_len == 0)
538                         last_len = PAGE_SIZE;
539         }
540         ucmd->buff_cached = cached_buff;
541
542         cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt,
543                         &ucmd->sgv, &dev->udev_mem_lim, ucmd);
544         if (cmd->sg != NULL) {
545                 struct scst_user_cmd *buf_ucmd =
546                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
547
548                 TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
549                         "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
550                         cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
551
552                 ucmd->ubuff = buf_ucmd->ubuff;
553                 ucmd->buf_ucmd = buf_ucmd;
554
555                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
556                                    (ucmd != buf_ucmd));
557
558                 if (last_len != 0) {
559                         cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
560                         cmd->sg[cmd->sg_cnt-1].length += last_len;
561                 }
562
563                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
564                         "last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
565                         cmd->sg[cmd->sg_cnt-1].length);
566
567                 if (cmd->data_direction == SCST_DATA_BIDI) {
568                         cmd->in_sg = &cmd->sg[out_sg_pages];
569                         cmd->in_sg_cnt = cmd->sg_cnt - out_sg_pages;
570                         cmd->sg_cnt = out_sg_pages;
571                         TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d",
572                                 cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt);
573                 }
574
575                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
576                         static int ll;
577                         if (ll < 10) {
578                                 PRINT_INFO("Unable to complete command due to "
579                                         "SG IO count limitation (requested %d, "
580                                         "available %d, tgt lim %d)",
581                                         cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt,
582                                         cmd->tgt->sg_tablesize);
583                                 ll++;
584                         }
585                         cmd->sg = NULL;
586                         /* sgv will be freed in dev_user_free_sgv() */
587                         res = -1;
588                 }
589         } else {
590                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
591                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
592                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
593                 if (unlikely(cmd->sg_cnt == 0)) {
594                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
595                         sBUG_ON(ucmd->sgv != NULL);
596                         res = -1;
597                 } else {
598                         switch (ucmd->state) {
599                         case UCMD_STATE_BUF_ALLOCING:
600                                 res = 1;
601                                 break;
602                         case UCMD_STATE_EXECING:
603                                 res = -1;
604                                 break;
605                         default:
606                                 sBUG();
607                                 break;
608                         }
609                 }
610         }
611
612 out:
613         TRACE_EXIT_RES(res);
614         return res;
615 }
616
617 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
618 {
619         int rc, res = SCST_CMD_STATE_DEFAULT;
620         struct scst_cmd *cmd = ucmd->cmd;
621
622         TRACE_ENTRY();
623
624         ucmd->state = UCMD_STATE_BUF_ALLOCING;
625         cmd->dh_data_buf_alloced = 1;
626
627         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
628         if (rc == 0)
629                 goto out;
630         else if (rc < 0) {
631                 scst_set_busy(cmd);
632                 res = scst_get_cmd_abnormal_done_state(cmd);
633                 goto out;
634         }
635
636         if ((cmd->data_direction != SCST_DATA_WRITE) &&
637             !scst_is_cmd_local(cmd)) {
638                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
639                 goto out;
640         }
641
642         ucmd->user_cmd_payload_len =
643                 offsetof(struct scst_user_get_cmd, alloc_cmd) +
644                 sizeof(ucmd->user_cmd.alloc_cmd);
645         ucmd->user_cmd.cmd_h = ucmd->h;
646         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
647         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
648         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb, cmd->cdb_len);
649         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
650         ucmd->user_cmd.alloc_cmd.ext_cdb_len = cmd->ext_cdb_len;
651         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
652                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
653         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
654         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
655         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
656
657         dev_user_add_to_ready(ucmd);
658
659         res = SCST_CMD_STATE_STOP;
660
661 out:
662         TRACE_EXIT_RES(res);
663         return res;
664 }
665
666 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
667         gfp_t gfp_mask)
668 {
669         struct scst_user_cmd *ucmd = NULL;
670
671         TRACE_ENTRY();
672
673 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
674         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
675         if (ucmd != NULL)
676                 memset(ucmd, 0, sizeof(*ucmd));
677 #else
678         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
679 #endif
680         if (unlikely(ucmd == NULL)) {
681                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
682                         "user cmd (gfp_mask %x)", gfp_mask);
683                 goto out;
684         }
685         ucmd->dev = dev;
686         atomic_set(&ucmd->ucmd_ref, 1);
687
688         cmd_insert_hash(ucmd);
689
690         TRACE_MEM("ucmd %p allocated", ucmd);
691
692 out:
693         TRACE_EXIT_HRES((unsigned long)ucmd);
694         return ucmd;
695 }
696
697 static int dev_user_get_block(struct scst_cmd *cmd)
698 {
699         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
700         /*
701          * No need for locks here, since *_detach() can not be
702          * called, when there are existing commands.
703          */
704         TRACE_EXIT_RES(dev->block);
705         return dev->block;
706 }
707
708 static int dev_user_parse(struct scst_cmd *cmd)
709 {
710         int rc, res = SCST_CMD_STATE_DEFAULT;
711         struct scst_user_cmd *ucmd;
712         int atomic = scst_cmd_atomic(cmd);
713         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
714         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
715
716         TRACE_ENTRY();
717
718         if (cmd->dh_priv == NULL) {
719                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
720                 if (unlikely(ucmd == NULL)) {
721                         if (atomic) {
722                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
723                                 goto out;
724                         } else {
725                                 scst_set_busy(cmd);
726                                 goto out_error;
727                         }
728                 }
729                 ucmd->cmd = cmd;
730                 cmd->dh_priv = ucmd;
731         } else {
732                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
733                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
734         }
735
736         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
737
738         if (ucmd->state != UCMD_STATE_NEW)
739                 goto alloc;
740
741         switch (dev->parse_type) {
742         case SCST_USER_PARSE_STANDARD:
743                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
744                 rc = dev->generic_parse(cmd, dev_user_get_block);
745                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_NOT_FOUND))
746                         goto out_invalid;
747                 break;
748
749         case SCST_USER_PARSE_EXCEPTION:
750                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
751                 rc = dev->generic_parse(cmd, dev_user_get_block);
752                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_NOT_FOUND)))
753                         break;
754                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
755                         TRACE_MEM("Restarting PARSE to thread context "
756                                 "(ucmd %p)", ucmd);
757                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
758                         goto out;
759                 }
760                 /* else go through */
761
762         case SCST_USER_PARSE_CALL:
763                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
764                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
765                 ucmd->user_cmd_payload_len =
766                         offsetof(struct scst_user_get_cmd, parse_cmd) +
767                         sizeof(ucmd->user_cmd.parse_cmd);
768                 ucmd->user_cmd.cmd_h = ucmd->h;
769                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
770                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
771                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb, cmd->cdb_len);
772                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
773                 ucmd->user_cmd.parse_cmd.ext_cdb_len = cmd->ext_cdb_len;
774                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
775                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
776                 ucmd->user_cmd.parse_cmd.in_bufflen = cmd->in_bufflen;
777                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
778                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
779                 ucmd->user_cmd.parse_cmd.expected_values_set =
780                                         cmd->expected_values_set;
781                 ucmd->user_cmd.parse_cmd.expected_data_direction =
782                                         cmd->expected_data_direction;
783                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
784                                         cmd->expected_transfer_len;
785                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
786                 ucmd->state = UCMD_STATE_PARSING;
787                 dev_user_add_to_ready(ucmd);
788                 res = SCST_CMD_STATE_STOP;
789                 goto out;
790
791         default:
792                 sBUG();
793                 goto out;
794         }
795
796 alloc:
797         if (cmd->bufflen == 0) {
798                 /*
799                  * According to SPC bufflen 0 for data transfer commands isn't
800                  * an error, so we need to fix the transfer direction.
801                  */
802                 cmd->data_direction = SCST_DATA_NONE;
803         }
804
805         if (cmd->data_direction != SCST_DATA_NONE)
806                 res = dev_user_alloc_space(ucmd);
807
808 out:
809         TRACE_EXIT_RES(res);
810         return res;
811
812 out_invalid:
813         PRINT_ERROR("PARSE failed (ucmd %p, rc %d)", ucmd, rc);
814         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
815
816 out_error:
817         res = scst_get_cmd_abnormal_done_state(cmd);
818         goto out;
819 }
820
821 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
822 {
823         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
824         unsigned long start = buf_ucmd->ubuff;
825         int i, bufflen = ucmd->cmd->bufflen;
826
827         TRACE_ENTRY();
828
829         if (start == 0)
830                 goto out;
831
832         /*
833          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
834          * faster, since it should be cache hot, while ucmd->buf_ucmd and
835          * buf_ucmd->data_pages are cache cold. But, from other side,
836          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
837          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
838          * data_pages array can lead to less cache misses. So, real numbers are
839          * needed. ToDo.
840          */
841
842         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
843                 struct page *page;
844                 page = buf_ucmd->data_pages[i];
845 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
846                 struct vm_area_struct *vma = find_vma(current->mm, start);
847                 if (vma != NULL)
848                         flush_anon_page(vma, page, start);
849 #endif
850                 flush_dcache_page(page);
851                 start += PAGE_SIZE;
852                 bufflen -= PAGE_SIZE;
853         }
854
855 out:
856         TRACE_EXIT();
857         return;
858 }
859
860 static int dev_user_exec(struct scst_cmd *cmd)
861 {
862         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
863         int res = SCST_EXEC_COMPLETED;
864
865         TRACE_ENTRY();
866
867 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
868        * context to complete the necessary actions, but all we are going to
869        * do in this function is, in fact, atomic, so let's skip this check.
870        */
871         if (scst_cmd_atomic(cmd)) {
872                 TRACE_DBG("%s", "User exec() can not be called in atomic "
873                         "context, rescheduling to the thread");
874                 res = SCST_EXEC_NEED_THREAD;
875                 goto out;
876         }
877 #endif
878
879         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
880                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
881                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
882
883         if (cmd->data_direction == SCST_DATA_WRITE)
884                 dev_user_flush_dcache(ucmd);
885
886         BUILD_BUG_ON(sizeof(ucmd->user_cmd.exec_cmd.cdb) != sizeof(cmd->cdb));
887
888         ucmd->user_cmd_payload_len =
889                 offsetof(struct scst_user_get_cmd, exec_cmd) +
890                 sizeof(ucmd->user_cmd.exec_cmd);
891         ucmd->user_cmd.cmd_h = ucmd->h;
892         ucmd->user_cmd.subcode = SCST_USER_EXEC;
893         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
894         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb, cmd->cdb_len);
895         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
896         ucmd->user_cmd.exec_cmd.ext_cdb_len = cmd->ext_cdb_len;
897         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
898         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
899         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
900         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
901                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
902                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
903         }
904         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
905         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
906         ucmd->user_cmd.exec_cmd.partial = 0;
907         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
908         ucmd->user_cmd.exec_cmd.p_in_buf = ucmd->ubuff +
909                                                 (cmd->sg_cnt << PAGE_SHIFT);
910         ucmd->user_cmd.exec_cmd.in_bufflen = cmd->in_bufflen;
911         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
912
913         ucmd->state = UCMD_STATE_EXECING;
914
915         dev_user_add_to_ready(ucmd);
916
917         TRACE_EXIT_RES(res);
918         return res;
919 }
920
921 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
922 {
923         if (ucmd->sgv != NULL) {
924                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
925                 ucmd->sgv = NULL;
926         } else if (ucmd->data_pages != NULL) {
927                 /* We mapped pages, but for some reason didn't allocate them */
928                 ucmd_get(ucmd);
929                 __dev_user_free_sg_entries(ucmd);
930         }
931         return;
932 }
933
934 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
935 {
936         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
937
938         TRACE_ENTRY();
939
940         if (unlikely(ucmd == NULL))
941                 goto out;
942
943         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
944                 ucmd->buff_cached, ucmd->ubuff);
945
946         ucmd->cmd = NULL;
947         if (cmd->data_direction == SCST_DATA_WRITE && ucmd->buf_ucmd != NULL)
948                 ucmd->buf_ucmd->buf_dirty = 1;
949
950         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
951                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
952                 /* The state assignment must be before freeing sgv! */
953                 goto out_reply;
954         }
955
956         if (unlikely(!ucmd->seen_by_user)) {
957                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
958                 goto out_reply;
959         }
960
961         ucmd->user_cmd_payload_len =
962                 offsetof(struct scst_user_get_cmd, on_free_cmd) +
963                 sizeof(ucmd->user_cmd.on_free_cmd);
964         ucmd->user_cmd.cmd_h = ucmd->h;
965         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
966         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
967         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
968         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
969         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
970         ucmd->user_cmd.on_free_cmd.status = cmd->status;
971         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
972
973         ucmd->state = UCMD_STATE_ON_FREEING;
974
975         dev_user_add_to_ready(ucmd);
976
977 out:
978         TRACE_EXIT();
979         return;
980
981 out_reply:
982         dev_user_process_reply_on_free(ucmd);
983         goto out;
984 }
985
986 static void dev_user_set_block(struct scst_cmd *cmd, int block)
987 {
988         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
989         /*
990          * No need for locks here, since *_detach() can not be
991          * called, when there are existing commands.
992          */
993         TRACE_DBG("dev %p, new block %d", dev, block);
994         if (block != 0)
995                 dev->block = block;
996         else
997                 dev->block = dev->def_block;
998         return;
999 }
1000
1001 static int dev_user_disk_done(struct scst_cmd *cmd)
1002 {
1003         int res = SCST_CMD_STATE_DEFAULT;
1004
1005         TRACE_ENTRY();
1006
1007         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
1008
1009         TRACE_EXIT_RES(res);
1010         return res;
1011 }
1012
1013 static int dev_user_tape_done(struct scst_cmd *cmd)
1014 {
1015         int res = SCST_CMD_STATE_DEFAULT;
1016
1017         TRACE_ENTRY();
1018
1019         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
1020
1021         TRACE_EXIT_RES(res);
1022         return res;
1023 }
1024
1025 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
1026 {
1027         struct scst_user_dev *dev = ucmd->dev;
1028         unsigned long flags;
1029         int do_wake = in_interrupt();
1030
1031         TRACE_ENTRY();
1032
1033         if (ucmd->cmd)
1034                 do_wake |= ucmd->cmd->preprocessing_only;
1035
1036         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1037
1038         ucmd->this_state_unjammed = 0;
1039
1040         if ((ucmd->state == UCMD_STATE_PARSING) ||
1041             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1042                 /*
1043                  * If we don't put such commands in the queue head, then under
1044                  * high load we might delay threads, waiting for memory
1045                  * allocations, for too long and start loosing NOPs, which
1046                  * would lead to consider us by remote initiators as
1047                  * unresponsive and stuck => broken connections, etc. If none
1048                  * of our commands completed in NOP timeout to allow the head
1049                  * commands to go, then we are really overloaded and/or stuck.
1050                  */
1051                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1052                         "cmd list", ucmd, ucmd->state);
1053                 list_add(&ucmd->ready_cmd_list_entry,
1054                         &dev->ready_cmd_list);
1055         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1056                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1057                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1058                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1059                         "ready cmd list", ucmd, ucmd->state);
1060                 list_add(&ucmd->ready_cmd_list_entry,
1061                         &dev->ready_cmd_list);
1062                 do_wake = 1;
1063         } else {
1064                 if ((ucmd->cmd != NULL) &&
1065                     unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1066                         TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list",
1067                                 ucmd);
1068                         list_add(&ucmd->ready_cmd_list_entry,
1069                                 &dev->ready_cmd_list);
1070                 } else {
1071                         TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1072                         list_add_tail(&ucmd->ready_cmd_list_entry,
1073                                       &dev->ready_cmd_list);
1074                 }
1075                 do_wake |= ((ucmd->state == UCMD_STATE_ON_CACHE_FREEING) ||
1076                             (ucmd->state == UCMD_STATE_ON_FREEING));
1077         }
1078
1079         if (do_wake) {
1080                 TRACE_DBG("Waking up dev %p", dev);
1081                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1082         }
1083
1084         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1085
1086         TRACE_EXIT();
1087         return;
1088 }
1089
1090 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1091         int num_pg)
1092 {
1093         int res = 0, rc;
1094         int i;
1095         struct task_struct *tsk = current;
1096
1097         TRACE_ENTRY();
1098
1099         if (unlikely(ubuff == 0))
1100                 goto out_nomem;
1101
1102         sBUG_ON(ucmd->data_pages != NULL);
1103
1104         ucmd->num_data_pages = num_pg;
1105
1106         ucmd->data_pages =
1107                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1108                           GFP_KERNEL);
1109         if (ucmd->data_pages == NULL) {
1110                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1111                         "(num_data_pages=%d)", ucmd->num_data_pages);
1112                 res = -ENOMEM;
1113                 goto out_nomem;
1114         }
1115
1116         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1117                 " first_page_offset %d, len %d)", ucmd, ubuff,
1118                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1119                 ucmd->cmd->bufflen);
1120
1121         down_read(&tsk->mm->mmap_sem);
1122         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1123                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1124         up_read(&tsk->mm->mmap_sem);
1125
1126         /* get_user_pages() flushes dcache */
1127
1128         if (rc < ucmd->num_data_pages)
1129                 goto out_unmap;
1130
1131         ucmd->ubuff = ubuff;
1132         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1133
1134 out:
1135         TRACE_EXIT_RES(res);
1136         return res;
1137
1138 out_nomem:
1139         scst_set_busy(ucmd->cmd);
1140         /* go through */
1141
1142 out_err:
1143         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1144         goto out;
1145
1146 out_unmap:
1147         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1148                 ucmd->num_data_pages, rc);
1149         if (rc > 0) {
1150                 for (i = 0; i < rc; i++)
1151                         page_cache_release(ucmd->data_pages[i]);
1152         }
1153         kfree(ucmd->data_pages);
1154         ucmd->data_pages = NULL;
1155         res = -EFAULT;
1156         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1157         goto out_err;
1158 }
1159
1160 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1161         struct scst_user_reply_cmd *reply)
1162 {
1163         int res = 0;
1164         struct scst_cmd *cmd = ucmd->cmd;
1165
1166         TRACE_ENTRY();
1167
1168         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1169
1170         if (likely(reply->alloc_reply.pbuf != 0)) {
1171                 int pages;
1172                 if (ucmd->buff_cached) {
1173                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1174                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1175                                         "page aligned",
1176                                         reply->alloc_reply.pbuf);
1177                                 goto out_hwerr;
1178                         }
1179                         pages = cmd->sg_cnt;
1180                 } else
1181                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1182                                             cmd->bufflen);
1183                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1184         } else {
1185                 scst_set_busy(ucmd->cmd);
1186                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1187         }
1188
1189 out_process:
1190         scst_process_active_cmd(cmd, false);
1191
1192         TRACE_EXIT_RES(res);
1193         return res;
1194
1195 out_hwerr:
1196         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1197         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1198         res = -EINVAL;
1199         goto out_process;
1200 }
1201
1202 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1203         struct scst_user_reply_cmd *reply)
1204 {
1205         int res = 0;
1206         struct scst_user_scsi_cmd_reply_parse *preply =
1207                 &reply->parse_reply;
1208         struct scst_cmd *cmd = ucmd->cmd;
1209
1210         TRACE_ENTRY();
1211
1212         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1213                 goto out_inval;
1214
1215         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1216                      (preply->data_direction != SCST_DATA_READ) &&
1217                      (preply->data_direction != SCST_DATA_NONE)))
1218                 goto out_inval;
1219
1220         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1221                      (preply->bufflen == 0)))
1222                 goto out_inval;
1223
1224         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1225                 goto out_inval;
1226
1227         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1228                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1229                 preply->data_direction, preply->bufflen, preply->data_len,
1230                 reply->alloc_reply.pbuf);
1231
1232         cmd->queue_type = preply->queue_type;
1233         cmd->data_direction = preply->data_direction;
1234         cmd->bufflen = preply->bufflen;
1235         cmd->data_len = preply->data_len;
1236         if (preply->write_medium)
1237                 cmd->op_flags |= SCST_WRITE_MEDIUM;
1238
1239 out_process:
1240         scst_process_active_cmd(cmd, false);
1241
1242         TRACE_EXIT_RES(res);
1243         return res;
1244
1245 out_inval:
1246         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1247                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1248         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1249         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1250         scst_set_cmd_abnormal_done_state(cmd);
1251         res = -EINVAL;
1252         goto out_process;
1253 }
1254
1255 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1256 {
1257         int res = 0;
1258
1259         TRACE_ENTRY();
1260
1261         TRACE_DBG("ON FREE ucmd %p", ucmd);
1262
1263         dev_user_free_sgv(ucmd);
1264         ucmd_put(ucmd);
1265
1266         TRACE_EXIT_RES(res);
1267         return res;
1268 }
1269
1270 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1271 {
1272         int res = 0;
1273
1274         TRACE_ENTRY();
1275
1276         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1277
1278         ucmd_put(ucmd);
1279
1280         TRACE_EXIT_RES(res);
1281         return res;
1282 }
1283
1284 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1285         struct scst_user_reply_cmd *reply)
1286 {
1287         int res = 0;
1288         struct scst_user_scsi_cmd_reply_exec *ereply =
1289                 &reply->exec_reply;
1290         struct scst_cmd *cmd = ucmd->cmd;
1291
1292         TRACE_ENTRY();
1293
1294         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1295                 if (ucmd->background_exec) {
1296                         TRACE_DBG("Background ucmd %p finished", ucmd);
1297                         ucmd_put(ucmd);
1298                         goto out;
1299                 }
1300                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1301                         goto out_inval;
1302                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1303                              (ereply->resp_data_len != 0)))
1304                         goto out_inval;
1305         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1306                 if (unlikely(ucmd->background_exec))
1307                         goto out_inval;
1308                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1309                              (cmd->resp_data_len != 0)))
1310                         goto out_inval;
1311                 /*
1312                  * background_exec assignment must be after ucmd get.
1313                  * Otherwise, due to reorder, in dev_user_process_reply()
1314                  * it is possible that ucmd is destroyed before it "got" here.
1315                  */
1316                 ucmd_get_ordered(ucmd);
1317                 ucmd->background_exec = 1;
1318                 TRACE_DBG("Background ucmd %p", ucmd);
1319                 goto out_compl;
1320         } else
1321                 goto out_inval;
1322
1323         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1324                 ereply->status, ereply->resp_data_len);
1325
1326          if (ereply->resp_data_len != 0) {
1327                 if (ucmd->ubuff == 0) {
1328                         int pages, rc;
1329                         if (unlikely(ereply->pbuf == 0))
1330                                 goto out_busy;
1331                         if (ucmd->buff_cached) {
1332                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1333                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1334                                                 "page aligned", ereply->pbuf);
1335                                         goto out_hwerr;
1336                                 }
1337                                 pages = cmd->sg_cnt;
1338                         } else
1339                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1340                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1341                         if ((rc != 0) || (ucmd->ubuff == 0))
1342                                 goto out_compl;
1343
1344                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1345                         if (unlikely(rc != 0))
1346                                 goto out_busy;
1347                 } else
1348                         dev_user_flush_dcache(ucmd);
1349                 cmd->may_need_dma_sync = 1;
1350                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1351         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1352                 if (ucmd->ubuff == 0)
1353                         cmd->resp_data_len = ereply->resp_data_len;
1354                 else
1355                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1356         }
1357
1358         cmd->status = ereply->status;
1359         if (ereply->sense_len != 0) {
1360                 res = scst_alloc_sense(cmd, 0);
1361                 if (res != 0)
1362                         goto out_compl;
1363                 res = copy_from_user(cmd->sense,
1364                         (void __user *)(unsigned long)ereply->psense_buffer,
1365                         min((int)cmd->sense_bufflen, (int)ereply->sense_len));
1366                 if (res < 0) {
1367                         PRINT_ERROR("%s", "Unable to get sense data");
1368                         goto out_hwerr_res_set;
1369                 }
1370         }
1371
1372 out_compl:
1373         cmd->completed = 1;
1374         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1375         /* !! At this point cmd can be already freed !! */
1376
1377 out:
1378         TRACE_EXIT_RES(res);
1379         return res;
1380
1381 out_inval:
1382         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1383                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1384         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1385
1386 out_hwerr:
1387         res = -EINVAL;
1388
1389 out_hwerr_res_set:
1390         if (ucmd->background_exec) {
1391                 ucmd_put(ucmd);
1392                 goto out;
1393         } else {
1394                 scst_set_cmd_error(cmd,
1395                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1396                 goto out_compl;
1397         }
1398
1399 out_busy:
1400         scst_set_busy(cmd);
1401         goto out_compl;
1402 }
1403
1404 static int dev_user_process_reply(struct scst_user_dev *dev,
1405         struct scst_user_reply_cmd *reply)
1406 {
1407         int res = 0;
1408         struct scst_user_cmd *ucmd;
1409         int state;
1410
1411         TRACE_ENTRY();
1412
1413         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1414
1415         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1416         if (unlikely(ucmd == NULL)) {
1417                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1418                 res = -ESRCH;
1419                 goto out_unlock;
1420         }
1421
1422         if (unlikely(ucmd_get_check(ucmd))) {
1423                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1424                 res = -ESRCH;
1425                 goto out_unlock;
1426         }
1427
1428         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1429         smp_mb();
1430         if (ucmd->background_exec) {
1431                 state = UCMD_STATE_EXECING;
1432                 goto unlock_process;
1433         }
1434
1435         if (unlikely(ucmd->this_state_unjammed)) {
1436                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1437                         ucmd);
1438                 goto out_unlock_put;
1439         }
1440
1441         if (unlikely(!ucmd->sent_to_user)) {
1442                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1443                         "state %x", ucmd, ucmd->state);
1444                 res = -EINVAL;
1445                 goto out_unlock_put;
1446         }
1447
1448         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1449                 goto out_wrong_state;
1450
1451         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1452                 goto out_wrong_state;
1453
1454         state = ucmd->state;
1455         ucmd->sent_to_user = 0;
1456
1457 unlock_process:
1458         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1459
1460         switch (state) {
1461         case UCMD_STATE_PARSING:
1462                 res = dev_user_process_reply_parse(ucmd, reply);
1463                 break;
1464
1465         case UCMD_STATE_BUF_ALLOCING:
1466                 res = dev_user_process_reply_alloc(ucmd, reply);
1467                 break;
1468
1469         case UCMD_STATE_EXECING:
1470                 res = dev_user_process_reply_exec(ucmd, reply);
1471                 break;
1472
1473         case UCMD_STATE_ON_FREEING:
1474                 res = dev_user_process_reply_on_free(ucmd);
1475                 break;
1476
1477         case UCMD_STATE_ON_CACHE_FREEING:
1478                 res = dev_user_process_reply_on_cache_free(ucmd);
1479                 break;
1480
1481         case UCMD_STATE_TM_EXECING:
1482                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1483                 break;
1484
1485         case UCMD_STATE_ATTACH_SESS:
1486         case UCMD_STATE_DETACH_SESS:
1487                 res = dev_user_process_reply_sess(ucmd, reply->result);
1488                 break;
1489
1490         default:
1491                 sBUG();
1492                 break;
1493         }
1494
1495 out_put:
1496         ucmd_put(ucmd);
1497
1498 out:
1499         TRACE_EXIT_RES(res);
1500         return res;
1501
1502 out_wrong_state:
1503         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1504                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1505                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1506                 reply->subcode, ucmd->user_cmd.subcode);
1507         res = -EINVAL;
1508         dev_user_unjam_cmd(ucmd, 0, NULL);
1509
1510 out_unlock_put:
1511         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1512         goto out_put;
1513
1514 out_unlock:
1515         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1516         goto out;
1517 }
1518
1519 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1520 {
1521         int res = 0;
1522         struct scst_user_dev *dev;
1523         struct scst_user_reply_cmd reply;
1524
1525         TRACE_ENTRY();
1526
1527         mutex_lock(&dev_priv_mutex);
1528         dev = (struct scst_user_dev *)file->private_data;
1529         res = dev_user_check_reg(dev);
1530         if (unlikely(res != 0)) {
1531                 mutex_unlock(&dev_priv_mutex);
1532                 goto out;
1533         }
1534         down_read(&dev->dev_rwsem);
1535         mutex_unlock(&dev_priv_mutex);
1536
1537         res = copy_from_user(&reply, arg, sizeof(reply));
1538         if (unlikely(res < 0))
1539                 goto out_up;
1540
1541         TRACE_DBG("Reply for dev %s", dev->name);
1542
1543         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1544
1545         res = dev_user_process_reply(dev, &reply);
1546         if (unlikely(res < 0))
1547                 goto out_up;
1548
1549 out_up:
1550         up_read(&dev->dev_rwsem);
1551
1552 out:
1553         TRACE_EXIT_RES(res);
1554         return res;
1555 }
1556
1557 static int dev_user_get_ext_cdb(struct file *file, void __user *arg)
1558 {
1559         int res = 0;
1560         struct scst_user_dev *dev;
1561         struct scst_user_cmd *ucmd;
1562         struct scst_cmd *cmd;
1563         struct scst_user_get_ext_cdb get;
1564
1565         TRACE_ENTRY();
1566
1567         mutex_lock(&dev_priv_mutex);
1568         dev = (struct scst_user_dev *)file->private_data;
1569         res = dev_user_check_reg(dev);
1570         if (unlikely(res != 0)) {
1571                 mutex_unlock(&dev_priv_mutex);
1572                 goto out;
1573         }
1574         down_read(&dev->dev_rwsem);
1575         mutex_unlock(&dev_priv_mutex);
1576
1577         res = copy_from_user(&get, arg, sizeof(get));
1578         if (unlikely(res < 0))
1579                 goto out_up;
1580
1581         TRACE_MGMT_DBG("Get ext cdb for dev %s", dev->name);
1582
1583         TRACE_BUFFER("Get ext cdb", &get, sizeof(get));
1584
1585         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1586
1587         ucmd = __ucmd_find_hash(dev, get.cmd_h);
1588         if (unlikely(ucmd == NULL)) {
1589                 TRACE_MGMT_DBG("cmd_h %d not found", get.cmd_h);
1590                 res = -ESRCH;
1591                 goto out_unlock;
1592         }
1593
1594         if (unlikely(ucmd_get_check(ucmd))) {
1595                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", get.cmd_h);
1596                 res = -ESRCH;
1597                 goto out_unlock;
1598         }
1599
1600         if ((ucmd->cmd != NULL) && (ucmd->state <= UCMD_STATE_EXECING) &&
1601             (ucmd->sent_to_user || ucmd->background_exec)) {
1602                 cmd = ucmd->cmd;
1603                 scst_cmd_get(cmd);
1604         } else {
1605                 TRACE_MGMT_DBG("Invalid ucmd state %d for cmd_h %d",
1606                         ucmd->state, get.cmd_h);
1607                 res = -EINVAL;
1608                 goto out_put;
1609         }
1610
1611         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1612
1613         if (cmd == NULL)
1614                 goto out_put;
1615
1616         if (cmd->ext_cdb == NULL)
1617                 goto out_cmd_put;
1618
1619         TRACE_BUFFER("EXT CDB", cmd->ext_cdb, cmd->ext_cdb_len);
1620         res = copy_to_user((void __user *)(unsigned long)get.ext_cdb_buffer,
1621                 cmd->ext_cdb, cmd->ext_cdb_len);
1622
1623 out_cmd_put:
1624         scst_cmd_put(cmd);
1625
1626 out_put:
1627         ucmd_put(ucmd);
1628
1629 out_up:
1630         up_read(&dev->dev_rwsem);
1631
1632 out:
1633         TRACE_EXIT_RES(res);
1634         return res;
1635
1636 out_unlock:
1637         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1638         goto out_up;
1639 }
1640
1641 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1642         __releases(&dev->cmd_lists.cmd_list_lock)
1643         __acquires(&dev->cmd_lists.cmd_list_lock)
1644 {
1645         int res = 0;
1646
1647         TRACE_ENTRY();
1648
1649         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1650                 struct scst_cmd *cmd = list_entry(
1651                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1652                         cmd_list_entry);
1653                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1654                 list_del(&cmd->cmd_list_entry);
1655                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1656                 scst_process_active_cmd(cmd, false);
1657                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1658                 res++;
1659         }
1660
1661         TRACE_EXIT_RES(res);
1662         return res;
1663 }
1664
1665 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1666 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1667         __releases(&dev->cmd_lists.cmd_list_lock)
1668         __acquires(&dev->cmd_lists.cmd_list_lock)
1669 {
1670         struct scst_user_cmd *u;
1671
1672 again:
1673         u = NULL;
1674         if (!list_empty(cmd_list)) {
1675                 u = list_entry(cmd_list->next, typeof(*u),
1676                                ready_cmd_list_entry);
1677
1678                 TRACE_DBG("Found ready ucmd %p", u);
1679                 list_del(&u->ready_cmd_list_entry);
1680
1681                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1682
1683                 if (u->cmd != NULL) {
1684                         if (u->state == UCMD_STATE_EXECING) {
1685                                 struct scst_user_dev *dev = u->dev;
1686                                 int rc;
1687
1688                                 EXTRACHECKS_BUG_ON(u->jammed);
1689
1690                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1691
1692                                 rc = scst_check_local_events(u->cmd);
1693                                 if (unlikely(rc != 0)) {
1694                                         u->cmd->scst_cmd_done(u->cmd,
1695                                                 SCST_CMD_STATE_DEFAULT,
1696                                                 SCST_CONTEXT_DIRECT);
1697                                         /*
1698                                          * !! At this point cmd & u can be !!
1699                                          * !! already freed                !!
1700                                          */
1701                                         spin_lock_irq(
1702                                                 &dev->cmd_lists.cmd_list_lock);
1703                                         goto again;
1704                                 }
1705
1706                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1707                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1708                                         &u->cmd->cmd_flags))) {
1709                                 switch (u->state) {
1710                                 case UCMD_STATE_PARSING:
1711                                 case UCMD_STATE_BUF_ALLOCING:
1712                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1713                                         dev_user_unjam_cmd(u, 0, NULL);
1714                                         goto again;
1715                                 case UCMD_STATE_EXECING:
1716                                         EXTRACHECKS_BUG_ON(1);
1717                                 }
1718                         }
1719                 }
1720                 u->sent_to_user = 1;
1721                 u->seen_by_user = 1;
1722         }
1723         return u;
1724 }
1725
1726 static inline int test_cmd_lists(struct scst_user_dev *dev)
1727 {
1728         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1729                   !list_empty(&dev->ready_cmd_list) ||
1730                   !dev->blocking || dev->cleanup_done ||
1731                   signal_pending(current);
1732         return res;
1733 }
1734
1735 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1736 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1737         struct scst_user_cmd **ucmd)
1738 {
1739         int res = 0;
1740         wait_queue_t wait;
1741
1742         TRACE_ENTRY();
1743
1744         init_waitqueue_entry(&wait, current);
1745
1746         while (1) {
1747                 if (!test_cmd_lists(dev)) {
1748                         add_wait_queue_exclusive_head(
1749                                 &dev->cmd_lists.cmd_list_waitQ,
1750                                 &wait);
1751                         for (;;) {
1752                                 set_current_state(TASK_INTERRUPTIBLE);
1753                                 if (test_cmd_lists(dev))
1754                                         break;
1755                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1756                                 schedule();
1757                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1758                         }
1759                         set_current_state(TASK_RUNNING);
1760                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1761                                 &wait);
1762                 }
1763
1764                 dev_user_process_scst_commands(dev);
1765
1766                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1767                 if (*ucmd != NULL)
1768                         break;
1769
1770                 if (!dev->blocking || dev->cleanup_done) {
1771                         res = -EAGAIN;
1772                         TRACE_DBG("No ready commands, returning %d", res);
1773                         break;
1774                 }
1775
1776                 if (signal_pending(current)) {
1777                         res = -EINTR;
1778                         TRACE_DBG("Signal pending, returning %d", res);
1779                         break;
1780                 }
1781         }
1782
1783         TRACE_EXIT_RES(res);
1784         return res;
1785 }
1786
1787 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1788 {
1789         int res = 0;
1790         struct scst_user_dev *dev;
1791         struct scst_user_get_cmd *cmd;
1792         struct scst_user_reply_cmd *reply;
1793         struct scst_user_cmd *ucmd;
1794         uint64_t ureply;
1795
1796         TRACE_ENTRY();
1797
1798         mutex_lock(&dev_priv_mutex);
1799         dev = (struct scst_user_dev *)file->private_data;
1800         res = dev_user_check_reg(dev);
1801         if (unlikely(res != 0)) {
1802                 mutex_unlock(&dev_priv_mutex);
1803                 goto out;
1804         }
1805         down_read(&dev->dev_rwsem);
1806         mutex_unlock(&dev_priv_mutex);
1807
1808         /* get_user() can't be used with 64-bit values on x86_32 */
1809         res = copy_from_user(&ureply, (uint64_t __user *)
1810                 &((struct scst_user_get_cmd __user *)arg)->preply,
1811                 sizeof(ureply));
1812         if (unlikely(res < 0))
1813                 goto out_up;
1814
1815         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1816                 dev->name);
1817
1818         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1819         if (unlikely(cmd == NULL)) {
1820                 res = -ENOMEM;
1821                 goto out_up;
1822         }
1823
1824         if (ureply != 0) {
1825                 unsigned long u = (unsigned long)ureply;
1826                 reply = (struct scst_user_reply_cmd *)cmd;
1827                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1828                 if (unlikely(res < 0))
1829                         goto out_free;
1830
1831                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1832
1833                 res = dev_user_process_reply(dev, reply);
1834                 if (unlikely(res < 0))
1835                         goto out_free;
1836         }
1837
1838         kmem_cache_free(user_get_cmd_cachep, cmd);
1839
1840         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1841 again:
1842         res = dev_user_get_next_cmd(dev, &ucmd);
1843         if (res == 0) {
1844                 int len;
1845                 /*
1846                  * A misbehaving user space handler can make ucmd to get dead
1847                  * immediately after we released the lock, which can lead to
1848                  * copy of dead data to the user space, which can lead to a
1849                  * leak of sensitive information.
1850                  */
1851                 if (unlikely(ucmd_get_check(ucmd))) {
1852                         /* Oops, this ucmd is already being destroyed. Retry. */
1853                         goto again;
1854                 }
1855                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1856
1857                 EXTRACHECKS_BUG_ON(ucmd->user_cmd_payload_len == 0);
1858
1859                 len = ucmd->user_cmd_payload_len;
1860                 TRACE_DBG("ucmd %p (user_cmd %p), payload_len %d (len %d)",
1861                         ucmd, &ucmd->user_cmd, ucmd->user_cmd_payload_len, len);
1862                 TRACE_BUFFER("UCMD", &ucmd->user_cmd, len);
1863                 res = copy_to_user(arg, &ucmd->user_cmd, len);
1864                 if (unlikely(res != 0)) {
1865                         /* Requeue ucmd back */
1866                         TRACE_DBG("Copy to user failed (%d), requeuing ucmd %p "
1867                                 "back to head of ready cmd list", res, ucmd);
1868                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1869                         list_add(&ucmd->ready_cmd_list_entry,
1870                                 &dev->ready_cmd_list);
1871                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1872                 }
1873 #ifdef CONFIG_SCST_EXTRACHECKS
1874                 else
1875                         ucmd->user_cmd_payload_len = 0;
1876 #endif
1877                 ucmd_put(ucmd);
1878         } else
1879                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1880
1881 out_up:
1882         up_read(&dev->dev_rwsem);
1883
1884 out:
1885         TRACE_EXIT_RES(res);
1886         return res;
1887
1888 out_free:
1889         kmem_cache_free(user_get_cmd_cachep, cmd);
1890         goto out_up;
1891 }
1892
1893 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1894         unsigned long arg)
1895 {
1896         long res;
1897
1898         TRACE_ENTRY();
1899
1900         switch (cmd) {
1901         case SCST_USER_REPLY_AND_GET_CMD:
1902                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1903                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1904                 break;
1905
1906         case SCST_USER_REPLY_CMD:
1907                 TRACE_DBG("%s", "REPLY_CMD");
1908                 res = dev_user_reply_cmd(file, (void __user *)arg);
1909                 break;
1910
1911         case SCST_USER_GET_EXTENDED_CDB:
1912                 TRACE_DBG("%s", "GET_EXTENDED_CDB");
1913                 res = dev_user_get_ext_cdb(file, (void __user *)arg);
1914                 break;
1915
1916         case SCST_USER_REGISTER_DEVICE:
1917         {
1918                 struct scst_user_dev_desc *dev_desc;
1919                 TRACE_DBG("%s", "REGISTER_DEVICE");
1920                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1921                 if (dev_desc == NULL) {
1922                         res = -ENOMEM;
1923                         goto out;
1924                 }
1925                 res = copy_from_user(dev_desc, (void __user *)arg,
1926                                      sizeof(*dev_desc));
1927                 if (res < 0) {
1928                         kfree(dev_desc);
1929                         goto out;
1930                 }
1931                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1932                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1933                 dev_desc->sgv_name[sizeof(dev_desc->sgv_name)-1] = '\0';
1934                 res = dev_user_register_dev(file, dev_desc);
1935                 kfree(dev_desc);
1936                 break;
1937         }
1938
1939         case SCST_USER_UNREGISTER_DEVICE:
1940                 TRACE_DBG("%s", "UNREGISTER_DEVICE");
1941                 res = dev_user_unregister_dev(file);
1942                 break;
1943
1944         case SCST_USER_FLUSH_CACHE:
1945                 TRACE_DBG("%s", "FLUSH_CACHE");
1946                 res = dev_user_flush_cache(file);
1947                 break;
1948
1949         case SCST_USER_SET_OPTIONS:
1950         {
1951                 struct scst_user_opt opt;
1952                 TRACE_DBG("%s", "SET_OPTIONS");
1953                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1954                 if (res < 0)
1955                         goto out;
1956                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1957                 res = dev_user_set_opt(file, &opt);
1958                 break;
1959         }
1960
1961         case SCST_USER_GET_OPTIONS:
1962                 TRACE_DBG("%s", "GET_OPTIONS");
1963                 res = dev_user_get_opt(file, (void __user *)arg);
1964                 break;
1965
1966         case SCST_USER_DEVICE_CAPACITY_CHANGED:
1967                 TRACE_DBG("%s", "CAPACITY_CHANGED");
1968                 res = dev_user_capacity_changed(file);
1969                 break;
1970
1971         default:
1972                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1973                 res = -EINVAL;
1974                 goto out;
1975         }
1976
1977 out:
1978         TRACE_EXIT_RES(res);
1979         return res;
1980 }
1981
1982 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1983 {
1984         int res = 0;
1985         struct scst_user_dev *dev;
1986
1987         TRACE_ENTRY();
1988
1989         mutex_lock(&dev_priv_mutex);
1990         dev = (struct scst_user_dev *)file->private_data;
1991         res = dev_user_check_reg(dev);
1992         if (unlikely(res != 0)) {
1993                 mutex_unlock(&dev_priv_mutex);
1994                 goto out;
1995         }
1996         down_read(&dev->dev_rwsem);
1997         mutex_unlock(&dev_priv_mutex);
1998
1999         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2000
2001         if (!list_empty(&dev->ready_cmd_list) ||
2002             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2003                 res |= POLLIN | POLLRDNORM;
2004                 goto out_unlock;
2005         }
2006
2007         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2008
2009         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
2010         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
2011         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
2012
2013         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2014
2015         if (!list_empty(&dev->ready_cmd_list) ||
2016             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2017                 res |= POLLIN | POLLRDNORM;
2018                 goto out_unlock;
2019         }
2020
2021 out_unlock:
2022         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2023
2024         up_read(&dev->dev_rwsem);
2025
2026 out:
2027         TRACE_EXIT_HRES(res);
2028         return res;
2029 }
2030
2031 /*
2032  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
2033  */
2034 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
2035         unsigned long *flags)
2036         __releases(&dev->cmd_lists.cmd_list_lock)
2037         __acquires(&dev->cmd_lists.cmd_list_lock)
2038 {
2039         int state = ucmd->state;
2040         struct scst_user_dev *dev = ucmd->dev;
2041
2042         TRACE_ENTRY();
2043
2044         if (ucmd->this_state_unjammed)
2045                 goto out;
2046
2047         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
2048                 state);
2049
2050         ucmd->jammed = 1;
2051         ucmd->this_state_unjammed = 1;
2052         ucmd->sent_to_user = 0;
2053
2054         switch (state) {
2055         case UCMD_STATE_PARSING:
2056         case UCMD_STATE_BUF_ALLOCING:
2057                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2058                         ucmd->aborted = 1;
2059                 else {
2060                         if (busy)
2061                                 scst_set_busy(ucmd->cmd);
2062                         else
2063                                 scst_set_cmd_error(ucmd->cmd,
2064                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2065                 }
2066                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
2067
2068                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
2069                 list_add(&ucmd->cmd->cmd_list_entry,
2070                         &ucmd->cmd->cmd_lists->active_cmd_list);
2071                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
2072                 break;
2073
2074         case UCMD_STATE_EXECING:
2075                 if (flags != NULL)
2076                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2077                                                *flags);
2078                 else
2079                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2080
2081                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
2082
2083                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2084                         ucmd->aborted = 1;
2085                 else {
2086                         if (busy)
2087                                 scst_set_busy(ucmd->cmd);
2088                         else
2089                                 scst_set_cmd_error(ucmd->cmd,
2090                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2091                 }
2092
2093                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
2094                         SCST_CONTEXT_DIRECT);
2095                 /* !! At this point cmd and ucmd can be already freed !! */
2096
2097                 if (flags != NULL)
2098                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2099                                           *flags);
2100                 else
2101                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2102                 break;
2103
2104         case UCMD_STATE_ON_FREEING:
2105         case UCMD_STATE_ON_CACHE_FREEING:
2106         case UCMD_STATE_TM_EXECING:
2107         case UCMD_STATE_ATTACH_SESS:
2108         case UCMD_STATE_DETACH_SESS:
2109                 if (flags != NULL)
2110                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2111                                                *flags);
2112                 else
2113                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2114
2115                 switch (state) {
2116                 case UCMD_STATE_ON_FREEING:
2117                         dev_user_process_reply_on_free(ucmd);
2118                         break;
2119
2120                 case UCMD_STATE_ON_CACHE_FREEING:
2121                         dev_user_process_reply_on_cache_free(ucmd);
2122                         break;
2123
2124                 case UCMD_STATE_TM_EXECING:
2125                         dev_user_process_reply_tm_exec(ucmd,
2126                                                        SCST_MGMT_STATUS_FAILED);
2127                         break;
2128
2129                 case UCMD_STATE_ATTACH_SESS:
2130                 case UCMD_STATE_DETACH_SESS:
2131                         dev_user_process_reply_sess(ucmd, -EFAULT);
2132                         break;
2133                 }
2134
2135                 if (flags != NULL)
2136                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2137                                           *flags);
2138                 else
2139                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2140                 break;
2141
2142         default:
2143                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
2144                 sBUG();
2145                 break;
2146         }
2147
2148 out:
2149         TRACE_EXIT();
2150         return;
2151 }
2152
2153 static int dev_user_unjam_dev(struct scst_user_dev *dev)
2154         __releases(&dev->cmd_lists.cmd_list_lock)
2155         __acquires(&dev->cmd_lists.cmd_list_lock)
2156 {
2157         int i, res = 0;
2158         struct scst_user_cmd *ucmd;
2159
2160         TRACE_ENTRY();
2161
2162         TRACE_MGMT_DBG("Unjamming dev %p", dev);
2163
2164         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2165
2166 repeat:
2167         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2168                 struct list_head *head = &dev->ucmd_hash[i];
2169
2170                 list_for_each_entry(ucmd, head, hash_list_entry) {
2171                         res++;
2172
2173                         if (!ucmd->sent_to_user)
2174                                 continue;
2175
2176                         if (ucmd_get_check(ucmd))
2177                                 continue;
2178
2179                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2180                                 ucmd->state, ucmd->cmd);
2181
2182                         dev_user_unjam_cmd(ucmd, 0, NULL);
2183
2184                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2185                         ucmd_put(ucmd);
2186                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2187
2188                         goto repeat;
2189                 }
2190         }
2191
2192         if (dev_user_process_scst_commands(dev) != 0)
2193                 goto repeat;
2194
2195         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2196
2197         TRACE_EXIT_RES(res);
2198         return res;
2199 }
2200
2201 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2202         int status)
2203 {
2204         int res = 0;
2205
2206         TRACE_ENTRY();
2207
2208         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2209                 ucmd->user_cmd.tm_cmd.fn, status);
2210
2211         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2212                 /*
2213                  * It is possible that user space seen TM cmd before cmd
2214                  * to abort or will never see it at all, because it was
2215                  * aborted on the way there. So, it is safe to return
2216                  * success instead, because, if there is the TM cmd at this
2217                  * point, then the cmd to abort apparrently does exist.
2218                  */
2219                 status = SCST_MGMT_STATUS_SUCCESS;
2220         }
2221
2222         scst_async_mcmd_completed(ucmd->mcmd, status);
2223
2224         ucmd_put(ucmd);
2225
2226         TRACE_EXIT_RES(res);
2227         return res;
2228 }
2229
2230 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2231 {
2232         struct scst_user_cmd *ucmd;
2233         unsigned long flags;
2234
2235         TRACE_ENTRY();
2236
2237         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2238 again:
2239         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2240                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2241                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2242                         switch (ucmd->state) {
2243                         case UCMD_STATE_PARSING:
2244                         case UCMD_STATE_BUF_ALLOCING:
2245                         case UCMD_STATE_EXECING:
2246                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2247                                 list_del(&ucmd->ready_cmd_list_entry);
2248                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2249                                 goto again;
2250                         }
2251                 }
2252         }
2253
2254         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2255
2256         TRACE_EXIT();
2257         return;
2258 }
2259
2260 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2261         struct scst_tgt_dev *tgt_dev)
2262 {
2263         struct scst_user_cmd *ucmd;
2264         struct scst_user_dev *dev =
2265                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2266         struct scst_user_cmd *ucmd_to_abort = NULL;
2267
2268         TRACE_ENTRY();
2269
2270         /*
2271          * In the used approach we don't do anything with hung devices, which
2272          * stopped responding and/or have stuck commands. We forcedly abort such
2273          * commands only if they not yet sent to the user space or if the device
2274          * is getting unloaded, e.g. if its handler program gets killed. This is
2275          * because it's pretty hard to distinguish between stuck and temporary
2276          * overloaded states of the device. There are several reasons for that:
2277          *
2278          * 1. Some commands need a lot of time to complete (several hours),
2279          *    so for an impatient user such command(s) will always look as
2280          *    stuck.
2281          *
2282          * 2. If we forcedly abort, i.e. abort before it's actually completed
2283          *    in the user space, just one command, we will have to put the whole
2284          *    device offline until we are sure that no more previously aborted
2285          *    commands will get executed. Otherwise, we might have a possibility
2286          *    for data corruption, when aborted and reported as completed
2287          *    command actually gets executed *after* new commands sent
2288          *    after the force abort was done. Many journaling file systems and
2289          *    databases use "provide required commands order via queue draining"
2290          *    approach and not putting the whole device offline after the forced
2291          *    abort will break it. This makes our decision, if a command stuck
2292          *    or not, cost a lot.
2293          *
2294          * So, we leave policy definition if a device stuck or not to
2295          * the user space and simply let all commands live until they are
2296          * completed or their devices get closed/killed. This approach is very
2297          * much OK, but can affect management commands, which need activity
2298          * suspending via scst_suspend_activity() function such as devices or
2299          * targets registration/removal. But during normal life such commands
2300          * should be rare. Plus, when possible, scst_suspend_activity() will
2301          * return after timeout EBUSY status to allow caller to not stuck
2302          * forever as well.
2303          *
2304          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2305          * stuck commands would affect only related devices.
2306          */
2307
2308         dev_user_abort_ready_commands(dev);
2309
2310         /* We can't afford missing TM command due to memory shortage */
2311         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2312
2313         ucmd->user_cmd_payload_len =
2314                 offsetof(struct scst_user_get_cmd, tm_cmd) +
2315                 sizeof(ucmd->user_cmd.tm_cmd);
2316         ucmd->user_cmd.cmd_h = ucmd->h;
2317         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2318         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2319         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2320         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2321         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2322
2323         if (mcmd->cmd_to_abort != NULL) {
2324                 ucmd_to_abort =
2325                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2326                 if (ucmd_to_abort != NULL)
2327                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2328         }
2329
2330         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2331                 "ucmd_to_abort %p, cmd_h_to_abort %d, mcmd %p)", ucmd, ucmd->h,
2332                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2333                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort, mcmd);
2334
2335         ucmd->mcmd = mcmd;
2336         ucmd->state = UCMD_STATE_TM_EXECING;
2337
2338         scst_prepare_async_mcmd(mcmd);
2339
2340         dev_user_add_to_ready(ucmd);
2341
2342         TRACE_EXIT();
2343         return SCST_DEV_TM_NOT_COMPLETED;
2344 }
2345
2346 static int dev_user_attach(struct scst_device *sdev)
2347 {
2348         int res = 0;
2349         struct scst_user_dev *dev = NULL, *d;
2350
2351         TRACE_ENTRY();
2352
2353         spin_lock(&dev_list_lock);
2354         list_for_each_entry(d, &dev_list, dev_list_entry) {
2355                 if (strcmp(d->name, sdev->virt_name) == 0) {
2356                         dev = d;
2357                         break;
2358                 }
2359         }
2360         spin_unlock(&dev_list_lock);
2361         if (dev == NULL) {
2362                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2363                 res = -EINVAL;
2364                 goto out;
2365         }
2366
2367         sdev->p_cmd_lists = &dev->cmd_lists;
2368         sdev->dh_priv = dev;
2369         sdev->tst = dev->tst;
2370         sdev->queue_alg = dev->queue_alg;
2371         sdev->swp = dev->swp;
2372         sdev->tas = dev->tas;
2373         sdev->d_sense = dev->d_sense;
2374         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2375
2376         dev->sdev = sdev;
2377
2378         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2379                 dev->name);
2380
2381 out:
2382         TRACE_EXIT();
2383         return res;
2384 }
2385
2386 static void dev_user_detach(struct scst_device *sdev)
2387 {
2388         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2389
2390         TRACE_ENTRY();
2391
2392         TRACE_DBG("virt_id %d", sdev->virt_id);
2393
2394         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2395                 dev->name);
2396
2397         /* dev will be freed by the caller */
2398         sdev->dh_priv = NULL;
2399         dev->sdev = NULL;
2400
2401         TRACE_EXIT();
2402         return;
2403 }
2404
2405 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2406 {
2407         int res = 0;
2408         unsigned long flags;
2409
2410         TRACE_ENTRY();
2411
2412         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2413
2414         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2415
2416         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2417                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2418                 ucmd->result = status;
2419         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2420                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2421         } else
2422                 sBUG();
2423
2424         if (ucmd->cmpl != NULL)
2425                 complete_all(ucmd->cmpl);
2426
2427         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2428
2429         ucmd_put(ucmd);
2430
2431         TRACE_EXIT_RES(res);
2432         return res;
2433 }
2434
2435 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2436 {
2437         struct scst_user_dev *dev =
2438                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2439         int res = 0, rc;
2440         struct scst_user_cmd *ucmd;
2441         DECLARE_COMPLETION_ONSTACK(cmpl);
2442
2443         TRACE_ENTRY();
2444
2445         /*
2446          * We can't replace tgt_dev->pool, because it can be used to allocate
2447          * memory for SCST local commands, like REPORT LUNS, where there is no
2448          * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
2449          */
2450         if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
2451                 tgt_dev->dh_priv = dev->pool_clust;
2452         else
2453                 tgt_dev->dh_priv = dev->pool;
2454
2455         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2456         if (ucmd == NULL)
2457                 goto out_nomem;
2458
2459         ucmd->cmpl = &cmpl;
2460
2461         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2462                 sizeof(ucmd->user_cmd.sess);
2463         ucmd->user_cmd.cmd_h = ucmd->h;
2464         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2465         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2466         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2467         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2468         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only;
2469         strncpy(ucmd->user_cmd.sess.initiator_name,
2470                 tgt_dev->sess->initiator_name,
2471                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2472         ucmd->user_cmd.sess.initiator_name[
2473                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2474         if (tgt_dev->sess->tgt->default_group_name != NULL) {
2475                 strncpy(ucmd->user_cmd.sess.target_name,
2476                         &tgt_dev->sess->tgt->default_group_name[sizeof(SCST_DEFAULT_ACG_NAME)],
2477                         sizeof(ucmd->user_cmd.sess.target_name)-1);
2478                 ucmd->user_cmd.sess.target_name[
2479                         sizeof(ucmd->user_cmd.sess.target_name)-1] = '\0';
2480         }
2481
2482         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2483                 "threads_num %d, rd_only %d, initiator %s, target %s)",
2484                 ucmd, ucmd->h, ucmd->user_cmd.sess.sess_h,
2485                 ucmd->user_cmd.sess.lun, ucmd->user_cmd.sess.threads_num,
2486                 ucmd->user_cmd.sess.rd_only, ucmd->user_cmd.sess.initiator_name,
2487                 ucmd->user_cmd.sess.target_name);
2488
2489         ucmd->state = UCMD_STATE_ATTACH_SESS;
2490
2491         ucmd_get(ucmd);
2492
2493         dev_user_add_to_ready(ucmd);
2494
2495         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2496         if (rc > 0)
2497                 res = ucmd->result;
2498         else {
2499                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2500                 res = -EFAULT;
2501         }
2502
2503         sBUG_ON(irqs_disabled());
2504
2505         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2506         ucmd->cmpl = NULL;
2507         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2508
2509         ucmd_put(ucmd);
2510
2511 out:
2512         TRACE_EXIT_RES(res);
2513         return res;
2514
2515 out_nomem:
2516         res = -ENOMEM;
2517         goto out;
2518 }
2519
2520 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2521 {
2522         struct scst_user_dev *dev =
2523                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2524         struct scst_user_cmd *ucmd;
2525
2526         TRACE_ENTRY();
2527
2528         /*
2529          * We can't miss TM command due to memory shortage, because it might
2530          * lead to a memory leak in the user space handler.
2531          */
2532         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2533         if (ucmd == NULL)
2534                 goto out;
2535
2536         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2537                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2538
2539         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2540                 sizeof(ucmd->user_cmd.sess);
2541         ucmd->user_cmd.cmd_h = ucmd->h;
2542         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2543         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2544
2545         ucmd->state = UCMD_STATE_DETACH_SESS;
2546
2547         dev_user_add_to_ready(ucmd);
2548
2549 out:
2550         TRACE_EXIT();
2551         return;
2552 }
2553
2554 /* No locks are needed, but the activity must be suspended */
2555 static void dev_user_setup_functions(struct scst_user_dev *dev)
2556 {
2557         TRACE_ENTRY();
2558
2559         dev->devtype.parse = dev_user_parse;
2560         dev->devtype.dev_done = NULL;
2561
2562         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2563                 switch (dev->devtype.type) {
2564                 case TYPE_DISK:
2565                         dev->generic_parse = scst_sbc_generic_parse;
2566                         dev->devtype.dev_done = dev_user_disk_done;
2567                         break;
2568
2569                 case TYPE_TAPE:
2570                         dev->generic_parse = scst_tape_generic_parse;
2571                         dev->devtype.dev_done = dev_user_tape_done;
2572                         break;
2573
2574                 case TYPE_MOD:
2575                         dev->generic_parse = scst_modisk_generic_parse;
2576                         dev->devtype.dev_done = dev_user_disk_done;
2577                         break;
2578
2579                 case TYPE_ROM:
2580                         dev->generic_parse = scst_cdrom_generic_parse;
2581                         dev->devtype.dev_done = dev_user_disk_done;
2582                         break;
2583
2584                 case TYPE_MEDIUM_CHANGER:
2585                         dev->generic_parse = scst_changer_generic_parse;
2586                         break;
2587
2588                 case TYPE_PROCESSOR:
2589                         dev->generic_parse = scst_processor_generic_parse;
2590                         break;
2591
2592                 case TYPE_RAID:
2593                         dev->generic_parse = scst_raid_generic_parse;
2594                         break;
2595
2596                 default:
2597                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2598                                 "for it", dev->devtype.type);
2599                         dev->parse_type = SCST_USER_PARSE_CALL;
2600                         break;
2601                 }
2602         } else {
2603                 dev->generic_parse = NULL;
2604                 dev->devtype.dev_done = NULL;
2605         }
2606
2607         TRACE_EXIT();
2608         return;
2609 }
2610
2611 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2612 {
2613         char ver[sizeof(DEV_USER_VERSION)+1];
2614         int res;
2615
2616         res = copy_from_user(ver,
2617                         (void __user *)(unsigned long)dev_desc->version_str,
2618                         sizeof(ver));
2619         if (res < 0) {
2620                 PRINT_ERROR("%s", "Unable to get version string");
2621                 goto out;
2622         }
2623         ver[sizeof(ver)-1] = '\0';
2624
2625         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2626                 /* ->name already 0-terminated in dev_user_ioctl() */
2627                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2628                         dev_desc->name, ver);
2629                 res = -EINVAL;
2630                 goto out;
2631         }
2632
2633 out:
2634         return res;
2635 }
2636
2637 static int dev_user_register_dev(struct file *file,
2638         const struct scst_user_dev_desc *dev_desc)
2639 {
2640         int res = -ENOMEM, i;
2641         struct scst_user_dev *dev, *d;
2642         int block;
2643
2644         TRACE_ENTRY();
2645
2646         res = dev_user_check_version(dev_desc);
2647         if (res != 0)
2648                 goto out;
2649
2650         switch (dev_desc->type) {
2651         case TYPE_DISK:
2652         case TYPE_ROM:
2653         case TYPE_MOD:
2654                 if (dev_desc->block_size == 0) {
2655                         PRINT_ERROR("Wrong block size %d",
2656                                     dev_desc->block_size);
2657                         res = -EINVAL;
2658                         goto out;
2659                 }
2660                 block = scst_calc_block_shift(dev_desc->block_size);
2661                 if (block == -1) {
2662                         res = -EINVAL;
2663                         goto out;
2664                 }
2665                 break;
2666         default:
2667                 block = dev_desc->block_size;
2668                 break;
2669         }
2670
2671         if (!try_module_get(THIS_MODULE)) {
2672                 PRINT_ERROR("%s", "Fail to get module");
2673                 goto out;
2674         }
2675
2676         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2677         if (dev == NULL)
2678                 goto out_put;
2679
2680         init_rwsem(&dev->dev_rwsem);
2681         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2682         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2683         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2684         INIT_LIST_HEAD(&dev->ready_cmd_list);
2685         if (file->f_flags & O_NONBLOCK) {
2686                 TRACE_DBG("%s", "Non-blocking operations");
2687                 dev->blocking = 0;
2688         } else
2689                 dev->blocking = 1;
2690         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2691                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2692
2693         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2694         dev->name[sizeof(dev->name)-1] = '\0';
2695
2696         scst_init_mem_lim(&dev->udev_mem_lim);
2697
2698         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2699                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2700                                                   dev_desc->sgv_name);
2701         dev->pool = sgv_pool_create(dev->devtype.name, sgv_no_clustering,
2702                                         dev_desc->sgv_shared);
2703         if (dev->pool == NULL)
2704                 goto out_free_dev;
2705         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2706                 dev_user_free_sg_entries);
2707
2708         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust",
2709                 (dev_desc->sgv_name[0] == '\0') ? dev->name :
2710                                                   dev_desc->sgv_name);
2711         dev->pool_clust = sgv_pool_create(dev->devtype.name,
2712                                 sgv_tail_clustering, dev_desc->sgv_shared);
2713         if (dev->pool_clust == NULL)
2714                 goto out_free0;
2715         sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
2716                 dev_user_free_sg_entries);
2717
2718         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s",
2719                 dev->name);
2720         dev->devtype.type = dev_desc->type;
2721         dev->devtype.threads_num = -1;
2722         dev->devtype.parse_atomic = 1;
2723         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2724         dev->devtype.dev_done_atomic = 1;
2725         dev->devtype.no_proc = 1;
2726         dev->devtype.attach = dev_user_attach;
2727         dev->devtype.detach = dev_user_detach;
2728         dev->devtype.attach_tgt = dev_user_attach_tgt;
2729         dev->devtype.detach_tgt = dev_user_detach_tgt;
2730         dev->devtype.exec = dev_user_exec;
2731         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2732         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2733
2734         init_completion(&dev->cleanup_cmpl);
2735         dev->block = block;
2736         dev->def_block = block;
2737
2738         res = __dev_user_set_opt(dev, &dev_desc->opt);
2739
2740         TRACE_MEM("dev %p, name %s", dev, dev->name);
2741
2742         spin_lock(&dev_list_lock);
2743
2744         list_for_each_entry(d, &dev_list, dev_list_entry) {
2745                 if (strcmp(d->name, dev->name) == 0) {
2746                         PRINT_ERROR("Device %s already exist",
2747                                 dev->name);
2748                         res = -EEXIST;
2749                         spin_unlock(&dev_list_lock);
2750                         goto out_free;
2751                 }
2752         }
2753
2754         list_add_tail(&dev->dev_list_entry, &dev_list);
2755
2756         spin_unlock(&dev_list_lock);
2757
2758         if (res != 0)
2759                 goto out_del_free;
2760
2761         res = scst_register_virtual_dev_driver(&dev->devtype);
2762         if (res < 0)
2763                 goto out_del_free;
2764
2765         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2766         if (dev->virt_id < 0) {
2767                 res = dev->virt_id;
2768                 goto out_unreg_handler;
2769         }
2770
2771         mutex_lock(&dev_priv_mutex);
2772         if (file->private_data != NULL) {
2773                 mutex_unlock(&dev_priv_mutex);
2774                 PRINT_ERROR("%s", "Device already registered");
2775                 res = -EINVAL;
2776                 goto out_unreg_drv;
2777         }
2778         file->private_data = dev;
2779         mutex_unlock(&dev_priv_mutex);
2780
2781 out:
2782         TRACE_EXIT_RES(res);
2783         return res;
2784
2785 out_unreg_drv:
2786         scst_unregister_virtual_device(dev->virt_id);
2787
2788 out_unreg_handler:
2789         scst_unregister_virtual_dev_driver(&dev->devtype);
2790
2791 out_del_free:
2792         spin_lock(&dev_list_lock);
2793         list_del(&dev->dev_list_entry);
2794         spin_unlock(&dev_list_lock);
2795
2796 out_free:
2797         sgv_pool_del(dev->pool_clust);
2798
2799 out_free0:
2800         sgv_pool_del(dev->pool);
2801
2802 out_free_dev:
2803         kfree(dev);
2804
2805 out_put:
2806         module_put(THIS_MODULE);
2807         goto out;
2808 }
2809
2810 static int dev_user_unregister_dev(struct file *file)
2811 {
2812         int res;
2813         struct scst_user_dev *dev;
2814
2815         TRACE_ENTRY();
2816
2817         mutex_lock(&dev_priv_mutex);
2818         dev = (struct scst_user_dev *)file->private_data;
2819         res = dev_user_check_reg(dev);
2820         if (res != 0) {
2821                 mutex_unlock(&dev_priv_mutex);
2822                 goto out;
2823         }
2824         down_read(&dev->dev_rwsem);
2825         mutex_unlock(&dev_priv_mutex);
2826
2827         res = scst_suspend_activity(true);
2828         if (res != 0)
2829                 goto out_up;
2830
2831         up_read(&dev->dev_rwsem);
2832
2833         dev_user_release(NULL, file);
2834
2835         scst_resume_activity();
2836
2837 out:
2838         TRACE_EXIT_RES(res);
2839         return res;
2840
2841 out_up:
2842         up_read(&dev->dev_rwsem);
2843         goto out;
2844 }
2845
2846 static int dev_user_flush_cache(struct file *file)
2847 {
2848         int res;
2849         struct scst_user_dev *dev;
2850
2851         TRACE_ENTRY();
2852
2853         mutex_lock(&dev_priv_mutex);
2854         dev = (struct scst_user_dev *)file->private_data;
2855         res = dev_user_check_reg(dev);
2856         if (res != 0) {
2857                 mutex_unlock(&dev_priv_mutex);
2858                 goto out;
2859         }
2860         down_read(&dev->dev_rwsem);
2861         mutex_unlock(&dev_priv_mutex);
2862
2863         res = scst_suspend_activity(true);
2864         if (res != 0)
2865                 goto out_up;
2866
2867         sgv_pool_flush(dev->pool);
2868         sgv_pool_flush(dev->pool_clust);
2869
2870         scst_resume_activity();
2871
2872 out_up:
2873         up_read(&dev->dev_rwsem);
2874
2875 out:
2876         TRACE_EXIT_RES(res);
2877         return res;
2878 }
2879
2880 static int dev_user_capacity_changed(struct file *file)
2881 {
2882         int res;
2883         struct scst_user_dev *dev;
2884
2885         TRACE_ENTRY();
2886
2887         mutex_lock(&dev_priv_mutex);
2888         dev = (struct scst_user_dev *)file->private_data;
2889         res = dev_user_check_reg(dev);
2890         if (res != 0) {
2891                 mutex_unlock(&dev_priv_mutex);
2892                 goto out;
2893         }
2894         down_read(&dev->dev_rwsem);
2895         mutex_unlock(&dev_priv_mutex);
2896
2897         scst_capacity_data_changed(dev->sdev);
2898
2899         up_read(&dev->dev_rwsem);
2900
2901 out:
2902         TRACE_EXIT_RES(res);
2903         return res;
2904 }
2905
2906
2907 static int __dev_user_set_opt(struct scst_user_dev *dev,
2908         const struct scst_user_opt *opt)
2909 {
2910         int res = 0;
2911
2912         TRACE_ENTRY();
2913
2914         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2915                 "memory_reuse_type %x, partial_transfers_type %x, "
2916                 "partial_len %d", dev->name, opt->parse_type,
2917                 opt->on_free_cmd_type, opt->memory_reuse_type,
2918                 opt->partial_transfers_type, opt->partial_len);
2919
2920         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
2921             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
2922             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
2923             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
2924                 PRINT_ERROR("%s", "Invalid option");
2925                 res = -EINVAL;
2926                 goto out;
2927         }
2928
2929         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2930              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2931             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2932              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2933             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) ||
2934             (opt->d_sense > 1)) {
2935                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
2936                         " tas %x, d_sense %d, has_own_order_mgmt %x)", opt->tst,
2937                         opt->queue_alg, opt->swp, opt->tas, opt->d_sense,
2938                         opt->has_own_order_mgmt);
2939                 res = -EINVAL;
2940                 goto out;
2941         }
2942
2943         dev->parse_type = opt->parse_type;
2944         dev->on_free_cmd_type = opt->on_free_cmd_type;
2945         dev->memory_reuse_type = opt->memory_reuse_type;
2946         dev->partial_transfers_type = opt->partial_transfers_type;
2947         dev->partial_len = opt->partial_len;
2948
2949         dev->tst = opt->tst;
2950         dev->queue_alg = opt->queue_alg;
2951         dev->swp = opt->swp;
2952         dev->tas = opt->tas;
2953         dev->tst = opt->tst;
2954         dev->d_sense = opt->d_sense;
2955         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2956         if (dev->sdev != NULL) {
2957                 dev->sdev->tst = opt->tst;
2958                 dev->sdev->queue_alg = opt->queue_alg;
2959                 dev->sdev->swp = opt->swp;
2960                 dev->sdev->tas = opt->tas;
2961                 dev->sdev->d_sense = opt->d_sense;
2962                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2963         }
2964
2965         dev_user_setup_functions(dev);
2966
2967 out:
2968         TRACE_EXIT_RES(res);
2969         return res;
2970 }
2971
2972 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2973 {
2974         int res;
2975         struct scst_user_dev *dev;
2976
2977         TRACE_ENTRY();
2978
2979         mutex_lock(&dev_priv_mutex);
2980         dev = (struct scst_user_dev *)file->private_data;
2981         res = dev_user_check_reg(dev);
2982         if (res != 0) {
2983                 mutex_unlock(&dev_priv_mutex);
2984                 goto out;
2985         }
2986         down_read(&dev->dev_rwsem);
2987         mutex_unlock(&dev_priv_mutex);
2988
2989         res = scst_suspend_activity(true);
2990         if (res != 0)
2991                 goto out;
2992
2993         res = __dev_user_set_opt(dev, opt);
2994
2995         scst_resume_activity();
2996
2997         up_read(&dev->dev_rwsem);
2998
2999 out:
3000         TRACE_EXIT_RES(res);
3001         return res;
3002 }
3003
3004 static int dev_user_get_opt(struct file *file, void __user *arg)
3005 {
3006         int res;
3007         struct scst_user_dev *dev;
3008         struct scst_user_opt opt;
3009
3010         TRACE_ENTRY();
3011
3012         mutex_lock(&dev_priv_mutex);
3013         dev = (struct scst_user_dev *)file->private_data;
3014         res = dev_user_check_reg(dev);
3015         if (res != 0) {
3016                 mutex_unlock(&dev_priv_mutex);
3017                 goto out;
3018         }
3019         down_read(&dev->dev_rwsem);
3020         mutex_unlock(&dev_priv_mutex);
3021
3022         opt.parse_type = dev->parse_type;
3023         opt.on_free_cmd_type = dev->on_free_cmd_type;
3024         opt.memory_reuse_type = dev->memory_reuse_type;
3025         opt.partial_transfers_type = dev->partial_transfers_type;
3026         opt.partial_len = dev->partial_len;
3027         opt.tst = dev->tst;
3028         opt.queue_alg = dev->queue_alg;
3029         opt.tas = dev->tas;
3030         opt.swp = dev->swp;
3031         opt.d_sense = dev->d_sense;
3032         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
3033
3034         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3035                 "memory_reuse_type %x, partial_transfers_type %x, "
3036                 "partial_len %d", dev->name, opt.parse_type,
3037                 opt.on_free_cmd_type, opt.memory_reuse_type,
3038                 opt.partial_transfers_type, opt.partial_len);
3039
3040         res = copy_to_user(arg, &opt, sizeof(opt));
3041
3042         up_read(&dev->dev_rwsem);
3043 out:
3044         TRACE_EXIT_RES(res);
3045         return res;
3046 }
3047
3048 static int dev_usr_parse(struct scst_cmd *cmd)
3049 {
3050         sBUG();
3051         return SCST_CMD_STATE_DEFAULT;
3052 }
3053
3054 /* Needed only for /proc support */
3055 #define USR_TYPE {                      \
3056         .name =         DEV_USER_NAME,  \
3057         .type =         -1,             \
3058         .parse =        dev_usr_parse,  \
3059 }
3060
3061 static struct scst_dev_type dev_user_devtype = USR_TYPE;
3062
3063 static int dev_user_release(struct inode *inode, struct file *file)
3064 {
3065         int res = 0;
3066         struct scst_user_dev *dev;
3067
3068         TRACE_ENTRY();
3069
3070         mutex_lock(&dev_priv_mutex);
3071         dev = (struct scst_user_dev *)file->private_data;
3072         if (dev == NULL) {
3073                 mutex_unlock(&dev_priv_mutex);
3074                 goto out;
3075         }
3076         file->private_data = NULL;
3077
3078         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
3079
3080         spin_lock(&dev_list_lock);
3081         list_del(&dev->dev_list_entry);
3082         spin_unlock(&dev_list_lock);
3083
3084         dev->blocking = 0;
3085         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
3086
3087         down_write(&dev->dev_rwsem);
3088         mutex_unlock(&dev_priv_mutex);
3089
3090         spin_lock(&cleanup_lock);
3091         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
3092         spin_unlock(&cleanup_lock);
3093
3094         wake_up(&cleanup_list_waitQ);
3095
3096         scst_unregister_virtual_device(dev->virt_id);
3097         scst_unregister_virtual_dev_driver(&dev->devtype);
3098
3099         sgv_pool_flush(dev->pool_clust);
3100         sgv_pool_flush(dev->pool);
3101
3102         TRACE_DBG("Unregistering finished (dev %p)", dev);
3103
3104         dev->cleanup_done = 1;
3105
3106         wake_up(&cleanup_list_waitQ);
3107         wake_up(&dev->cmd_lists.cmd_list_waitQ);
3108
3109         wait_for_completion(&dev->cleanup_cmpl);
3110
3111         sgv_pool_del(dev->pool_clust);
3112         sgv_pool_del(dev->pool);
3113
3114         up_write(&dev->dev_rwsem); /* to make the debug check happy */
3115
3116         TRACE_DBG("Releasing completed (dev %p)", dev);
3117
3118         kfree(dev);
3119
3120         module_put(THIS_MODULE);
3121
3122 out:
3123         TRACE_EXIT_RES(res);
3124         return res;
3125 }
3126
3127 static int dev_user_process_cleanup(struct scst_user_dev *dev)
3128 {
3129         struct scst_user_cmd *ucmd;
3130         int rc = 0, res = 1;
3131
3132         TRACE_ENTRY();
3133
3134         sBUG_ON(dev->blocking);
3135         wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
3136
3137         while (1) {
3138                 int rc1;
3139
3140                 TRACE_DBG("Cleanuping dev %p", dev);
3141
3142                 rc1 = dev_user_unjam_dev(dev);
3143                 if ((rc1 == 0) && (rc == -EAGAIN) && dev->cleanup_done)
3144                         break;
3145
3146                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
3147
3148                 rc = dev_user_get_next_cmd(dev, &ucmd);
3149                 if (rc == 0)
3150                         dev_user_unjam_cmd(ucmd, 1, NULL);
3151
3152                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
3153
3154                 if (rc == -EAGAIN) {
3155                         if (!dev->cleanup_done) {
3156                                 TRACE_DBG("No more commands (dev %p)", dev);
3157                                 goto out;
3158                         }
3159                 }
3160         }
3161
3162 #ifdef CONFIG_SCST_EXTRACHECKS
3163 {
3164         int i;
3165         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3166                 struct list_head *head = &dev->ucmd_hash[i];
3167                 struct scst_user_cmd *ucmd2;
3168 again:
3169                 list_for_each_entry(ucmd2, head, hash_list_entry) {
3170                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
3171                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
3172                         ucmd_put(ucmd2);
3173                         goto again;
3174                 }
3175         }
3176 }
3177 #endif
3178
3179         TRACE_DBG("Cleanuping done (dev %p)", dev);
3180         complete_all(&dev->cleanup_cmpl);
3181         res = 0;
3182
3183 out:
3184         TRACE_EXIT_RES(res);
3185         return res;
3186 }
3187
3188 static inline int test_cleanup_list(void)
3189 {
3190         int res = !list_empty(&cleanup_list) ||
3191                   unlikely(kthread_should_stop());
3192         return res;
3193 }
3194
3195 static int dev_user_cleanup_thread(void *arg)
3196 {
3197         TRACE_ENTRY();
3198
3199         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
3200
3201         current->flags |= PF_NOFREEZE;
3202
3203         spin_lock(&cleanup_lock);
3204         while (!kthread_should_stop()) {
3205                 wait_queue_t wait;
3206                 init_waitqueue_entry(&wait, current);
3207
3208                 if (!test_cleanup_list()) {
3209                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
3210                         for (;;) {
3211                                 set_current_state(TASK_INTERRUPTIBLE);
3212                                 if (test_cleanup_list())
3213                                         break;
3214                                 spin_unlock(&cleanup_lock);
3215                                 schedule();
3216                                 spin_lock(&cleanup_lock);
3217                         }
3218                         set_current_state(TASK_RUNNING);
3219                         remove_wait_queue(&cleanup_list_waitQ, &wait);
3220                 }
3221
3222                 /*
3223                  * We have to poll devices, because commands can go from SCST
3224                  * core on cmd_list_waitQ and we have no practical way to
3225                  * detect them.
3226                  */
3227
3228                 while (1) {
3229                         struct scst_user_dev *dev;
3230                         LIST_HEAD(cl_devs);
3231
3232                         while (!list_empty(&cleanup_list)) {
3233                                 int rc;
3234
3235                                 dev = list_entry(cleanup_list.next,
3236                                         typeof(*dev), cleanup_list_entry);
3237                                 list_del(&dev->cleanup_list_entry);
3238
3239                                 spin_unlock(&cleanup_lock);
3240                                 rc = dev_user_process_cleanup(dev);
3241                                 spin_lock(&cleanup_lock);
3242
3243                                 if (rc != 0)
3244                                         list_add_tail(&dev->cleanup_list_entry,
3245                                                 &cl_devs);
3246                         }
3247
3248                         if (list_empty(&cl_devs))
3249                                 break;
3250
3251                         spin_unlock(&cleanup_lock);
3252                         msleep(100);
3253                         spin_lock(&cleanup_lock);
3254
3255                         while (!list_empty(&cl_devs)) {
3256                                 dev = list_entry(cl_devs.next, typeof(*dev),
3257                                         cleanup_list_entry);
3258                                 list_move_tail(&dev->cleanup_list_entry,
3259                                         &cleanup_list);
3260                         }
3261                 }
3262         }
3263         spin_unlock(&cleanup_lock);
3264
3265         /*
3266          * If kthread_should_stop() is true, we are guaranteed to be
3267          * on the module unload, so cleanup_list must be empty.
3268          */
3269         sBUG_ON(!list_empty(&cleanup_list));
3270
3271         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
3272
3273         TRACE_EXIT();
3274         return 0;
3275 }
3276
3277 static int __init init_scst_user(void)
3278 {
3279         int res = 0;
3280         struct max_get_reply {
3281                 union {
3282                         struct scst_user_get_cmd g;
3283                         struct scst_user_reply_cmd r;
3284                 };
3285         };
3286 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3287         struct class_device *class_member;
3288 #else
3289         struct device *dev;
3290 #endif
3291
3292         TRACE_ENTRY();
3293
3294 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
3295         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
3296                 "Consider changing VMSPLIT option or use a 64-bit "
3297                 "configuration instead. See README file for details.");
3298         res = -EINVAL;
3299         goto out;
3300 #endif
3301
3302         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
3303         if (user_cmd_cachep == NULL) {
3304                 res = -ENOMEM;
3305                 goto out;
3306         }
3307
3308         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
3309         if (user_get_cmd_cachep == NULL) {
3310                 res = -ENOMEM;
3311                 goto out_cache;
3312         }
3313
3314         dev_user_devtype.module = THIS_MODULE;
3315
3316         res = scst_register_virtual_dev_driver(&dev_user_devtype);
3317         if (res < 0)
3318                 goto out_cache1;
3319
3320         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
3321         if (res != 0)
3322                 goto out_unreg;
3323
3324         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
3325         if (IS_ERR(dev_user_sysfs_class)) {
3326                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
3327                         "space handler");
3328                 res = PTR_ERR(dev_user_sysfs_class);
3329                 goto out_proc;
3330         }
3331
3332         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
3333         if (res) {
3334                 PRINT_ERROR("Unable to get major %d for SCSI tapes",
3335                             DEV_USER_MAJOR);
3336                 goto out_class;
3337         }
3338
3339 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3340         class_member = class_device_create(dev_user_sysfs_class, NULL,
3341                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
3342         if (IS_ERR(class_member)) {
3343                 res = PTR_ERR(class_member);
3344                 goto out_chrdev;
3345         }
3346 #else
3347         dev = device_create(dev_user_sysfs_class, NULL,
3348                             MKDEV(DEV_USER_MAJOR, 0),
3349 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
3350                                 NULL,
3351 #endif
3352                                 DEV_USER_NAME);
3353         if (IS_ERR(dev)) {
3354                 res = PTR_ERR(dev);
3355                 goto out_chrdev;
3356         }
3357 #endif
3358
3359         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
3360                 "scst_usr_cleanupd");
3361         if (IS_ERR(cleanup_thread)) {
3362                 res = PTR_ERR(cleanup_thread);
3363                 PRINT_ERROR("kthread_create() failed: %d", res);
3364                 goto out_dev;
3365         }
3366
3367 out:
3368         TRACE_EXIT_RES(res);
3369         return res;
3370
3371 out_dev:
3372 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3373         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3374 #else
3375         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3376 #endif
3377
3378 out_chrdev:
3379         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3380
3381 out_class:
3382         class_destroy(dev_user_sysfs_class);
3383
3384 out_proc:
3385         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3386
3387 out_unreg:
3388         scst_unregister_dev_driver(&dev_user_devtype);
3389
3390 out_cache1:
3391         kmem_cache_destroy(user_get_cmd_cachep);
3392
3393 out_cache:
3394         kmem_cache_destroy(user_cmd_cachep);
3395         goto out;
3396 }
3397
3398 static void __exit exit_scst_user(void)
3399 {
3400         int rc;
3401
3402         TRACE_ENTRY();
3403
3404         rc = kthread_stop(cleanup_thread);
3405         if (rc < 0)
3406                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3407
3408         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3409 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3410         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3411 #else
3412         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3413 #endif
3414         class_destroy(dev_user_sysfs_class);
3415
3416         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3417         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3418
3419         kmem_cache_destroy(user_get_cmd_cachep);
3420         kmem_cache_destroy(user_cmd_cachep);
3421
3422         TRACE_EXIT();
3423         return;
3424 }
3425
3426 module_init(init_scst_user);
3427 module_exit(exit_scst_user);
3428
3429 MODULE_AUTHOR("Vladislav Bolkhovitin");
3430 MODULE_LICENSE("GPL");
3431 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3432 MODULE_VERSION(SCST_VERSION_STRING);
3433 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);