SRPT 1.0.0 tag added
[mirror/scst/.git] / scst / src / dev_handlers / scst_user.c
1 /*
2  *  scst_user.c
3  *
4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
6  *
7  *  SCSI virtual user space device handler
8  *
9  *  This program is free software; you can redistribute it and/or
10  *  modify it under the terms of the GNU General Public License
11  *  as published by the Free Software Foundation, version 2
12  *  of the License.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  *  GNU General Public License for more details.
18  */
19
20 #include <linux/kthread.h>
21 #include <linux/delay.h>
22 #include <linux/poll.h>
23
24 #define LOG_PREFIX              DEV_USER_NAME
25
26 #include "scst.h"
27 #include "scst_user.h"
28 #include "scst_dev_handler.h"
29
30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
31 #warning "HIGHMEM kernel configurations are not supported by this module, \
32         because nowadays it doesn't worth the effort. Consider change \
33         VMSPLIT option or use 64-bit configuration instead. See README file \
34         for details."
35 #endif
36
37 #define DEV_USER_MAJOR                  237
38
39 #define DEV_USER_CMD_HASH_ORDER         6
40
41 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
42
43 struct scst_user_dev {
44         struct rw_semaphore dev_rwsem;
45
46         struct scst_cmd_lists cmd_lists;
47
48         /* Protected by cmd_lists.cmd_list_lock */
49         struct list_head ready_cmd_list;
50
51         /* Protected by dev_rwsem or don't need any protection */
52         unsigned int blocking:1;
53         unsigned int cleanup_done:1;
54         unsigned int tst:3;
55         unsigned int queue_alg:4;
56         unsigned int tas:1;
57         unsigned int swp:1;
58         unsigned int has_own_order_mgmt:1;
59
60         int (*generic_parse)(struct scst_cmd *cmd,
61                 int (*get_block)(struct scst_cmd *cmd));
62
63         int block;
64         int def_block;
65
66         struct scst_mem_lim udev_mem_lim;
67         struct sgv_pool *pool;
68
69         uint8_t parse_type;
70         uint8_t on_free_cmd_type;
71         uint8_t memory_reuse_type;
72         uint8_t partial_transfers_type;
73         uint32_t partial_len;
74
75         struct scst_dev_type devtype;
76
77         /* Both protected by cmd_lists.cmd_list_lock */
78         unsigned int handle_counter;
79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
80
81         struct scst_device *sdev;
82
83         int virt_id;
84         struct list_head dev_list_entry;
85         char name[SCST_MAX_NAME];
86
87         struct list_head cleanup_list_entry;
88         /* ToDo: make it on-stack */
89         struct completion cleanup_cmpl;
90 };
91
92 /* Most fields are unprotected, since only one thread at time can access them */
93 struct scst_user_cmd {
94         struct scst_cmd *cmd;
95         struct scst_user_dev *dev;
96
97         atomic_t ucmd_ref;
98
99         unsigned int buff_cached:1;
100         unsigned int buf_dirty:1;
101         unsigned int background_exec:1;
102         unsigned int aborted:1;
103
104         struct scst_user_cmd *buf_ucmd;
105
106         int cur_data_page;
107         int num_data_pages;
108         int first_page_offset;
109         unsigned long ubuff;
110         struct page **data_pages;
111         struct sgv_pool_obj *sgv;
112
113         /*
114          * Special flags, which can be accessed asynchronously (hence "long").
115          * Protected by cmd_lists.cmd_list_lock.
116          */
117         unsigned long sent_to_user:1;
118         unsigned long jammed:1;
119         unsigned long this_state_unjammed:1;
120         unsigned long seen_by_user:1; /* here only as a small optimization */
121
122         unsigned int state;
123
124         struct list_head ready_cmd_list_entry;
125
126         unsigned int h;
127         struct list_head hash_list_entry;
128
129         struct scst_user_get_cmd user_cmd;
130
131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
132         union {
133                 struct completion *cmpl;
134                 struct scst_mgmt_cmd *mcmd;
135         };
136         int result;
137 };
138
139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
140         gfp_t gfp_mask);
141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
142
143 static int dev_user_parse(struct scst_cmd *cmd);
144 static int dev_user_exec(struct scst_cmd *cmd);
145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
147         struct scst_tgt_dev *tgt_dev);
148
149 static int dev_user_disk_done(struct scst_cmd *cmd);
150 static int dev_user_tape_done(struct scst_cmd *cmd);
151
152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
153         gfp_t gfp_mask, void *priv);
154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
155                                      void *priv);
156
157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
158
159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
160         unsigned long *flags);
161
162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
164         int status);
165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
166 static int dev_user_register_dev(struct file *file,
167         const struct scst_user_dev_desc *dev_desc);
168 static int __dev_user_set_opt(struct scst_user_dev *dev,
169         const struct scst_user_opt *opt);
170 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
171 static int dev_user_get_opt(struct file *file, void __user *arg);
172
173 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
174 static long dev_user_ioctl(struct file *file, unsigned int cmd,
175         unsigned long arg);
176 static int dev_user_release(struct inode *inode, struct file *file);
177
178 /** Data **/
179
180 static struct kmem_cache *user_cmd_cachep;
181
182 static DEFINE_MUTEX(dev_priv_mutex);
183
184 static struct file_operations dev_user_fops = {
185         .poll           = dev_user_poll,
186         .unlocked_ioctl = dev_user_ioctl,
187 #ifdef CONFIG_COMPAT
188         .compat_ioctl   = dev_user_ioctl,
189 #endif
190         .release        = dev_user_release,
191 };
192
193 static struct class *dev_user_sysfs_class;
194
195 static DEFINE_SPINLOCK(dev_list_lock);
196 static LIST_HEAD(dev_list);
197
198 static DEFINE_SPINLOCK(cleanup_lock);
199 static LIST_HEAD(cleanup_list);
200 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
201 static struct task_struct *cleanup_thread;
202
203 /*
204  * Skip this command if result is not 0. Must be called under
205  * cmd_lists.cmd_list_lock and IRQ off.
206  */
207 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
208 {
209         int r = atomic_inc_return(&ucmd->ucmd_ref);
210         int res;
211         if (unlikely(r == 1)) {
212                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
213                 atomic_dec(&ucmd->ucmd_ref);
214                 res = true;
215                 /*
216                  * Necessary code is serialized by cmd_list_lock in
217                  * cmd_remove_hash()
218                  */
219         } else {
220                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
221                         atomic_read(&ucmd->ucmd_ref));
222                 res = false;
223         }
224         return res;
225 }
226
227 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
228 {
229         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
230         atomic_inc(&ucmd->ucmd_ref);
231         if (barrier)
232                 smp_mb__after_atomic_inc();
233 }
234
235 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
236 {
237         __ucmd_get(ucmd, true);
238 }
239
240 static inline void ucmd_get(struct scst_user_cmd *ucmd)
241 {
242         __ucmd_get(ucmd, false);
243 }
244
245 /* Must not be called under cmd_list_lock!! */
246 static inline void ucmd_put(struct scst_user_cmd *ucmd)
247 {
248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
249
250         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
251
252         if (atomic_dec_and_test(&ucmd->ucmd_ref))
253                 dev_user_free_ucmd(ucmd);
254 }
255
256 static inline int calc_num_pg(unsigned long buf, int len)
257 {
258         len += buf & ~PAGE_MASK;
259         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
260 }
261
262 static inline int is_need_offs_page(unsigned long buf, int len)
263 {
264         return ((buf & ~PAGE_MASK) != 0) &&
265                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
266 }
267
268 static void __dev_user_not_reg(void)
269 {
270         PRINT_ERROR("%s", "Device not registered");
271         return;
272 }
273
274 static inline int dev_user_check_reg(struct scst_user_dev *dev)
275 {
276         if (dev == NULL) {
277                 __dev_user_not_reg();
278                 return -EINVAL;
279         }
280         return 0;
281 }
282
283 static inline int scst_user_cmd_hashfn(int h)
284 {
285         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
286 }
287
288 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
289         unsigned int h)
290 {
291         struct list_head *head;
292         struct scst_user_cmd *ucmd;
293
294         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
295         list_for_each_entry(ucmd, head, hash_list_entry) {
296                 if (ucmd->h == h) {
297                         TRACE_DBG("Found ucmd %p", ucmd);
298                         return ucmd;
299                 }
300         }
301         return NULL;
302 }
303
304 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
305 {
306         struct list_head *head;
307         struct scst_user_dev *dev = ucmd->dev;
308         struct scst_user_cmd *u;
309         unsigned long flags;
310
311         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
312         do {
313                 ucmd->h = dev->handle_counter++;
314                 u = __ucmd_find_hash(dev, ucmd->h);
315         } while (u != NULL);
316         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
317         list_add_tail(&ucmd->hash_list_entry, head);
318         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
319
320         TRACE_DBG("Inserted ucmd %p, h=%d", ucmd, ucmd->h);
321         return;
322 }
323
324 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
325 {
326         unsigned long flags;
327
328         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
329         list_del(&ucmd->hash_list_entry);
330         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
331
332         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
333         return;
334 }
335
336 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
337 {
338         TRACE_ENTRY();
339
340         TRACE_MEM("Freeing ucmd %p", ucmd);
341
342         cmd_remove_hash(ucmd);
343         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
344
345         kmem_cache_free(user_cmd_cachep, ucmd);
346
347         TRACE_EXIT();
348         return;
349 }
350
351 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
352         gfp_t gfp_mask, void *priv)
353 {
354         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
355         int offset = 0;
356
357         TRACE_ENTRY();
358
359         /* *sg supposed to be zeroed */
360
361         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
362                 ucmd->ubuff, ucmd->cur_data_page);
363
364         if (ucmd->cur_data_page == 0) {
365                 TRACE_MEM("ucmd->first_page_offset %d",
366                         ucmd->first_page_offset);
367                 offset = ucmd->first_page_offset;
368                 ucmd_get(ucmd);
369         }
370
371         if (ucmd->cur_data_page >= ucmd->num_data_pages)
372                 goto out;
373
374         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
375                 PAGE_SIZE - offset, offset);
376         ucmd->cur_data_page++;
377
378         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
379                 sg->offset);
380         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
381
382 out:
383         TRACE_EXIT();
384         return sg_page(sg);
385 }
386
387 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
388 {
389         TRACE_ENTRY();
390
391         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
392                 ucmd, ucmd->h, ucmd->ubuff);
393
394         ucmd->user_cmd.cmd_h = ucmd->h;
395         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
396         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
397
398         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
399
400         dev_user_add_to_ready(ucmd);
401
402         TRACE_EXIT();
403         return;
404 }
405
406 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
407 {
408         int i;
409
410         TRACE_ENTRY();
411
412         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
413                 ucmd->ubuff, ucmd->num_data_pages);
414
415         for (i = 0; i < ucmd->num_data_pages; i++) {
416                 struct page *page = ucmd->data_pages[i];
417
418                 if (ucmd->buf_dirty)
419                         SetPageDirty(page);
420
421                 page_cache_release(page);
422         }
423
424         kfree(ucmd->data_pages);
425         ucmd->data_pages = NULL;
426
427         TRACE_EXIT();
428         return;
429 }
430
431 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
432 {
433         TRACE_ENTRY();
434
435         sBUG_ON(ucmd->data_pages == NULL);
436
437         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
438                 ucmd, ucmd->ubuff, ucmd->buff_cached);
439
440         dev_user_unmap_buf(ucmd);
441
442         if (ucmd->buff_cached)
443                 dev_user_on_cached_mem_free(ucmd);
444         else
445                 ucmd_put(ucmd);
446
447         TRACE_EXIT();
448         return;
449 }
450
451 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
452         void *priv)
453 {
454         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
455
456         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
457                 sg_count, ucmd);
458
459         __dev_user_free_sg_entries(ucmd);
460
461         return;
462 }
463
464 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
465 {
466         int mem_reuse_type = ucmd->dev->memory_reuse_type;
467
468         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
469             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
470              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
471             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
472              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
473                 return 1;
474         else
475                 return 0;
476 }
477
478 /*
479  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
480  * Unmaps the buffer, if needed in case of error
481  */
482 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
483 {
484         int res = 0;
485         struct scst_cmd *cmd = ucmd->cmd;
486         struct scst_user_dev *dev = ucmd->dev;
487         gfp_t gfp_mask;
488         int flags = 0;
489         int bufflen = cmd->bufflen;
490         int last_len = 0;
491
492         TRACE_ENTRY();
493
494         sBUG_ON(bufflen == 0);
495
496         gfp_mask = __GFP_NOWARN;
497         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
498
499         if (cached_buff) {
500                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
501                 if (ucmd->ubuff == 0)
502                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
503         } else {
504                 TRACE_MEM("%s", "Not cached buff");
505                 flags |= SCST_POOL_ALLOC_NO_CACHED;
506                 if (ucmd->ubuff == 0) {
507                         res = 1;
508                         goto out;
509                 }
510                 bufflen += ucmd->first_page_offset;
511                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
512                         last_len = bufflen & ~PAGE_MASK;
513                 else
514                         last_len = cmd->bufflen & ~PAGE_MASK;
515                 if (last_len == 0)
516                         last_len = PAGE_SIZE;
517         }
518         ucmd->buff_cached = cached_buff;
519
520         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
521                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
522         if (cmd->sg != NULL) {
523                 struct scst_user_cmd *buf_ucmd =
524                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
525
526                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
527
528                 ucmd->ubuff = buf_ucmd->ubuff;
529                 ucmd->buf_ucmd = buf_ucmd;
530
531                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
532                                    (ucmd != buf_ucmd));
533
534                 if (last_len != 0) {
535                         /* We don't use clustering, so the assignment is safe */
536                         cmd->sg[cmd->sg_cnt-1].length = last_len;
537                 }
538
539                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
540                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
541                         last_len, cmd->sg[cmd->sg_cnt-1].length);
542
543                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
544                         static int ll;
545                         if (ll < 10) {
546                                 PRINT_INFO("Unable to complete command due to "
547                                         "SG IO count limitation (requested %d, "
548                                         "available %d, tgt lim %d)", cmd->sg_cnt,
549                                         cmd->tgt_dev->max_sg_cnt,
550                                         cmd->tgt->sg_tablesize);
551                                 ll++;
552                         }
553                         cmd->sg = NULL;
554                         /* sgv will be freed in dev_user_free_sgv() */
555                         res = -1;
556                 }
557         } else {
558                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
559                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
560                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
561                 if (unlikely(cmd->sg_cnt == 0)) {
562                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
563                         sBUG_ON(ucmd->sgv != NULL);
564                         res = -1;
565                 } else {
566                         switch (ucmd->state) {
567                         case UCMD_STATE_BUF_ALLOCING:
568                                 res = 1;
569                                 break;
570                         case UCMD_STATE_EXECING:
571                                 res = -1;
572                                 break;
573                         default:
574                                 sBUG();
575                                 break;
576                         }
577                 }
578         }
579
580 out:
581         TRACE_EXIT_RES(res);
582         return res;
583 }
584
585 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
586 {
587         int rc, res = SCST_CMD_STATE_DEFAULT;
588         struct scst_cmd *cmd = ucmd->cmd;
589
590         TRACE_ENTRY();
591
592         if (unlikely(ucmd->cmd->data_buf_tgt_alloc)) {
593                 PRINT_ERROR("Target driver %s requested own memory "
594                         "allocation", ucmd->cmd->tgtt->name);
595                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
596                 res = scst_get_cmd_abnormal_done_state(cmd);
597                 goto out;
598         }
599
600         ucmd->state = UCMD_STATE_BUF_ALLOCING;
601         cmd->data_buf_alloced = 1;
602
603         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
604         if (rc == 0)
605                 goto out;
606         else if (rc < 0) {
607                 scst_set_busy(cmd);
608                 res = scst_get_cmd_abnormal_done_state(cmd);
609                 goto out;
610         }
611
612         if ((cmd->data_direction != SCST_DATA_WRITE) &&
613             !scst_is_cmd_local(cmd)) {
614                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
615                 goto out;
616         }
617
618         ucmd->user_cmd.cmd_h = ucmd->h;
619         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
620         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
621         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
622                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
623         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
624         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
625                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
626         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
627         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
628         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
629
630         dev_user_add_to_ready(ucmd);
631
632         res = SCST_CMD_STATE_STOP;
633
634 out:
635         TRACE_EXIT_RES(res);
636         return res;
637 }
638
639 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
640         gfp_t gfp_mask)
641 {
642         struct scst_user_cmd *ucmd = NULL;
643
644         TRACE_ENTRY();
645
646 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
647         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
648         if (ucmd != NULL)
649                 memset(ucmd, 0, sizeof(*ucmd));
650 #else
651         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
652 #endif
653         if (unlikely(ucmd == NULL)) {
654                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
655                         "user cmd (gfp_mask %x)", gfp_mask);
656                 goto out;
657         }
658         ucmd->dev = dev;
659         atomic_set(&ucmd->ucmd_ref, 1);
660
661         cmd_insert_hash(ucmd);
662
663         TRACE_MEM("ucmd %p allocated", ucmd);
664
665 out:
666         TRACE_EXIT_HRES((unsigned long)ucmd);
667         return ucmd;
668 }
669
670 static int dev_user_get_block(struct scst_cmd *cmd)
671 {
672         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
673         /*
674          * No need for locks here, since *_detach() can not be
675          * called, when there are existing commands.
676          */
677         TRACE_EXIT_RES(dev->block);
678         return dev->block;
679 }
680
681 static int dev_user_parse(struct scst_cmd *cmd)
682 {
683         int rc, res = SCST_CMD_STATE_DEFAULT;
684         struct scst_user_cmd *ucmd;
685         int atomic = scst_cmd_atomic(cmd);
686         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
687         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
688
689         TRACE_ENTRY();
690
691         if (cmd->dh_priv == NULL) {
692                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
693                 if (unlikely(ucmd == NULL)) {
694                         if (atomic) {
695                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
696                                 goto out;
697                         } else {
698                                 scst_set_busy(cmd);
699                                 goto out_error;
700                         }
701                 }
702                 ucmd->cmd = cmd;
703                 cmd->dh_priv = ucmd;
704         } else {
705                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
706                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
707         }
708
709         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
710
711         if (ucmd->state != UCMD_STATE_NEW)
712                 goto alloc;
713
714         switch (dev->parse_type) {
715         case SCST_USER_PARSE_STANDARD:
716                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
717                 rc = dev->generic_parse(cmd, dev_user_get_block);
718                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
719                         goto out_invalid;
720                 break;
721
722         case SCST_USER_PARSE_EXCEPTION:
723                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
724                 rc = dev->generic_parse(cmd, dev_user_get_block);
725                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
726                         break;
727                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
728                         TRACE_MEM("Restarting PARSE to thread context "
729                                 "(ucmd %p)", ucmd);
730                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
731                         goto out;
732                 }
733                 /* else go through */
734
735         case SCST_USER_PARSE_CALL:
736                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
737                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
738                 ucmd->user_cmd.cmd_h = ucmd->h;
739                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
740                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
741                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
742                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
743                             sizeof(cmd->cdb)));
744                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
745                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
746                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
747                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
748                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
749                 ucmd->user_cmd.parse_cmd.expected_values_set =
750                                         cmd->expected_values_set;
751                 ucmd->user_cmd.parse_cmd.expected_data_direction =
752                                         cmd->expected_data_direction;
753                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
754                                         cmd->expected_transfer_len;
755                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
756                 ucmd->state = UCMD_STATE_PARSING;
757                 dev_user_add_to_ready(ucmd);
758                 res = SCST_CMD_STATE_STOP;
759                 goto out;
760
761         default:
762                 sBUG();
763                 goto out;
764         }
765
766 alloc:
767         if (cmd->data_direction != SCST_DATA_NONE)
768                 res = dev_user_alloc_space(ucmd);
769
770 out:
771         TRACE_EXIT_RES(res);
772         return res;
773
774 out_invalid:
775         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
776                 cmd->op_flags & SCST_INFO_INVALID);
777         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
778
779 out_error:
780         res = scst_get_cmd_abnormal_done_state(cmd);
781         goto out;
782 }
783
784 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
785 {
786         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
787         unsigned long start = buf_ucmd->ubuff;
788         int i;
789
790         TRACE_ENTRY();
791
792         if (start == 0)
793                 goto out;
794
795         for (i = 0; i < buf_ucmd->num_data_pages; i++) {
796                 struct page *page;
797                 page = buf_ucmd->data_pages[i];
798 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
799                 struct vm_area_struct *vma = find_vma(current->mm, start);
800                 if (vma != NULL)
801                         flush_anon_page(vma, page, start);
802 #endif
803                 flush_dcache_page(page);
804                 start += PAGE_SIZE;
805         }
806
807 out:
808         TRACE_EXIT();
809         return;
810 }
811
812 static int dev_user_exec(struct scst_cmd *cmd)
813 {
814         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
815         int res = SCST_EXEC_COMPLETED;
816
817         TRACE_ENTRY();
818
819 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
820        * context to complete the necessary actions, but all we are going to
821        * do in this function is, in fact, atomic, so let's skip this check.
822        */
823         if (scst_cmd_atomic(cmd)) {
824                 TRACE_DBG("%s", "User exec() can not be called in atomic "
825                         "context, rescheduling to the thread");
826                 res = SCST_EXEC_NEED_THREAD;
827                 goto out;
828         }
829 #endif
830
831         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
832                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
833                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
834
835         if (cmd->data_direction == SCST_DATA_WRITE)
836                 dev_user_flush_dcache(ucmd);
837
838         ucmd->user_cmd.cmd_h = ucmd->h;
839         ucmd->user_cmd.subcode = SCST_USER_EXEC;
840         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
841         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
842                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
843                     sizeof(cmd->cdb)));
844         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
845         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
846         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
847         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
848         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
849                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
850                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
851         }
852         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
853         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
854         ucmd->user_cmd.exec_cmd.partial = 0;
855         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
856         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
857
858         ucmd->state = UCMD_STATE_EXECING;
859
860         dev_user_add_to_ready(ucmd);
861
862         TRACE_EXIT_RES(res);
863         return res;
864 }
865
866 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
867 {
868         if (ucmd->sgv != NULL) {
869                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
870                 ucmd->sgv = NULL;
871         } else if (ucmd->data_pages != NULL) {
872                 /* We mapped pages, but for some reason didn't allocate them */
873                 ucmd_get(ucmd);
874                 __dev_user_free_sg_entries(ucmd);
875         }
876         return;
877 }
878
879 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
880 {
881         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
882
883         TRACE_ENTRY();
884
885         if (unlikely(ucmd == NULL))
886                 goto out;
887
888         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
889                 ucmd->buff_cached, ucmd->ubuff);
890
891         ucmd->cmd = NULL;
892         if ((cmd->data_direction == SCST_DATA_WRITE) && (ucmd->buf_ucmd != NULL))
893                 ucmd->buf_ucmd->buf_dirty = 1;
894
895         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
896                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
897                 /* The state assignment must be before freeing sgv! */
898                 goto out_reply;
899         }
900
901         if (unlikely(!ucmd->seen_by_user)) {
902                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
903                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
904                 goto out_reply;
905         }
906
907         ucmd->user_cmd.cmd_h = ucmd->h;
908         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
909
910         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
911         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
912         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
913         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
914         ucmd->user_cmd.on_free_cmd.status = cmd->status;
915         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
916
917         ucmd->state = UCMD_STATE_ON_FREEING;
918
919         dev_user_add_to_ready(ucmd);
920
921 out:
922         TRACE_EXIT();
923         return;
924
925 out_reply:
926         dev_user_process_reply_on_free(ucmd);
927         goto out;
928 }
929
930 static void dev_user_set_block(struct scst_cmd *cmd, int block)
931 {
932         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
933         /*
934          * No need for locks here, since *_detach() can not be
935          * called, when there are existing commands.
936          */
937         TRACE_DBG("dev %p, new block %d", dev, block);
938         if (block != 0)
939                 dev->block = block;
940         else
941                 dev->block = dev->def_block;
942         return;
943 }
944
945 static int dev_user_disk_done(struct scst_cmd *cmd)
946 {
947         int res = SCST_CMD_STATE_DEFAULT;
948
949         TRACE_ENTRY();
950
951         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
952
953         TRACE_EXIT_RES(res);
954         return res;
955 }
956
957 static int dev_user_tape_done(struct scst_cmd *cmd)
958 {
959         int res = SCST_CMD_STATE_DEFAULT;
960
961         TRACE_ENTRY();
962
963         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
964
965         TRACE_EXIT_RES(res);
966         return res;
967 }
968
969 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
970 {
971         struct scst_user_dev *dev = ucmd->dev;
972         unsigned long flags;
973         int do_wake;
974
975         TRACE_ENTRY();
976
977         do_wake = (in_interrupt() ||
978                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
979         if (ucmd->cmd)
980                 do_wake |= ucmd->cmd->preprocessing_only;
981
982         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
983
984         ucmd->this_state_unjammed = 0;
985
986         if ((ucmd->state == UCMD_STATE_PARSING) ||
987             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
988                 /*
989                  * If we don't put such commands in the queue head, then under
990                  * high load we might delay threads, waiting for memory
991                  * allocations, for too long and start loosing NOPs, which
992                  * would lead to consider us by remote initiators as
993                  * unresponsive and stuck => broken connections, etc. If none
994                  * of our commands completed in NOP timeout to allow the head
995                  * commands to go, then we are really overloaded and/or stuck.
996                  */
997                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
998                         "cmd list", ucmd, ucmd->state);
999                 list_add(&ucmd->ready_cmd_list_entry,
1000                         &dev->ready_cmd_list);
1001                 do_wake = 1;
1002         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1003                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1004                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1005                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1006                         "ready cmd list", ucmd, ucmd->state);
1007                 list_add(&ucmd->ready_cmd_list_entry,
1008                         &dev->ready_cmd_list);
1009                 do_wake = 1;
1010         } else if ((ucmd->cmd != NULL) &&
1011                    unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1012                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1013                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1014         } else {
1015                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1016                 list_add_tail(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1017         }
1018
1019         if (do_wake) {
1020                 TRACE_DBG("Waking up dev %p", dev);
1021                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1022         }
1023
1024         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1025
1026         TRACE_EXIT();
1027         return;
1028 }
1029
1030 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1031         int num_pg)
1032 {
1033         int res = 0, rc;
1034         int i;
1035
1036         TRACE_ENTRY();
1037
1038         if (unlikely(ubuff == 0))
1039                 goto out_nomem;
1040
1041         sBUG_ON(ucmd->data_pages != NULL);
1042
1043         ucmd->num_data_pages = num_pg;
1044
1045         ucmd->data_pages = kzalloc(sizeof(*ucmd->data_pages)*ucmd->num_data_pages,
1046                 GFP_KERNEL);
1047         if (ucmd->data_pages == NULL) {
1048                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1049                         "(num_data_pages=%d)", ucmd->num_data_pages);
1050                 res = -ENOMEM;
1051                 goto out_nomem;
1052         }
1053
1054         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d, "
1055                 "first_page_offset %d, len %d)", ucmd, ubuff,
1056                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1057                 ucmd->cmd->bufflen);
1058
1059         down_read(&current->mm->mmap_sem);
1060         rc = get_user_pages(current, current->mm, ubuff, ucmd->num_data_pages,
1061                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1062         up_read(&current->mm->mmap_sem);
1063
1064         /* get_user_pages() flushes dcache */
1065
1066         if (rc < ucmd->num_data_pages)
1067                 goto out_unmap;
1068
1069         ucmd->ubuff = ubuff;
1070         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1071
1072 out:
1073         TRACE_EXIT_RES(res);
1074         return res;
1075
1076 out_nomem:
1077         scst_set_busy(ucmd->cmd);
1078         /* go through */
1079
1080 out_err:
1081         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1082         goto out;
1083
1084 out_unmap:
1085         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1086                 ucmd->num_data_pages, rc);
1087         if (rc > 0) {
1088                 for (i = 0; i < rc; i++)
1089                         page_cache_release(ucmd->data_pages[i]);
1090         }
1091         kfree(ucmd->data_pages);
1092         ucmd->data_pages = NULL;
1093         res = -EFAULT;
1094         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1095         goto out_err;
1096 }
1097
1098 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1099         struct scst_user_reply_cmd *reply)
1100 {
1101         int res = 0;
1102         struct scst_cmd *cmd = ucmd->cmd;
1103
1104         TRACE_ENTRY();
1105
1106         TRACE_DBG("ucmd %p, pbuf %Lx", ucmd, reply->alloc_reply.pbuf);
1107
1108         if (likely(reply->alloc_reply.pbuf != 0)) {
1109                 int pages;
1110                 if (ucmd->buff_cached) {
1111                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1112                                 PRINT_ERROR("Supplied pbuf %Lx isn't "
1113                                         "page aligned", reply->alloc_reply.pbuf);
1114                                 goto out_hwerr;
1115                         }
1116                         pages = cmd->sg_cnt;
1117                 } else
1118                         pages = calc_num_pg(reply->alloc_reply.pbuf, cmd->bufflen);
1119                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1120         } else {
1121                 scst_set_busy(ucmd->cmd);
1122                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1123         }
1124
1125 out_process:
1126         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1127
1128         TRACE_EXIT_RES(res);
1129         return res;
1130
1131 out_hwerr:
1132         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1133         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1134         res = -EINVAL;
1135         goto out_process;
1136 }
1137
1138 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1139         struct scst_user_reply_cmd *reply)
1140 {
1141         int res = 0;
1142         struct scst_user_scsi_cmd_reply_parse *preply =
1143                 &reply->parse_reply;
1144         struct scst_cmd *cmd = ucmd->cmd;
1145
1146         TRACE_ENTRY();
1147
1148         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1149                 goto out_inval;
1150
1151         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1152                      (preply->data_direction != SCST_DATA_READ) &&
1153                      (preply->data_direction != SCST_DATA_NONE)))
1154                 goto out_inval;
1155
1156         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1157                      (preply->bufflen == 0)))
1158                 goto out_inval;
1159
1160         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1161                 goto out_inval;
1162
1163         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1164                 "data_len %d, pbuf %Lx", ucmd, preply->queue_type,
1165                 preply->data_direction, preply->bufflen, preply->data_len,
1166                 reply->alloc_reply.pbuf);
1167
1168         cmd->queue_type = preply->queue_type;
1169         cmd->data_direction = preply->data_direction;
1170         cmd->bufflen = preply->bufflen;
1171         cmd->data_len = preply->data_len;
1172
1173 out_process:
1174         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1175
1176         TRACE_EXIT_RES(res);
1177         return res;
1178
1179 out_inval:
1180         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1181                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1182         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1183         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1184         scst_set_cmd_abnormal_done_state(cmd);
1185         res = -EINVAL;
1186         goto out_process;
1187 }
1188
1189 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1190 {
1191         int res = 0;
1192
1193         TRACE_ENTRY();
1194
1195         TRACE_DBG("ON FREE ucmd %p", ucmd);
1196
1197         dev_user_free_sgv(ucmd);
1198         ucmd_put(ucmd);
1199
1200         TRACE_EXIT_RES(res);
1201         return res;
1202 }
1203
1204 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1205 {
1206         int res = 0;
1207
1208         TRACE_ENTRY();
1209
1210         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1211
1212         ucmd_put(ucmd);
1213
1214         TRACE_EXIT_RES(res);
1215         return res;
1216 }
1217
1218 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1219         struct scst_user_reply_cmd *reply)
1220 {
1221         int res = 0;
1222         struct scst_user_scsi_cmd_reply_exec *ereply =
1223                 &reply->exec_reply;
1224         struct scst_cmd *cmd = ucmd->cmd;
1225
1226         TRACE_ENTRY();
1227
1228         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1229                 if (ucmd->background_exec) {
1230                         TRACE_DBG("Background ucmd %p finished", ucmd);
1231                         ucmd_put(ucmd);
1232                         goto out;
1233                 }
1234                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1235                         goto out_inval;
1236                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1237                              (ereply->resp_data_len != 0)))
1238                         goto out_inval;
1239         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1240                 if (unlikely(ucmd->background_exec))
1241                         goto out_inval;
1242                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1243                              (cmd->resp_data_len != 0)))
1244                         goto out_inval;
1245                 ucmd_get_ordered(ucmd);
1246                 ucmd->background_exec = 1;
1247                 TRACE_DBG("Background ucmd %p", ucmd);
1248                 goto out_compl;
1249         } else
1250                 goto out_inval;
1251
1252         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1253                 ereply->status, ereply->resp_data_len);
1254
1255          if (ereply->resp_data_len != 0) {
1256                 if (ucmd->ubuff == 0) {
1257                         int pages, rc;
1258                         if (unlikely(ereply->pbuf == 0))
1259                                 goto out_busy;
1260                         if (ucmd->buff_cached) {
1261                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1262                                         PRINT_ERROR("Supplied pbuf %Lx isn't "
1263                                                 "page aligned", ereply->pbuf);
1264                                         goto out_hwerr;
1265                                 }
1266                                 pages = cmd->sg_cnt;
1267                         } else
1268                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1269                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1270                         if ((rc != 0) || (ucmd->ubuff == 0))
1271                                 goto out_compl;
1272
1273                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1274                         if (unlikely(rc != 0))
1275                                 goto out_busy;
1276                 } else
1277                         dev_user_flush_dcache(ucmd);
1278                 cmd->may_need_dma_sync = 1;
1279                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1280         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1281                 if (ucmd->ubuff == 0)
1282                         cmd->resp_data_len = ereply->resp_data_len;
1283                 else
1284                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1285         }
1286
1287         cmd->status = ereply->status;
1288         if (ereply->sense_len != 0) {
1289                 res = scst_alloc_sense(cmd, 0);
1290                 if (res != 0)
1291                         goto out_compl;
1292                 res = copy_from_user(cmd->sense,
1293                         (void __user *)(unsigned long)ereply->psense_buffer,
1294                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1295                                 (unsigned int)ereply->sense_len));
1296                 if (res < 0) {
1297                         PRINT_ERROR("%s", "Unable to get sense data");
1298                         goto out_hwerr_res_set;
1299                 }
1300         }
1301
1302 out_compl:
1303         cmd->completed = 1;
1304         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1305         /* !! At this point cmd can be already freed !! */
1306
1307 out:
1308         TRACE_EXIT_RES(res);
1309         return res;
1310
1311 out_inval:
1312         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1313                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1314         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1315
1316 out_hwerr:
1317         res = -EINVAL;
1318
1319 out_hwerr_res_set:
1320         if (ucmd->background_exec) {
1321                 ucmd_put(ucmd);
1322                 goto out;
1323         } else {
1324                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1325                 goto out_compl;
1326         }
1327
1328 out_busy:
1329         scst_set_busy(cmd);
1330         goto out_compl;
1331 }
1332
1333 static int dev_user_process_reply(struct scst_user_dev *dev,
1334         struct scst_user_reply_cmd *reply)
1335 {
1336         int res = 0;
1337         struct scst_user_cmd *ucmd;
1338         int state;
1339
1340         TRACE_ENTRY();
1341
1342         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1343
1344         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1345         if (unlikely(ucmd == NULL)) {
1346                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1347                 res = -ESRCH;
1348                 goto out_unlock;
1349         }
1350
1351         if (unlikely(ucmd_get_check(ucmd))) {
1352                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1353                 res = -ESRCH;
1354                 goto out_unlock;
1355         }
1356
1357         if (ucmd->background_exec) {
1358                 state = UCMD_STATE_EXECING;
1359                 goto unlock_process;
1360         }
1361
1362         if (unlikely(ucmd->this_state_unjammed)) {
1363                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1364                         ucmd);
1365                 goto out_unlock_put;
1366         }
1367
1368         if (unlikely(!ucmd->sent_to_user)) {
1369                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1370                         "state %x", ucmd, ucmd->state);
1371                 res = -EINVAL;
1372                 goto out_unlock_put;
1373         }
1374
1375         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1376                 goto out_wrong_state;
1377
1378         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1379                 goto out_wrong_state;
1380
1381         state = ucmd->state;
1382         ucmd->sent_to_user = 0;
1383
1384 unlock_process:
1385         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1386
1387         switch (state) {
1388         case UCMD_STATE_PARSING:
1389                 res = dev_user_process_reply_parse(ucmd, reply);
1390                 break;
1391
1392         case UCMD_STATE_BUF_ALLOCING:
1393                 res = dev_user_process_reply_alloc(ucmd, reply);
1394                 break;
1395
1396         case UCMD_STATE_EXECING:
1397                 res = dev_user_process_reply_exec(ucmd, reply);
1398                 break;
1399
1400         case UCMD_STATE_ON_FREEING:
1401                 res = dev_user_process_reply_on_free(ucmd);
1402                 break;
1403
1404         case UCMD_STATE_ON_CACHE_FREEING:
1405                 res = dev_user_process_reply_on_cache_free(ucmd);
1406                 break;
1407
1408         case UCMD_STATE_TM_EXECING:
1409                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1410                 break;
1411
1412         case UCMD_STATE_ATTACH_SESS:
1413         case UCMD_STATE_DETACH_SESS:
1414                 res = dev_user_process_reply_sess(ucmd, reply->result);
1415                 break;
1416
1417         default:
1418                 sBUG();
1419                 break;
1420         }
1421
1422 out_put:
1423         ucmd_put(ucmd);
1424
1425 out:
1426         TRACE_EXIT_RES(res);
1427         return res;
1428
1429 out_wrong_state:
1430         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1431                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1432                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1433                 reply->subcode, ucmd->user_cmd.subcode);
1434         res = -EINVAL;
1435         dev_user_unjam_cmd(ucmd, 0, NULL);
1436
1437 out_unlock_put:
1438         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1439         goto out_put;
1440
1441 out_unlock:
1442         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1443         goto out;
1444 }
1445
1446 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1447 {
1448         int res = 0;
1449         struct scst_user_dev *dev;
1450         struct scst_user_reply_cmd *reply;
1451
1452         TRACE_ENTRY();
1453
1454         mutex_lock(&dev_priv_mutex);
1455         dev = (struct scst_user_dev *)file->private_data;
1456         res = dev_user_check_reg(dev);
1457         if (res != 0) {
1458                 mutex_unlock(&dev_priv_mutex);
1459                 goto out;
1460         }
1461         down_read(&dev->dev_rwsem);
1462         mutex_unlock(&dev_priv_mutex);
1463
1464         reply = kzalloc(sizeof(*reply), GFP_KERNEL);
1465         if (reply == NULL) {
1466                 res = -ENOMEM;
1467                 goto out_up;
1468         }
1469
1470         res = copy_from_user(reply, arg, sizeof(*reply));
1471         if (res < 0)
1472                 goto out_free;
1473
1474         TRACE_BUFFER("Reply", reply, sizeof(*reply));
1475
1476         res = dev_user_process_reply(dev, reply);
1477         if (res < 0)
1478                 goto out_free;
1479
1480 out_free:
1481         kfree(reply);
1482
1483 out_up:
1484         up_read(&dev->dev_rwsem);
1485
1486 out:
1487         TRACE_EXIT_RES(res);
1488         return res;
1489 }
1490
1491 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1492 {
1493         int res = 0;
1494
1495         TRACE_ENTRY();
1496
1497         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1498                 struct scst_cmd *cmd = list_entry(
1499                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1500                         cmd_list_entry);
1501                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1502                 list_del(&cmd->cmd_list_entry);
1503                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1504                 scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT |
1505                                                  SCST_CONTEXT_PROCESSABLE);
1506                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1507                 res++;
1508         }
1509
1510         TRACE_EXIT_RES(res);
1511         return res;
1512 }
1513
1514 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1515 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1516 {
1517         struct scst_user_cmd *u;
1518
1519 again:
1520         u = NULL;
1521         if (!list_empty(cmd_list)) {
1522                 u = list_entry(cmd_list->next, typeof(*u), ready_cmd_list_entry);
1523
1524                 TRACE_DBG("Found ready ucmd %p", u);
1525                 list_del(&u->ready_cmd_list_entry);
1526
1527                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1528
1529                 if (u->cmd != NULL) {
1530                         if (u->state == UCMD_STATE_EXECING) {
1531                                 struct scst_user_dev *dev = u->dev;
1532                                 int rc;
1533
1534                                 EXTRACHECKS_BUG_ON(u->jammed);
1535
1536                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1537
1538                                 rc = scst_check_local_events(u->cmd);
1539                                 if (unlikely(rc != 0)) {
1540                                         u->cmd->scst_cmd_done(u->cmd,
1541                                                 SCST_CMD_STATE_DEFAULT);
1542                                         /*
1543                                          * !! At this point cmd & u can be !!
1544                                          * !! already freed                !!
1545                                          */
1546                                         spin_lock_irq(
1547                                                 &dev->cmd_lists.cmd_list_lock);
1548                                         goto again;
1549                                 }
1550
1551                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1552                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1553                                         &u->cmd->cmd_flags))) {
1554                                 switch (u->state) {
1555                                 case UCMD_STATE_PARSING:
1556                                 case UCMD_STATE_BUF_ALLOCING:
1557                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1558                                         dev_user_unjam_cmd(u, 0, NULL);
1559                                         goto again;
1560                                 case UCMD_STATE_EXECING:
1561                                         EXTRACHECKS_BUG_ON(1);
1562                                 }
1563                         }
1564                 }
1565                 u->sent_to_user = 1;
1566                 u->seen_by_user = 1;
1567         }
1568         return u;
1569 }
1570
1571 static inline int test_cmd_lists(struct scst_user_dev *dev)
1572 {
1573         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1574                   !list_empty(&dev->ready_cmd_list) ||
1575                   !dev->blocking || dev->cleanup_done ||
1576                   signal_pending(current);
1577         return res;
1578 }
1579
1580 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1581 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1582         struct scst_user_cmd **ucmd)
1583 {
1584         int res = 0;
1585         wait_queue_t wait;
1586
1587         TRACE_ENTRY();
1588
1589         init_waitqueue_entry(&wait, current);
1590
1591         while (1) {
1592                 if (!test_cmd_lists(dev)) {
1593                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1594                                 &wait);
1595                         for (;;) {
1596                                 set_current_state(TASK_INTERRUPTIBLE);
1597                                 if (test_cmd_lists(dev))
1598                                         break;
1599                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1600                                 schedule();
1601                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1602                         }
1603                         set_current_state(TASK_RUNNING);
1604                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1605                                 &wait);
1606                 }
1607
1608                 dev_user_process_scst_commands(dev);
1609
1610                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1611                 if (*ucmd != NULL)
1612                         break;
1613
1614                 if (!dev->blocking || dev->cleanup_done) {
1615                         res = -EAGAIN;
1616                         TRACE_DBG("No ready commands, returning %d", res);
1617                         break;
1618                 }
1619
1620                 if (signal_pending(current)) {
1621                         res = -EINTR;
1622                         TRACE_DBG("Signal pending, returning %d", res);
1623                         break;
1624                 }
1625         }
1626
1627         TRACE_EXIT_RES(res);
1628         return res;
1629 }
1630
1631 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1632 {
1633         int res = 0;
1634         struct scst_user_dev *dev;
1635         struct scst_user_get_cmd *cmd;
1636         struct scst_user_reply_cmd *reply;
1637         struct scst_user_cmd *ucmd;
1638         uint64_t ureply;
1639
1640         TRACE_ENTRY();
1641
1642         mutex_lock(&dev_priv_mutex);
1643         dev = (struct scst_user_dev *)file->private_data;
1644         res = dev_user_check_reg(dev);
1645         if (res != 0) {
1646                 mutex_unlock(&dev_priv_mutex);
1647                 goto out;
1648         }
1649         down_read(&dev->dev_rwsem);
1650         mutex_unlock(&dev_priv_mutex);
1651
1652         res = copy_from_user(&ureply, arg, sizeof(ureply));
1653         if (res < 0)
1654                 goto out_up;
1655
1656         TRACE_DBG("ureply %Ld", (long long unsigned int)ureply);
1657
1658         cmd = kzalloc(max(sizeof(*cmd), sizeof(*reply)), GFP_KERNEL);
1659         if (cmd == NULL) {
1660                 res = -ENOMEM;
1661                 goto out_up;
1662         }
1663
1664         if (ureply != 0) {
1665                 unsigned long u = (unsigned long)ureply;
1666                 reply = (struct scst_user_reply_cmd *)cmd;
1667                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1668                 if (res < 0)
1669                         goto out_free;
1670
1671                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1672
1673                 res = dev_user_process_reply(dev, reply);
1674                 if (res < 0)
1675                         goto out_free;
1676         }
1677
1678         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1679         res = dev_user_get_next_cmd(dev, &ucmd);
1680         if (res == 0) {
1681                 *cmd = ucmd->user_cmd;
1682                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1683                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1684                 res = copy_to_user(arg, cmd, sizeof(*cmd));
1685         } else
1686                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1687
1688 out_free:
1689         kfree(cmd);
1690
1691 out_up:
1692         up_read(&dev->dev_rwsem);
1693
1694 out:
1695         TRACE_EXIT_RES(res);
1696         return res;
1697 }
1698
1699 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1700         unsigned long arg)
1701 {
1702         long res;
1703
1704         TRACE_ENTRY();
1705
1706         switch (cmd) {
1707         case SCST_USER_REPLY_AND_GET_CMD:
1708                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1709                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1710                 break;
1711
1712         case SCST_USER_REPLY_CMD:
1713                 TRACE_DBG("%s", "REPLY_CMD");
1714                 res = dev_user_reply_cmd(file, (void __user *)arg);
1715                 break;
1716
1717         case SCST_USER_REGISTER_DEVICE:
1718         {
1719                 struct scst_user_dev_desc *dev_desc;
1720                 TRACE_DBG("%s", "REGISTER_DEVICE");
1721                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1722                 if (dev_desc == NULL) {
1723                         res = -ENOMEM;
1724                         goto out;
1725                 }
1726                 res = copy_from_user(dev_desc, (void __user *)arg, sizeof(*dev_desc));
1727                 if (res < 0) {
1728                         kfree(dev_desc);
1729                         goto out;
1730                 }
1731                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1732                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1733                 res = dev_user_register_dev(file, dev_desc);
1734                 kfree(dev_desc);
1735                 break;
1736         }
1737
1738         case SCST_USER_SET_OPTIONS:
1739         {
1740                 struct scst_user_opt opt;
1741                 TRACE_DBG("%s", "SET_OPTIONS");
1742                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1743                 if (res < 0)
1744                         goto out;
1745                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1746                 res = dev_user_set_opt(file, &opt);
1747                 break;
1748         }
1749
1750         case SCST_USER_GET_OPTIONS:
1751                 TRACE_DBG("%s", "GET_OPTIONS");
1752                 res = dev_user_get_opt(file, (void __user *)arg);
1753                 break;
1754
1755         default:
1756                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1757                 res = -EINVAL;
1758                 goto out;
1759         }
1760
1761 out:
1762         TRACE_EXIT_RES(res);
1763         return res;
1764 }
1765
1766 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1767 {
1768         int res = 0;
1769         struct scst_user_dev *dev;
1770
1771         TRACE_ENTRY();
1772
1773         mutex_lock(&dev_priv_mutex);
1774         dev = (struct scst_user_dev *)file->private_data;
1775         res = dev_user_check_reg(dev);
1776         if (res != 0) {
1777                 mutex_unlock(&dev_priv_mutex);
1778                 goto out;
1779         }
1780         down_read(&dev->dev_rwsem);
1781         mutex_unlock(&dev_priv_mutex);
1782
1783         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1784
1785         if (!list_empty(&dev->ready_cmd_list) ||
1786             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1787                 res |= POLLIN | POLLRDNORM;
1788                 goto out_unlock;
1789         }
1790
1791         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1792
1793         TRACE_DBG("Before poll_wait() (dev %p)", dev);
1794         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1795         TRACE_DBG("After poll_wait() (dev %p)", dev);
1796
1797         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1798
1799         if (!list_empty(&dev->ready_cmd_list) ||
1800             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1801                 res |= POLLIN | POLLRDNORM;
1802                 goto out_unlock;
1803         }
1804
1805 out_unlock:
1806         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1807
1808         up_read(&dev->dev_rwsem);
1809
1810 out:
1811         TRACE_EXIT_HRES(res);
1812         return res;
1813 }
1814
1815 /*
1816  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reaquire.
1817  */
1818 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1819         unsigned long *flags)
1820 {
1821         int state = ucmd->state;
1822         struct scst_user_dev *dev = ucmd->dev;
1823
1824         TRACE_ENTRY();
1825
1826         if (ucmd->this_state_unjammed)
1827                 goto out;
1828
1829         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1830                 state);
1831
1832         ucmd->jammed = 1;
1833         ucmd->this_state_unjammed = 1;
1834         ucmd->sent_to_user = 0;
1835
1836         switch (state) {
1837         case UCMD_STATE_PARSING:
1838         case UCMD_STATE_BUF_ALLOCING:
1839                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1840                         ucmd->aborted = 1;
1841                 else {
1842                         if (busy)
1843                                 scst_set_busy(ucmd->cmd);
1844                         else
1845                                 scst_set_cmd_error(ucmd->cmd,
1846                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1847                 }
1848                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1849
1850                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1851                 list_add(&ucmd->cmd->cmd_list_entry,
1852                         &ucmd->cmd->cmd_lists->active_cmd_list);
1853                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1854                 break;
1855
1856         case UCMD_STATE_EXECING:
1857                 if (flags != NULL)
1858                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1859                 else
1860                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1861
1862                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1863
1864                 if (test_bit(SCST_CMD_ABORTED,  &ucmd->cmd->cmd_flags))
1865                         ucmd->aborted = 1;
1866                 else {
1867                         if (busy)
1868                                 scst_set_busy(ucmd->cmd);
1869                         else
1870                                 scst_set_cmd_error(ucmd->cmd,
1871                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1872                 }
1873
1874                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT);
1875                 /* !! At this point cmd and ucmd can be already freed !! */
1876
1877                 if (flags != NULL)
1878                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1879                 else
1880                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1881                 break;
1882
1883         case UCMD_STATE_ON_FREEING:
1884         case UCMD_STATE_ON_CACHE_FREEING:
1885         case UCMD_STATE_TM_EXECING:
1886         case UCMD_STATE_ATTACH_SESS:
1887         case UCMD_STATE_DETACH_SESS:
1888         {
1889                 if (flags != NULL)
1890                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1891                 else
1892                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1893
1894                 switch (state) {
1895                 case UCMD_STATE_ON_FREEING:
1896                         dev_user_process_reply_on_free(ucmd);
1897                         break;
1898
1899                 case UCMD_STATE_ON_CACHE_FREEING:
1900                         dev_user_process_reply_on_cache_free(ucmd);
1901                         break;
1902
1903                 case UCMD_STATE_TM_EXECING:
1904                         dev_user_process_reply_tm_exec(ucmd, SCST_MGMT_STATUS_FAILED);
1905                         break;
1906
1907                 case UCMD_STATE_ATTACH_SESS:
1908                 case UCMD_STATE_DETACH_SESS:
1909                         dev_user_process_reply_sess(ucmd, -EFAULT);
1910                         break;
1911                 }
1912
1913                 if (flags != NULL)
1914                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1915                 else
1916                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1917                 break;
1918         }
1919
1920         default:
1921                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1922                 sBUG();
1923                 break;
1924         }
1925
1926 out:
1927         TRACE_EXIT();
1928         return;
1929 }
1930
1931 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1932 {
1933         int i;
1934         struct scst_user_cmd *ucmd;
1935
1936         TRACE_ENTRY();
1937
1938         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1939
1940         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1941
1942 repeat:
1943         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1944                 struct list_head *head = &dev->ucmd_hash[i];
1945
1946                 list_for_each_entry(ucmd, head, hash_list_entry) {
1947                         if (!ucmd->sent_to_user)
1948                                 continue;
1949
1950                         if (ucmd_get_check(ucmd))
1951                                 continue;
1952
1953                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1954                                 ucmd->state, ucmd->cmd);
1955
1956                         dev_user_unjam_cmd(ucmd, 0, NULL);
1957
1958                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1959                         ucmd_put(ucmd);
1960                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1961
1962                         goto repeat;
1963                 }
1964         }
1965
1966         if (dev_user_process_scst_commands(dev) != 0)
1967                 goto repeat;
1968
1969         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1970
1971         TRACE_EXIT();
1972         return;
1973 }
1974
1975 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
1976         int status)
1977 {
1978         int res = 0;
1979
1980         TRACE_ENTRY();
1981
1982         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
1983                 ucmd->user_cmd.tm_cmd.fn, status);
1984
1985         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
1986                 /*
1987                  * It is possible that user space seen TM cmd before cmd
1988                  * to abort or will never see it at all, because it was
1989                  * aborted on the way there. So, it is safe to return
1990                  * success instead, because, if there is the TM cmd at this
1991                  * point, then the cmd to abort apparrently does exist.
1992                  */
1993                 status = SCST_MGMT_STATUS_SUCCESS;
1994         }
1995
1996         scst_async_mcmd_completed(ucmd->mcmd, status);
1997
1998         ucmd_put(ucmd);
1999
2000         TRACE_EXIT_RES(res);
2001         return res;
2002 }
2003
2004 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2005 {
2006         struct scst_user_cmd *ucmd;
2007         unsigned long flags;
2008
2009         TRACE_ENTRY();
2010
2011         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2012 again:
2013         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2014                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2015                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2016                         switch (ucmd->state) {
2017                         case UCMD_STATE_PARSING:
2018                         case UCMD_STATE_BUF_ALLOCING:
2019                         case UCMD_STATE_EXECING:
2020                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2021                                 list_del(&ucmd->ready_cmd_list_entry);
2022                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2023                                 goto again;
2024                         }
2025                 }
2026         }
2027
2028         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2029
2030         TRACE_EXIT();
2031         return;
2032 }
2033
2034 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2035         struct scst_tgt_dev *tgt_dev)
2036 {
2037         struct scst_user_cmd *ucmd;
2038         struct scst_user_dev *dev = (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2039         struct scst_user_cmd *ucmd_to_abort = NULL;
2040
2041         TRACE_ENTRY();
2042
2043         /*
2044          * In the used approach we don't do anything with hung devices, which
2045          * stopped responding and/or have stuck commands. We forcedly abort such
2046          * commands only if they not yet sent to the user space or if the device
2047          * is getting unloaded, e.g. if its handler program gets killed. This is
2048          * because it's pretty hard to distinguish between stuck and temporary
2049          * overloaded states of the device. There are several reasons for that:
2050          *
2051          * 1. Some commands need a lot of time to complete (several hours),
2052          *    so for an impatient user such command(s) will always look as
2053          *    stuck.
2054          *
2055          * 2. If we forcedly abort, i.e. abort before it's actually completed
2056          *    in the user space, just one command, we will have to put the whole
2057          *    device offline until we are sure that no more previously aborted
2058          *    commands will get executed. Otherwise, we might have a possibility
2059          *    for data corruption, when aborted and reported as completed
2060          *    command actually gets executed *after* new commands sent
2061          *    after the force abort was done. Many journaling file systems and
2062          *    databases use "provide required commands order via queue draining"
2063          *    approach and not putting the whole device offline after the forced
2064          *    abort will break it. This makes our decision, if a command stuck
2065          *    or not, cost a lot.
2066          *
2067          * So, we leave policy definition if a device stuck or not to
2068          * the user space and simply let all commands live until they are
2069          * completed or their devices get closed/killed. This approach is very
2070          * much OK, but can affect management commands, which need activity
2071          * suspending via scst_suspend_activity() function such as devices or
2072          * targets registration/removal. But during normal life such commands
2073          * should be rare. Plus, when possible, scst_suspend_activity() will
2074          * return after timeout EBUSY status to allow caller to not stuck
2075          * forever as well.
2076          *
2077          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2078          * stuck commands would affect only related devices.
2079          */
2080
2081         dev_user_abort_ready_commands(dev);
2082
2083         /* We can't afford missing TM command due to memory shortage */
2084         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2085
2086         ucmd->user_cmd.cmd_h = ucmd->h;
2087         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2088         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2089         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2090         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2091         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2092
2093         if (mcmd->cmd_to_abort != NULL) {
2094                 ucmd_to_abort = (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2095                 if (ucmd_to_abort != NULL)
2096                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2097         }
2098
2099         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2100                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2101                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2102                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2103
2104         ucmd->mcmd = mcmd;
2105         ucmd->state = UCMD_STATE_TM_EXECING;
2106
2107         scst_prepare_async_mcmd(mcmd);
2108
2109         dev_user_add_to_ready(ucmd);
2110
2111         TRACE_EXIT();
2112         return SCST_DEV_TM_NOT_COMPLETED;
2113 }
2114
2115 static int dev_user_attach(struct scst_device *sdev)
2116 {
2117         int res = 0;
2118         struct scst_user_dev *dev = NULL, *d;
2119
2120         TRACE_ENTRY();
2121
2122         spin_lock(&dev_list_lock);
2123         list_for_each_entry(d, &dev_list, dev_list_entry) {
2124                 if (strcmp(d->name, sdev->virt_name) == 0) {
2125                         dev = d;
2126                         break;
2127                 }
2128         }
2129         spin_unlock(&dev_list_lock);
2130         if (dev == NULL) {
2131                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2132                 res = -EINVAL;
2133                 goto out;
2134         }
2135
2136         sdev->p_cmd_lists = &dev->cmd_lists;
2137         sdev->dh_priv = dev;
2138         sdev->tst = dev->tst;
2139         sdev->queue_alg = dev->queue_alg;
2140         sdev->swp = dev->swp;
2141         sdev->tas = dev->tas;
2142         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2143
2144         dev->sdev = sdev;
2145
2146         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2147                 dev->name);
2148
2149 out:
2150         TRACE_EXIT();
2151         return res;
2152 }
2153
2154 static void dev_user_detach(struct scst_device *sdev)
2155 {
2156         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2157
2158         TRACE_ENTRY();
2159
2160         TRACE_DBG("virt_id %d", sdev->virt_id);
2161
2162         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2163                 dev->name);
2164
2165         /* dev will be freed by the caller */
2166         sdev->dh_priv = NULL;
2167         dev->sdev = NULL;
2168
2169         TRACE_EXIT();
2170         return;
2171 }
2172
2173 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2174 {
2175         int res = 0;
2176         unsigned long flags;
2177
2178         TRACE_ENTRY();
2179
2180         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2181
2182         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2183
2184         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2185                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2186                 ucmd->result = status;
2187         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2188                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2189         } else
2190                 sBUG();
2191
2192         if (ucmd->cmpl != NULL)
2193                 complete_all(ucmd->cmpl);
2194
2195         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2196
2197         ucmd_put(ucmd);
2198
2199         TRACE_EXIT_RES(res);
2200         return res;
2201 }
2202
2203 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2204 {
2205         struct scst_user_dev *dev =
2206                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2207         int res = 0, rc;
2208         struct scst_user_cmd *ucmd;
2209
2210         TRACE_ENTRY();
2211
2212         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2213         if (ucmd == NULL)
2214                 goto out_nomem;
2215
2216         ucmd->cmpl = kmalloc(sizeof(*ucmd->cmpl), GFP_KERNEL);
2217         if (ucmd->cmpl == NULL)
2218                 goto out_put_nomem;
2219
2220         init_completion(ucmd->cmpl);
2221
2222         ucmd->user_cmd.cmd_h = ucmd->h;
2223         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2224         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2225         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2226         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2227         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2228         strncpy(ucmd->user_cmd.sess.initiator_name,
2229                 tgt_dev->sess->initiator_name,
2230                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2231         ucmd->user_cmd.sess.initiator_name[
2232                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2233
2234         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %Lx, LUN %Lx, "
2235                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2236                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2237                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2238                 ucmd->user_cmd.sess.initiator_name);
2239
2240         ucmd->state = UCMD_STATE_ATTACH_SESS;
2241
2242         ucmd_get(ucmd);
2243
2244         dev_user_add_to_ready(ucmd);
2245
2246         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2247         if (rc > 0)
2248                 res = ucmd->result;
2249         else {
2250                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2251                 res = -EFAULT;
2252         }
2253
2254         sBUG_ON(irqs_disabled());
2255
2256         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2257         kfree(ucmd->cmpl);
2258         ucmd->cmpl = NULL;
2259         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2260
2261         ucmd_put(ucmd);
2262
2263 out:
2264         TRACE_EXIT_RES(res);
2265         return res;
2266
2267 out_put_nomem:
2268         ucmd_put(ucmd);
2269
2270 out_nomem:
2271         res = -ENOMEM;
2272         goto out;
2273 }
2274
2275 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2276 {
2277         struct scst_user_dev *dev =
2278                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2279         struct scst_user_cmd *ucmd;
2280
2281         TRACE_ENTRY();
2282
2283         /*
2284          * We can't miss TM command due to memory shortage, because it might
2285          * lead to a memory leak in the user space handler.
2286          */
2287         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2288         if (ucmd == NULL)
2289                 goto out;
2290
2291         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %Lx)", ucmd,
2292                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2293
2294         ucmd->user_cmd.cmd_h = ucmd->h;
2295         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2296         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2297
2298         ucmd->state = UCMD_STATE_DETACH_SESS;
2299
2300         dev_user_add_to_ready(ucmd);
2301
2302 out:
2303         TRACE_EXIT();
2304         return;
2305 }
2306
2307 /* No locks are needed, but the activity must be suspended */
2308 static void dev_user_setup_functions(struct scst_user_dev *dev)
2309 {
2310         TRACE_ENTRY();
2311
2312         dev->devtype.parse = dev_user_parse;
2313         dev->devtype.dev_done = NULL;
2314
2315         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2316                 switch (dev->devtype.type) {
2317                 case TYPE_DISK:
2318                         dev->generic_parse = scst_sbc_generic_parse;
2319                         dev->devtype.dev_done = dev_user_disk_done;
2320                         break;
2321
2322                 case TYPE_TAPE:
2323                         dev->generic_parse = scst_tape_generic_parse;
2324                         dev->devtype.dev_done = dev_user_tape_done;
2325                         break;
2326
2327                 case TYPE_MOD:
2328                         dev->generic_parse = scst_modisk_generic_parse;
2329                         dev->devtype.dev_done = dev_user_disk_done;
2330                         break;
2331
2332                 case TYPE_ROM:
2333                         dev->generic_parse = scst_cdrom_generic_parse;
2334                         dev->devtype.dev_done = dev_user_disk_done;
2335                         break;
2336
2337                 case TYPE_MEDIUM_CHANGER:
2338                         dev->generic_parse = scst_changer_generic_parse;
2339                         break;
2340
2341                 case TYPE_PROCESSOR:
2342                         dev->generic_parse = scst_processor_generic_parse;
2343                         break;
2344
2345                 case TYPE_RAID:
2346                         dev->generic_parse = scst_raid_generic_parse;
2347                         break;
2348
2349                 default:
2350                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2351                                 "for it", dev->devtype.type);
2352                         dev->parse_type = SCST_USER_PARSE_CALL;
2353                         break;
2354                 }
2355         } else {
2356                 dev->generic_parse = NULL;
2357                 dev->devtype.dev_done = NULL;
2358         }
2359
2360         TRACE_EXIT();
2361         return;
2362 }
2363
2364 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2365 {
2366         char ver[sizeof(DEV_USER_VERSION)+1];
2367         int res;
2368
2369         res = copy_from_user(ver, (void __user *)(unsigned long)dev_desc->version_str,
2370                                 sizeof(ver));
2371         if (res < 0) {
2372                 PRINT_ERROR("%s", "Unable to get version string");
2373                 goto out;
2374         }
2375         ver[sizeof(ver)-1] = '\0';
2376
2377         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2378                 /* ->name already 0-terminated in dev_user_ioctl() */
2379                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2380                         dev_desc->name, ver);
2381                 res = -EINVAL;
2382                 goto out;
2383         }
2384
2385 out:
2386         return res;
2387 }
2388
2389 static int dev_user_register_dev(struct file *file,
2390         const struct scst_user_dev_desc *dev_desc)
2391 {
2392         int res = -ENOMEM, i;
2393         struct scst_user_dev *dev, *d;
2394         int block;
2395
2396         TRACE_ENTRY();
2397
2398         res = dev_user_check_version(dev_desc);
2399         if (res != 0)
2400                 goto out;
2401
2402         switch (dev_desc->type) {
2403         case TYPE_DISK:
2404         case TYPE_ROM:
2405         case TYPE_MOD:
2406                 if (dev_desc->block_size == 0) {
2407                         PRINT_ERROR("Wrong block size %d", dev_desc->block_size);
2408                         res = -EINVAL;
2409                         goto out;
2410                 }
2411                 block = scst_calc_block_shift(dev_desc->block_size);
2412                 if (block == -1) {
2413                         res = -EINVAL;
2414                         goto out;
2415                 }
2416                 break;
2417         default:
2418                 block = dev_desc->block_size;
2419                 break;
2420         }
2421
2422         if (!try_module_get(THIS_MODULE)) {
2423                 PRINT_ERROR("%s", "Fail to get module");
2424                 goto out;
2425         }
2426
2427         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2428         if (dev == NULL)
2429                 goto out_put;
2430
2431         init_rwsem(&dev->dev_rwsem);
2432         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2433         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2434         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2435         INIT_LIST_HEAD(&dev->ready_cmd_list);
2436         if (file->f_flags & O_NONBLOCK) {
2437                 TRACE_DBG("%s", "Non-blocking operations");
2438                 dev->blocking = 0;
2439         } else
2440                 dev->blocking = 1;
2441         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2442                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2443
2444         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2445         dev->name[sizeof(dev->name)-1] = '\0';
2446
2447         scst_init_mem_lim(&dev->udev_mem_lim);
2448
2449         /*
2450          * We don't use clustered pool, since it implies pages reordering,
2451          * which isn't possible with user space supplied buffers. Although
2452          * it's still possible to cluster pages by the tail of each other,
2453          * seems it doesn't worth the effort.
2454          */
2455         dev->pool = sgv_pool_create(dev->name, 0);
2456         if (dev->pool == NULL)
2457                 goto out_put;
2458         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2459                 dev_user_free_sg_entries);
2460
2461         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2462                 dev->name);
2463         dev->devtype.type = dev_desc->type;
2464         dev->devtype.threads_num = -1;
2465         dev->devtype.parse_atomic = 1;
2466         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2467         dev->devtype.dev_done_atomic = 1;
2468         dev->devtype.no_proc = 1;
2469         dev->devtype.attach = dev_user_attach;
2470         dev->devtype.detach = dev_user_detach;
2471         dev->devtype.attach_tgt = dev_user_attach_tgt;
2472         dev->devtype.detach_tgt = dev_user_detach_tgt;
2473         dev->devtype.exec = dev_user_exec;
2474         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2475         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2476
2477         init_completion(&dev->cleanup_cmpl);
2478         dev->block = block;
2479         dev->def_block = block;
2480
2481         res = __dev_user_set_opt(dev, &dev_desc->opt);
2482
2483         TRACE_MEM("dev %p, name %s", dev, dev->name);
2484
2485         spin_lock(&dev_list_lock);
2486
2487         list_for_each_entry(d, &dev_list, dev_list_entry) {
2488                 if (strcmp(d->name, dev->name) == 0) {
2489                         PRINT_ERROR("Device %s already exist",
2490                                 dev->name);
2491                         res = -EEXIST;
2492                         spin_unlock(&dev_list_lock);
2493                         goto out_free;
2494                 }
2495         }
2496
2497         list_add_tail(&dev->dev_list_entry, &dev_list);
2498
2499         spin_unlock(&dev_list_lock);
2500
2501         if (res != 0)
2502                 goto out_del_free;
2503
2504         res = scst_register_virtual_dev_driver(&dev->devtype);
2505         if (res < 0)
2506                 goto out_del_free;
2507
2508         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2509         if (dev->virt_id < 0) {
2510                 res = dev->virt_id;
2511                 goto out_unreg_handler;
2512         }
2513
2514         mutex_lock(&dev_priv_mutex);
2515         if (file->private_data != NULL) {
2516                 mutex_unlock(&dev_priv_mutex);
2517                 PRINT_ERROR("%s", "Device already registered");
2518                 res = -EINVAL;
2519                 goto out_unreg_drv;
2520         }
2521         file->private_data = dev;
2522         mutex_unlock(&dev_priv_mutex);
2523
2524 out:
2525         TRACE_EXIT_RES(res);
2526         return res;
2527
2528 out_unreg_drv:
2529         scst_unregister_virtual_device(dev->virt_id);
2530
2531 out_unreg_handler:
2532         scst_unregister_virtual_dev_driver(&dev->devtype);
2533
2534 out_del_free:
2535         spin_lock(&dev_list_lock);
2536         list_del(&dev->dev_list_entry);
2537         spin_unlock(&dev_list_lock);
2538
2539 out_free:
2540         sgv_pool_destroy(dev->pool);
2541         kfree(dev);
2542         goto out_put;
2543
2544 out_put:
2545         module_put(THIS_MODULE);
2546         goto out;
2547 }
2548
2549 static int __dev_user_set_opt(struct scst_user_dev *dev,
2550         const struct scst_user_opt *opt)
2551 {
2552         int res = 0;
2553
2554         TRACE_ENTRY();
2555
2556         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2557                 "partial_transfers_type %x, partial_len %d", opt->parse_type,
2558                 opt->on_free_cmd_type, opt->memory_reuse_type,
2559                 opt->partial_transfers_type, opt->partial_len);
2560
2561         if ((opt->parse_type > SCST_USER_MAX_PARSE_OPT) ||
2562             (opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT) ||
2563             (opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT) ||
2564             (opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT)) {
2565                 PRINT_ERROR("%s", "Invalid option");
2566                 res = -EINVAL;
2567                 goto out;
2568         }
2569
2570         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2571              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2572             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2573              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2574             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2575                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x, "
2576                         "tas %x, has_own_order_mgmt %x)", opt->tst,
2577                         opt->queue_alg, opt->swp, opt->tas, opt->has_own_order_mgmt);
2578                 res = -EINVAL;
2579                 goto out;
2580         }
2581
2582         dev->parse_type = opt->parse_type;
2583         dev->on_free_cmd_type = opt->on_free_cmd_type;
2584         dev->memory_reuse_type = opt->memory_reuse_type;
2585         dev->partial_transfers_type = opt->partial_transfers_type;
2586         dev->partial_len = opt->partial_len;
2587
2588         dev->tst = opt->tst;
2589         dev->queue_alg = opt->queue_alg;
2590         dev->swp = opt->swp;
2591         dev->tas = opt->tas;
2592         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2593         if (dev->sdev != NULL) {
2594                 dev->sdev->tst = opt->tst;
2595                 dev->sdev->queue_alg = opt->queue_alg;
2596                 dev->sdev->swp = opt->swp;
2597                 dev->sdev->tas = opt->tas;
2598                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2599         }
2600
2601         dev_user_setup_functions(dev);
2602
2603 out:
2604         TRACE_EXIT_RES(res);
2605         return res;
2606 }
2607
2608 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2609 {
2610         int res = 0;
2611         struct scst_user_dev *dev;
2612
2613         TRACE_ENTRY();
2614
2615         mutex_lock(&dev_priv_mutex);
2616         dev = (struct scst_user_dev *)file->private_data;
2617         res = dev_user_check_reg(dev);
2618         if (res != 0) {
2619                 mutex_unlock(&dev_priv_mutex);
2620                 goto out;
2621         }
2622         down_write(&dev->dev_rwsem);
2623         mutex_unlock(&dev_priv_mutex);
2624
2625         res = scst_suspend_activity(true);
2626         if (res != 0)
2627                 goto out;
2628
2629         res = __dev_user_set_opt(dev, opt);
2630
2631         scst_resume_activity();
2632
2633         up_write(&dev->dev_rwsem);
2634
2635 out:
2636         TRACE_EXIT_RES(res);
2637         return res;
2638 }
2639
2640 static int dev_user_get_opt(struct file *file, void __user *arg)
2641 {
2642         int res = 0;
2643         struct scst_user_dev *dev;
2644         struct scst_user_opt opt;
2645
2646         TRACE_ENTRY();
2647
2648         mutex_lock(&dev_priv_mutex);
2649         dev = (struct scst_user_dev *)file->private_data;
2650         res = dev_user_check_reg(dev);
2651         if (res != 0) {
2652                 mutex_unlock(&dev_priv_mutex);
2653                 goto out;
2654         }
2655         down_read(&dev->dev_rwsem);
2656         mutex_unlock(&dev_priv_mutex);
2657
2658         opt.parse_type = dev->parse_type;
2659         opt.on_free_cmd_type = dev->on_free_cmd_type;
2660         opt.memory_reuse_type = dev->memory_reuse_type;
2661         opt.partial_transfers_type = dev->partial_transfers_type;
2662         opt.partial_len = dev->partial_len;
2663         opt.tst = dev->tst;
2664         opt.queue_alg = dev->queue_alg;
2665         opt.tas = dev->tas;
2666         opt.swp = dev->swp;
2667         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2668
2669         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2670                 "partial_transfers_type %x, partial_len %d", opt.parse_type,
2671                 opt.on_free_cmd_type, opt.memory_reuse_type,
2672                 opt.partial_transfers_type, opt.partial_len);
2673
2674         res = copy_to_user(arg, &opt, sizeof(opt));
2675
2676         up_read(&dev->dev_rwsem);
2677 out:
2678         TRACE_EXIT_RES(res);
2679         return res;
2680 }
2681
2682 static int dev_usr_parse(struct scst_cmd *cmd)
2683 {
2684         sBUG();
2685         return SCST_CMD_STATE_DEFAULT;
2686 }
2687
2688 /* Needed only for /proc support */
2689 #define USR_TYPE {                      \
2690         .name =         DEV_USER_NAME,  \
2691         .type =         -1,             \
2692         .parse =        dev_usr_parse,  \
2693 }
2694
2695 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2696
2697 static int dev_user_release(struct inode *inode, struct file *file)
2698 {
2699         int res = 0;
2700         struct scst_user_dev *dev;
2701
2702         TRACE_ENTRY();
2703
2704         mutex_lock(&dev_priv_mutex);
2705         dev = (struct scst_user_dev *)file->private_data;
2706         if (dev == NULL) {
2707                 mutex_unlock(&dev_priv_mutex);
2708                 goto out;
2709         }
2710         file->private_data = NULL;
2711
2712         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2713
2714         spin_lock(&dev_list_lock);
2715         list_del(&dev->dev_list_entry);
2716         spin_unlock(&dev_list_lock);
2717
2718         mutex_unlock(&dev_priv_mutex);
2719
2720         down_write(&dev->dev_rwsem);
2721
2722         spin_lock(&cleanup_lock);
2723         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2724         spin_unlock(&cleanup_lock);
2725
2726         wake_up(&cleanup_list_waitQ);
2727         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2728
2729         scst_unregister_virtual_device(dev->virt_id);
2730         scst_unregister_virtual_dev_driver(&dev->devtype);
2731
2732         sgv_pool_destroy(dev->pool);
2733
2734         TRACE_DBG("Unregistering finished (dev %p)", dev);
2735
2736         dev->cleanup_done = 1;
2737
2738         wake_up(&cleanup_list_waitQ);
2739         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2740
2741         wait_for_completion(&dev->cleanup_cmpl);
2742
2743         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2744
2745         TRACE_DBG("Releasing completed (dev %p)", dev);
2746
2747         kfree(dev);
2748
2749         module_put(THIS_MODULE);
2750
2751 out:
2752         TRACE_EXIT_RES(res);
2753         return res;
2754 }
2755
2756 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2757 {
2758         struct scst_user_cmd *ucmd;
2759         int rc, res = 1;
2760
2761         TRACE_ENTRY();
2762
2763         dev->blocking = 0;
2764
2765         while (1) {
2766                 TRACE_DBG("Cleanuping dev %p", dev);
2767
2768                 dev_user_unjam_dev(dev);
2769
2770                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2771
2772                 rc = dev_user_get_next_cmd(dev, &ucmd);
2773                 if (rc == 0)
2774                         dev_user_unjam_cmd(ucmd, 1, NULL);
2775
2776                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2777
2778                 if (rc == -EAGAIN) {
2779                         if (dev->cleanup_done)
2780                                 break;
2781                         else {
2782                                 TRACE_DBG("No more commands (dev %p)", dev);
2783                                 goto out;
2784                         }
2785                 }
2786         }
2787
2788 #ifdef CONFIG_SCST_EXTRACHECKS
2789 {
2790         int i;
2791         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2792                 struct list_head *head = &dev->ucmd_hash[i];
2793                 struct scst_user_cmd *ucmd;
2794 again:
2795                 list_for_each_entry(ucmd, head, hash_list_entry) {
2796                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd,
2797                                 ucmd->state, atomic_read(&ucmd->ucmd_ref));
2798                         ucmd_put(ucmd);
2799                         goto again;
2800                 }
2801         }
2802 }
2803 #endif
2804
2805         TRACE_DBG("Cleanuping done (dev %p)", dev);
2806         complete_all(&dev->cleanup_cmpl);
2807         res = 0;
2808
2809 out:
2810         TRACE_EXIT_RES(res);
2811         return res;
2812 }
2813
2814 static inline int test_cleanup_list(void)
2815 {
2816         int res = !list_empty(&cleanup_list) ||
2817                   unlikely(kthread_should_stop());
2818         return res;
2819 }
2820
2821 static int dev_user_cleanup_thread(void *arg)
2822 {
2823         TRACE_ENTRY();
2824
2825         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2826
2827         current->flags |= PF_NOFREEZE;
2828
2829         spin_lock(&cleanup_lock);
2830         while (!kthread_should_stop()) {
2831                 wait_queue_t wait;
2832                 init_waitqueue_entry(&wait, current);
2833
2834                 if (!test_cleanup_list()) {
2835                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2836                         for (;;) {
2837                                 set_current_state(TASK_INTERRUPTIBLE);
2838                                 if (test_cleanup_list())
2839                                         break;
2840                                 spin_unlock(&cleanup_lock);
2841                                 schedule();
2842                                 spin_lock(&cleanup_lock);
2843                         }
2844                         set_current_state(TASK_RUNNING);
2845                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2846                 }
2847
2848                 /*
2849                  * We have to poll devices, because commands can go from SCST
2850                  * core on cmd_list_waitQ and we have no practical way to
2851                  * detect them.
2852                  */
2853
2854                 while (1) {
2855                         struct scst_user_dev *dev;
2856                         LIST_HEAD(cl_devs);
2857
2858                         while (!list_empty(&cleanup_list)) {
2859                                 int rc;
2860
2861                                 dev = list_entry(cleanup_list.next,
2862                                         typeof(*dev), cleanup_list_entry);
2863                                 list_del(&dev->cleanup_list_entry);
2864
2865                                 spin_unlock(&cleanup_lock);
2866                                 rc = dev_user_process_cleanup(dev);
2867                                 spin_lock(&cleanup_lock);
2868
2869                                 if (rc != 0)
2870                                         list_add_tail(&dev->cleanup_list_entry,
2871                                                 &cl_devs);
2872                         }
2873
2874                         if (list_empty(&cl_devs))
2875                                 break;
2876
2877                         spin_unlock(&cleanup_lock);
2878                         msleep(100);
2879                         spin_lock(&cleanup_lock);
2880
2881                         while (!list_empty(&cl_devs)) {
2882                                 dev = list_entry(cl_devs.next, typeof(*dev),
2883                                         cleanup_list_entry);
2884                                 list_move_tail(&dev->cleanup_list_entry,
2885                                         &cleanup_list);
2886                         }
2887                 }
2888         }
2889         spin_unlock(&cleanup_lock);
2890
2891         /*
2892          * If kthread_should_stop() is true, we are guaranteed to be
2893          * on the module unload, so cleanup_list must be empty.
2894          */
2895         sBUG_ON(!list_empty(&cleanup_list));
2896
2897         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2898
2899         TRACE_EXIT();
2900         return 0;
2901 }
2902
2903 static int __init init_scst_user(void)
2904 {
2905         int res = 0;
2906         struct class_device *class_member;
2907
2908         TRACE_ENTRY();
2909
2910 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2911         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2912                 "Consider change VMSPLIT option or use 64-bit "
2913                 "configuration instead. See README file for details.");
2914         res = -EINVAL;
2915         goto out;
2916 #endif
2917
2918         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2919         if (user_cmd_cachep == NULL) {
2920                 res = -ENOMEM;
2921                 goto out;
2922         }
2923
2924         dev_user_devtype.module = THIS_MODULE;
2925
2926         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2927         if (res < 0)
2928                 goto out_cache;
2929
2930         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2931         if (res != 0)
2932                 goto out_unreg;
2933
2934         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2935         if (IS_ERR(dev_user_sysfs_class)) {
2936                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2937                         "space handler");
2938                 res = PTR_ERR(dev_user_sysfs_class);
2939                 goto out_proc;
2940         }
2941
2942         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2943         if (res) {
2944                 PRINT_ERROR("Unable to get major %d for SCSI tapes", DEV_USER_MAJOR);
2945                 goto out_class;
2946         }
2947
2948         class_member = class_device_create(dev_user_sysfs_class, NULL,
2949                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
2950         if (IS_ERR(class_member)) {
2951                 res = PTR_ERR(class_member);
2952                 goto out_chrdev;
2953         }
2954
2955         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
2956                 "scst_usr_cleanupd");
2957         if (IS_ERR(cleanup_thread)) {
2958                 res = PTR_ERR(cleanup_thread);
2959                 PRINT_ERROR("kthread_create() failed: %d", res);
2960                 goto out_dev;
2961         }
2962
2963 out:
2964         TRACE_EXIT_RES(res);
2965         return res;
2966
2967 out_dev:
2968         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2969
2970 out_chrdev:
2971         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2972
2973 out_class:
2974         class_destroy(dev_user_sysfs_class);
2975
2976 out_proc:
2977         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
2978
2979 out_unreg:
2980         scst_unregister_dev_driver(&dev_user_devtype);
2981
2982 out_cache:
2983         kmem_cache_destroy(user_cmd_cachep);
2984         goto out;
2985 }
2986
2987 static void __exit exit_scst_user(void)
2988 {
2989         int rc;
2990
2991         TRACE_ENTRY();
2992
2993         rc = kthread_stop(cleanup_thread);
2994         if (rc < 0)
2995                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2996
2997         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2998         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2999         class_destroy(dev_user_sysfs_class);
3000
3001         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3002         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3003
3004         kmem_cache_destroy(user_cmd_cachep);
3005
3006         TRACE_EXIT();
3007         return;
3008 }
3009
3010 module_init(init_scst_user);
3011 module_exit(exit_scst_user);
3012
3013 MODULE_AUTHOR("Vladislav Bolkhovitin");
3014 MODULE_LICENSE("GPL");
3015 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3016 MODULE_VERSION(SCST_VERSION_STRING);
3017 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);