scst/src/dev_handlers/scst_user.c

   1 /*
   2  *  scst_user.c
   3  *
   4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   6  *
   7  *  SCSI virtual user space device handler
   8  *
   9  *  This program is free software; you can redistribute it and/or
  10  *  modify it under the terms of the GNU General Public License
  11  *  as published by the Free Software Foundation, version 2
  12  *  of the License.
  13  *
  14  *  This program is distributed in the hope that it will be useful,
  15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  *  GNU General Public License for more details.
  18  */
  19
  20 #include <linux/kthread.h>
  21 #include <linux/delay.h>
  22 #include <linux/poll.h>
  23
  24 #define LOG_PREFIX              DEV_USER_NAME
  25
  26 #include "scst.h"
  27 #include "scst_user.h"
  28 #include "scst_dev_handler.h"
  29
  30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
  31 #warning "HIGHMEM kernel configurations are not supported by this module, \
  32         because nowadays it doesn't worth the effort. Consider change \
  33         VMSPLIT option or use 64-bit configuration instead. See README file \
  34         for details."
  35 #endif
  36
  37 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18))
  38 #define COMPLETION_INITIALIZER_ONSTACK(work) \
  39         ({ init_completion(&work); work; })
  40
  41 /*
  42  * Lockdep needs to run a non-constant initializer for on-stack
  43  * completions - so we use the _ONSTACK() variant for those that
  44  * are on the kernel stack:
  45  */
  46 #ifdef CONFIG_LOCKDEP
  47 # define DECLARE_COMPLETION_ONSTACK(work) \
  48         struct completion work = COMPLETION_INITIALIZER_ONSTACK(work)
  49 #else
  50 # define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work)
  51 #endif
  52
  53 #endif
  54
  55 #define DEV_USER_MAJOR                  237
  56
  57 #define DEV_USER_CMD_HASH_ORDER         6
  58
  59 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
  60
  61 struct scst_user_dev {
  62         struct rw_semaphore dev_rwsem;
  63
  64         struct scst_cmd_lists cmd_lists;
  65
  66         /* Protected by cmd_lists.cmd_list_lock */
  67         struct list_head ready_cmd_list;
  68
  69         /* Protected by dev_rwsem or don't need any protection */
  70         unsigned int blocking:1;
  71         unsigned int cleanup_done:1;
  72         unsigned int tst:3;
  73         unsigned int queue_alg:4;
  74         unsigned int tas:1;
  75         unsigned int swp:1;
  76         unsigned int has_own_order_mgmt:1;
  77
  78         int (*generic_parse)(struct scst_cmd *cmd,
  79                 int (*get_block)(struct scst_cmd *cmd));
  80
  81         int block;
  82         int def_block;
  83
  84         struct scst_mem_lim udev_mem_lim;
  85         struct sgv_pool *pool;
  86
  87         uint8_t parse_type;
  88         uint8_t on_free_cmd_type;
  89         uint8_t memory_reuse_type;
  90         uint8_t partial_transfers_type;
  91         uint32_t partial_len;
  92
  93         struct scst_dev_type devtype;
  94
  95         /* Both protected by cmd_lists.cmd_list_lock */
  96         unsigned int handle_counter;
  97         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
  98
  99         struct scst_device *sdev;
 100
 101         int virt_id;
 102         struct list_head dev_list_entry;
 103         char name[SCST_MAX_NAME];
 104
 105         struct list_head cleanup_list_entry;
 106         /* ToDo: make it on-stack */
 107         struct completion cleanup_cmpl;
 108 };
 109
 110 /* Most fields are unprotected, since only one thread at time can access them */
 111 struct scst_user_cmd {
 112         struct scst_cmd *cmd;
 113         struct scst_user_dev *dev;
 114
 115         atomic_t ucmd_ref;
 116
 117         unsigned int buff_cached:1;
 118         unsigned int buf_dirty:1;
 119         unsigned int background_exec:1;
 120         unsigned int aborted:1;
 121
 122         struct scst_user_cmd *buf_ucmd;
 123
 124         int cur_data_page;
 125         int num_data_pages;
 126         int first_page_offset;
 127         unsigned long ubuff;
 128         struct page **data_pages;
 129         struct sgv_pool_obj *sgv;
 130
 131         /*
 132          * Special flags, which can be accessed asynchronously (hence "long").
 133          * Protected by cmd_lists.cmd_list_lock.
 134          */
 135         unsigned long sent_to_user:1;
 136         unsigned long jammed:1;
 137         unsigned long this_state_unjammed:1;
 138         unsigned long seen_by_user:1; /* here only as a small optimization */
 139
 140         unsigned int state;
 141
 142         struct list_head ready_cmd_list_entry;
 143
 144         unsigned int h;
 145         struct list_head hash_list_entry;
 146
 147         struct scst_user_get_cmd user_cmd;
 148
 149         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
 150         union {
 151                 struct completion *cmpl;
 152                 struct scst_mgmt_cmd *mcmd;
 153         };
 154         int result;
 155 };
 156
 157 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 158         gfp_t gfp_mask);
 159 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
 160
 161 static int dev_user_parse(struct scst_cmd *cmd);
 162 static int dev_user_exec(struct scst_cmd *cmd);
 163 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
 164 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
 165         struct scst_tgt_dev *tgt_dev);
 166
 167 static int dev_user_disk_done(struct scst_cmd *cmd);
 168 static int dev_user_tape_done(struct scst_cmd *cmd);
 169
 170 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 171         gfp_t gfp_mask, void *priv);
 172 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 173                                      void *priv);
 174
 175 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
 176
 177 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
 178         unsigned long *flags);
 179
 180 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
 181 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
 182         int status);
 183 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
 184 static int dev_user_register_dev(struct file *file,
 185         const struct scst_user_dev_desc *dev_desc);
 186 static int __dev_user_set_opt(struct scst_user_dev *dev,
 187         const struct scst_user_opt *opt);
 188 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
 189 static int dev_user_get_opt(struct file *file, void __user *arg);
 190
 191 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
 192 static long dev_user_ioctl(struct file *file, unsigned int cmd,
 193         unsigned long arg);
 194 static int dev_user_release(struct inode *inode, struct file *file);
 195
 196 /** Data **/
 197
 198 static struct kmem_cache *user_cmd_cachep;
 199 static struct kmem_cache *user_get_cmd_cachep;
 200
 201 static DEFINE_MUTEX(dev_priv_mutex);
 202
 203 static struct file_operations dev_user_fops = {
 204         .poll           = dev_user_poll,
 205         .unlocked_ioctl = dev_user_ioctl,
 206 #ifdef CONFIG_COMPAT
 207         .compat_ioctl   = dev_user_ioctl,
 208 #endif
 209         .release        = dev_user_release,
 210 };
 211
 212 static struct class *dev_user_sysfs_class;
 213
 214 static DEFINE_SPINLOCK(dev_list_lock);
 215 static LIST_HEAD(dev_list);
 216
 217 static DEFINE_SPINLOCK(cleanup_lock);
 218 static LIST_HEAD(cleanup_list);
 219 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
 220 static struct task_struct *cleanup_thread;
 221
 222 /*
 223  * Skip this command if result is not 0. Must be called under
 224  * cmd_lists.cmd_list_lock and IRQ off.
 225  */
 226 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
 227 {
 228         int r = atomic_inc_return(&ucmd->ucmd_ref);
 229         int res;
 230         if (unlikely(r == 1)) {
 231                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
 232                 atomic_dec(&ucmd->ucmd_ref);
 233                 res = true;
 234                 /*
 235                  * Necessary code is serialized by cmd_list_lock in
 236                  * cmd_remove_hash()
 237                  */
 238         } else {
 239                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
 240                         atomic_read(&ucmd->ucmd_ref));
 241                 res = false;
 242         }
 243         return res;
 244 }
 245
 246 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
 247 {
 248         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 249         atomic_inc(&ucmd->ucmd_ref);
 250         if (barrier)
 251                 smp_mb__after_atomic_inc();
 252 }
 253
 254 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
 255 {
 256         __ucmd_get(ucmd, true);
 257 }
 258
 259 static inline void ucmd_get(struct scst_user_cmd *ucmd)
 260 {
 261         __ucmd_get(ucmd, false);
 262 }
 263
 264 /* Must not be called under cmd_list_lock!! */
 265 static inline void ucmd_put(struct scst_user_cmd *ucmd)
 266 {
 267         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 268
 269         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
 270
 271         if (atomic_dec_and_test(&ucmd->ucmd_ref))
 272                 dev_user_free_ucmd(ucmd);
 273 }
 274
 275 static inline int calc_num_pg(unsigned long buf, int len)
 276 {
 277         len += buf & ~PAGE_MASK;
 278         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
 279 }
 280
 281 static inline int is_need_offs_page(unsigned long buf, int len)
 282 {
 283         return ((buf & ~PAGE_MASK) != 0) &&
 284                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
 285 }
 286
 287 static void __dev_user_not_reg(void)
 288 {
 289         PRINT_ERROR("%s", "Device not registered");
 290         return;
 291 }
 292
 293 static inline int dev_user_check_reg(struct scst_user_dev *dev)
 294 {
 295         if (dev == NULL) {
 296                 __dev_user_not_reg();
 297                 return -EINVAL;
 298         }
 299         return 0;
 300 }
 301
 302 static inline int scst_user_cmd_hashfn(int h)
 303 {
 304         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
 305 }
 306
 307 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
 308         unsigned int h)
 309 {
 310         struct list_head *head;
 311         struct scst_user_cmd *ucmd;
 312
 313         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
 314         list_for_each_entry(ucmd, head, hash_list_entry) {
 315                 if (ucmd->h == h) {
 316                         TRACE_DBG("Found ucmd %p", ucmd);
 317                         return ucmd;
 318                 }
 319         }
 320         return NULL;
 321 }
 322
 323 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
 324 {
 325         struct list_head *head;
 326         struct scst_user_dev *dev = ucmd->dev;
 327         struct scst_user_cmd *u;
 328         unsigned long flags;
 329
 330         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
 331         do {
 332                 ucmd->h = dev->handle_counter++;
 333                 u = __ucmd_find_hash(dev, ucmd->h);
 334         } while (u != NULL);
 335         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
 336         list_add_tail(&ucmd->hash_list_entry, head);
 337         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
 338
 339         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
 340         return;
 341 }
 342
 343 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
 344 {
 345         unsigned long flags;
 346
 347         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 348         list_del(&ucmd->hash_list_entry);
 349         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 350
 351         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
 352         return;
 353 }
 354
 355 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
 356 {
 357         TRACE_ENTRY();
 358
 359         TRACE_MEM("Freeing ucmd %p", ucmd);
 360
 361         cmd_remove_hash(ucmd);
 362         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
 363
 364         kmem_cache_free(user_cmd_cachep, ucmd);
 365
 366         TRACE_EXIT();
 367         return;
 368 }
 369
 370 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 371         gfp_t gfp_mask, void *priv)
 372 {
 373         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 374         int offset = 0;
 375
 376         TRACE_ENTRY();
 377
 378         /* *sg supposed to be zeroed */
 379
 380         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
 381                 ucmd->ubuff, ucmd->cur_data_page);
 382
 383         if (ucmd->cur_data_page == 0) {
 384                 TRACE_MEM("ucmd->first_page_offset %d",
 385                         ucmd->first_page_offset);
 386                 offset = ucmd->first_page_offset;
 387                 ucmd_get(ucmd);
 388         }
 389
 390         if (ucmd->cur_data_page >= ucmd->num_data_pages)
 391                 goto out;
 392
 393         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
 394                 PAGE_SIZE - offset, offset);
 395         ucmd->cur_data_page++;
 396
 397         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
 398                 sg->offset);
 399         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
 400
 401 out:
 402         TRACE_EXIT();
 403         return sg_page(sg);
 404 }
 405
 406 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
 407 {
 408         TRACE_ENTRY();
 409
 410         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
 411                 ucmd, ucmd->h, ucmd->ubuff);
 412
 413         ucmd->user_cmd.cmd_h = ucmd->h;
 414         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
 415         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
 416
 417         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
 418
 419         dev_user_add_to_ready(ucmd);
 420
 421         TRACE_EXIT();
 422         return;
 423 }
 424
 425 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
 426 {
 427         int i;
 428
 429         TRACE_ENTRY();
 430
 431         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
 432                 ucmd->ubuff, ucmd->num_data_pages);
 433
 434         for (i = 0; i < ucmd->num_data_pages; i++) {
 435                 struct page *page = ucmd->data_pages[i];
 436
 437                 if (ucmd->buf_dirty)
 438                         SetPageDirty(page);
 439
 440                 page_cache_release(page);
 441         }
 442
 443         kfree(ucmd->data_pages);
 444         ucmd->data_pages = NULL;
 445
 446         TRACE_EXIT();
 447         return;
 448 }
 449
 450 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
 451 {
 452         TRACE_ENTRY();
 453
 454         sBUG_ON(ucmd->data_pages == NULL);
 455
 456         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
 457                 ucmd, ucmd->ubuff, ucmd->buff_cached);
 458
 459         dev_user_unmap_buf(ucmd);
 460
 461         if (ucmd->buff_cached)
 462                 dev_user_on_cached_mem_free(ucmd);
 463         else
 464                 ucmd_put(ucmd);
 465
 466         TRACE_EXIT();
 467         return;
 468 }
 469
 470 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 471         void *priv)
 472 {
 473         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 474
 475         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
 476                 sg_count, ucmd);
 477
 478         __dev_user_free_sg_entries(ucmd);
 479
 480         return;
 481 }
 482
 483 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
 484 {
 485         int mem_reuse_type = ucmd->dev->memory_reuse_type;
 486
 487         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
 488             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
 489              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
 490             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
 491              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
 492                 return 1;
 493         else
 494                 return 0;
 495 }
 496
 497 /*
 498  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
 499  * Unmaps the buffer, if needed in case of error
 500  */
 501 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 502 {
 503         int res = 0;
 504         struct scst_cmd *cmd = ucmd->cmd;
 505         struct scst_user_dev *dev = ucmd->dev;
 506         gfp_t gfp_mask;
 507         int flags = 0;
 508         int bufflen = cmd->bufflen;
 509         int last_len = 0;
 510
 511         TRACE_ENTRY();
 512
 513         sBUG_ON(bufflen == 0);
 514
 515         gfp_mask = __GFP_NOWARN;
 516         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
 517
 518         if (cached_buff) {
 519                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
 520                 if (ucmd->ubuff == 0)
 521                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
 522         } else {
 523                 TRACE_MEM("%s", "Not cached buff");
 524                 flags |= SCST_POOL_ALLOC_NO_CACHED;
 525                 if (ucmd->ubuff == 0) {
 526                         res = 1;
 527                         goto out;
 528                 }
 529                 bufflen += ucmd->first_page_offset;
 530                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
 531                         last_len = bufflen & ~PAGE_MASK;
 532                 else
 533                         last_len = cmd->bufflen & ~PAGE_MASK;
 534                 if (last_len == 0)
 535                         last_len = PAGE_SIZE;
 536         }
 537         ucmd->buff_cached = cached_buff;
 538
 539         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
 540                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
 541         if (cmd->sg != NULL) {
 542                 struct scst_user_cmd *buf_ucmd =
 543                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
 544
 545                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
 546
 547                 ucmd->ubuff = buf_ucmd->ubuff;
 548                 ucmd->buf_ucmd = buf_ucmd;
 549
 550                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
 551                                    (ucmd != buf_ucmd));
 552
 553                 if (last_len != 0) {
 554                         /* We don't use clustering, so the assignment is safe */
 555                         cmd->sg[cmd->sg_cnt-1].length = last_len;
 556                 }
 557
 558                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
 559                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
 560                         last_len, cmd->sg[cmd->sg_cnt-1].length);
 561
 562                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
 563                         static int ll;
 564                         if (ll < 10) {
 565                                 PRINT_INFO("Unable to complete command due to "
 566                                         "SG IO count limitation (requested %d, "
 567                                         "available %d, tgt lim %d)",
 568                                         cmd->sg_cnt,
 569                                         cmd->tgt_dev->max_sg_cnt,
 570                                         cmd->tgt->sg_tablesize);
 571                                 ll++;
 572                         }
 573                         cmd->sg = NULL;
 574                         /* sgv will be freed in dev_user_free_sgv() */
 575                         res = -1;
 576                 }
 577         } else {
 578                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
 579                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
 580                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
 581                 if (unlikely(cmd->sg_cnt == 0)) {
 582                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
 583                         sBUG_ON(ucmd->sgv != NULL);
 584                         res = -1;
 585                 } else {
 586                         switch (ucmd->state) {
 587                         case UCMD_STATE_BUF_ALLOCING:
 588                                 res = 1;
 589                                 break;
 590                         case UCMD_STATE_EXECING:
 591                                 res = -1;
 592                                 break;
 593                         default:
 594                                 sBUG();
 595                                 break;
 596                         }
 597                 }
 598         }
 599
 600 out:
 601         TRACE_EXIT_RES(res);
 602         return res;
 603 }
 604
 605 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
 606 {
 607         int rc, res = SCST_CMD_STATE_DEFAULT;
 608         struct scst_cmd *cmd = ucmd->cmd;
 609
 610         TRACE_ENTRY();
 611
 612         ucmd->state = UCMD_STATE_BUF_ALLOCING;
 613         cmd->dh_data_buf_alloced = 1;
 614
 615         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
 616         if (rc == 0)
 617                 goto out;
 618         else if (rc < 0) {
 619                 scst_set_busy(cmd);
 620                 res = scst_get_cmd_abnormal_done_state(cmd);
 621                 goto out;
 622         }
 623
 624         if ((cmd->data_direction != SCST_DATA_WRITE) &&
 625             !scst_is_cmd_local(cmd)) {
 626                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
 627                 goto out;
 628         }
 629
 630         ucmd->user_cmd.cmd_h = ucmd->h;
 631         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
 632         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 633         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
 634                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
 635         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
 636         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
 637                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 638         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
 639         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
 640         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
 641
 642         dev_user_add_to_ready(ucmd);
 643
 644         res = SCST_CMD_STATE_STOP;
 645
 646 out:
 647         TRACE_EXIT_RES(res);
 648         return res;
 649 }
 650
 651 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 652         gfp_t gfp_mask)
 653 {
 654         struct scst_user_cmd *ucmd = NULL;
 655
 656         TRACE_ENTRY();
 657
 658 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
 659         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
 660         if (ucmd != NULL)
 661                 memset(ucmd, 0, sizeof(*ucmd));
 662 #else
 663         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
 664 #endif
 665         if (unlikely(ucmd == NULL)) {
 666                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
 667                         "user cmd (gfp_mask %x)", gfp_mask);
 668                 goto out;
 669         }
 670         ucmd->dev = dev;
 671         atomic_set(&ucmd->ucmd_ref, 1);
 672
 673         cmd_insert_hash(ucmd);
 674
 675         TRACE_MEM("ucmd %p allocated", ucmd);
 676
 677 out:
 678         TRACE_EXIT_HRES((unsigned long)ucmd);
 679         return ucmd;
 680 }
 681
 682 static int dev_user_get_block(struct scst_cmd *cmd)
 683 {
 684         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 685         /*
 686          * No need for locks here, since *_detach() can not be
 687          * called, when there are existing commands.
 688          */
 689         TRACE_EXIT_RES(dev->block);
 690         return dev->block;
 691 }
 692
 693 static int dev_user_parse(struct scst_cmd *cmd)
 694 {
 695         int rc, res = SCST_CMD_STATE_DEFAULT;
 696         struct scst_user_cmd *ucmd;
 697         int atomic = scst_cmd_atomic(cmd);
 698         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 699         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
 700
 701         TRACE_ENTRY();
 702
 703         if (cmd->dh_priv == NULL) {
 704                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
 705                 if (unlikely(ucmd == NULL)) {
 706                         if (atomic) {
 707                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
 708                                 goto out;
 709                         } else {
 710                                 scst_set_busy(cmd);
 711                                 goto out_error;
 712                         }
 713                 }
 714                 ucmd->cmd = cmd;
 715                 cmd->dh_priv = ucmd;
 716         } else {
 717                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 718                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
 719         }
 720
 721         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
 722
 723         if (ucmd->state != UCMD_STATE_NEW)
 724                 goto alloc;
 725
 726         switch (dev->parse_type) {
 727         case SCST_USER_PARSE_STANDARD:
 728                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
 729                 rc = dev->generic_parse(cmd, dev_user_get_block);
 730                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
 731                         goto out_invalid;
 732                 break;
 733
 734         case SCST_USER_PARSE_EXCEPTION:
 735                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
 736                 rc = dev->generic_parse(cmd, dev_user_get_block);
 737                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
 738                         break;
 739                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
 740                         TRACE_MEM("Restarting PARSE to thread context "
 741                                 "(ucmd %p)", ucmd);
 742                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
 743                         goto out;
 744                 }
 745                 /* else go through */
 746
 747         case SCST_USER_PARSE_CALL:
 748                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
 749                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
 750                 ucmd->user_cmd.cmd_h = ucmd->h;
 751                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
 752                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 753                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
 754                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
 755                             sizeof(cmd->cdb)));
 756                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
 757                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
 758                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
 759                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
 760                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
 761                 ucmd->user_cmd.parse_cmd.expected_values_set =
 762                                         cmd->expected_values_set;
 763                 ucmd->user_cmd.parse_cmd.expected_data_direction =
 764                                         cmd->expected_data_direction;
 765                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
 766                                         cmd->expected_transfer_len;
 767                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
 768                 ucmd->state = UCMD_STATE_PARSING;
 769                 dev_user_add_to_ready(ucmd);
 770                 res = SCST_CMD_STATE_STOP;
 771                 goto out;
 772
 773         default:
 774                 sBUG();
 775                 goto out;
 776         }
 777
 778 alloc:
 779         if (cmd->data_direction != SCST_DATA_NONE)
 780                 res = dev_user_alloc_space(ucmd);
 781
 782 out:
 783         TRACE_EXIT_RES(res);
 784         return res;
 785
 786 out_invalid:
 787         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
 788                 cmd->op_flags & SCST_INFO_INVALID);
 789         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 790
 791 out_error:
 792         res = scst_get_cmd_abnormal_done_state(cmd);
 793         goto out;
 794 }
 795
 796 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
 797 {
 798         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
 799         unsigned long start = buf_ucmd->ubuff;
 800         int i, bufflen = ucmd->cmd->bufflen;
 801
 802         TRACE_ENTRY();
 803
 804         if (start == 0)
 805                 goto out;
 806
 807         /*
 808          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
 809          * faster, since it should be cache hot, while ucmd->buf_ucmd and
 810          * buf_ucmd->data_pages are cache cold. But, from other side,
 811          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
 812          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
 813          * data_pages array can lead to less cache misses. So, real numbers are
 814          * needed. ToDo.
 815          */
 816
 817         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
 818                 struct page *page;
 819                 page = buf_ucmd->data_pages[i];
 820 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
 821                 struct vm_area_struct *vma = find_vma(current->mm, start);
 822                 if (vma != NULL)
 823                         flush_anon_page(vma, page, start);
 824 #endif
 825                 flush_dcache_page(page);
 826                 start += PAGE_SIZE;
 827                 bufflen -= PAGE_SIZE;
 828         }
 829
 830 out:
 831         TRACE_EXIT();
 832         return;
 833 }
 834
 835 static int dev_user_exec(struct scst_cmd *cmd)
 836 {
 837         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 838         int res = SCST_EXEC_COMPLETED;
 839
 840         TRACE_ENTRY();
 841
 842 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
 843        * context to complete the necessary actions, but all we are going to
 844        * do in this function is, in fact, atomic, so let's skip this check.
 845        */
 846         if (scst_cmd_atomic(cmd)) {
 847                 TRACE_DBG("%s", "User exec() can not be called in atomic "
 848                         "context, rescheduling to the thread");
 849                 res = SCST_EXEC_NEED_THREAD;
 850                 goto out;
 851         }
 852 #endif
 853
 854         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
 855                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
 856                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
 857
 858         if (cmd->data_direction == SCST_DATA_WRITE)
 859                 dev_user_flush_dcache(ucmd);
 860
 861         ucmd->user_cmd.cmd_h = ucmd->h;
 862         ucmd->user_cmd.subcode = SCST_USER_EXEC;
 863         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 864         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
 865                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
 866                     sizeof(cmd->cdb)));
 867         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
 868         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
 869         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
 870         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
 871         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
 872                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
 873                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 874         }
 875         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
 876         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
 877         ucmd->user_cmd.exec_cmd.partial = 0;
 878         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
 879         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
 880
 881         ucmd->state = UCMD_STATE_EXECING;
 882
 883         dev_user_add_to_ready(ucmd);
 884
 885         TRACE_EXIT_RES(res);
 886         return res;
 887 }
 888
 889 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
 890 {
 891         if (ucmd->sgv != NULL) {
 892                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
 893                 ucmd->sgv = NULL;
 894         } else if (ucmd->data_pages != NULL) {
 895                 /* We mapped pages, but for some reason didn't allocate them */
 896                 ucmd_get(ucmd);
 897                 __dev_user_free_sg_entries(ucmd);
 898         }
 899         return;
 900 }
 901
 902 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
 903 {
 904         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 905
 906         TRACE_ENTRY();
 907
 908         if (unlikely(ucmd == NULL))
 909                 goto out;
 910
 911         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
 912                 ucmd->buff_cached, ucmd->ubuff);
 913
 914         ucmd->cmd = NULL;
 915         if (cmd->data_direction == SCST_DATA_WRITE && ucmd->buf_ucmd != NULL)
 916                 ucmd->buf_ucmd->buf_dirty = 1;
 917
 918         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
 919                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
 920                 /* The state assignment must be before freeing sgv! */
 921                 goto out_reply;
 922         }
 923
 924         if (unlikely(!ucmd->seen_by_user)) {
 925                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
 926                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
 927                 goto out_reply;
 928         }
 929
 930         ucmd->user_cmd.cmd_h = ucmd->h;
 931         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
 932
 933         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
 934         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
 935         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
 936         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
 937         ucmd->user_cmd.on_free_cmd.status = cmd->status;
 938         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
 939
 940         ucmd->state = UCMD_STATE_ON_FREEING;
 941
 942         dev_user_add_to_ready(ucmd);
 943
 944 out:
 945         TRACE_EXIT();
 946         return;
 947
 948 out_reply:
 949         dev_user_process_reply_on_free(ucmd);
 950         goto out;
 951 }
 952
 953 static void dev_user_set_block(struct scst_cmd *cmd, int block)
 954 {
 955         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 956         /*
 957          * No need for locks here, since *_detach() can not be
 958          * called, when there are existing commands.
 959          */
 960         TRACE_DBG("dev %p, new block %d", dev, block);
 961         if (block != 0)
 962                 dev->block = block;
 963         else
 964                 dev->block = dev->def_block;
 965         return;
 966 }
 967
 968 static int dev_user_disk_done(struct scst_cmd *cmd)
 969 {
 970         int res = SCST_CMD_STATE_DEFAULT;
 971
 972         TRACE_ENTRY();
 973
 974         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
 975
 976         TRACE_EXIT_RES(res);
 977         return res;
 978 }
 979
 980 static int dev_user_tape_done(struct scst_cmd *cmd)
 981 {
 982         int res = SCST_CMD_STATE_DEFAULT;
 983
 984         TRACE_ENTRY();
 985
 986         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
 987
 988         TRACE_EXIT_RES(res);
 989         return res;
 990 }
 991
 992 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
 993 {
 994         struct scst_user_dev *dev = ucmd->dev;
 995         unsigned long flags;
 996         int do_wake;
 997
 998         TRACE_ENTRY();
 999
1000         do_wake = (in_interrupt() ||
1001                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
1002         if (ucmd->cmd)
1003                 do_wake |= ucmd->cmd->preprocessing_only;
1004
1005         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1006
1007         ucmd->this_state_unjammed = 0;
1008
1009         if ((ucmd->state == UCMD_STATE_PARSING) ||
1010             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1011                 /*
1012                  * If we don't put such commands in the queue head, then under
1013                  * high load we might delay threads, waiting for memory
1014                  * allocations, for too long and start loosing NOPs, which
1015                  * would lead to consider us by remote initiators as
1016                  * unresponsive and stuck => broken connections, etc. If none
1017                  * of our commands completed in NOP timeout to allow the head
1018                  * commands to go, then we are really overloaded and/or stuck.
1019                  */
1020                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1021                         "cmd list", ucmd, ucmd->state);
1022                 list_add(&ucmd->ready_cmd_list_entry,
1023                         &dev->ready_cmd_list);
1024                 do_wake = 1;
1025         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1026                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1027                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1028                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1029                         "ready cmd list", ucmd, ucmd->state);
1030                 list_add(&ucmd->ready_cmd_list_entry,
1031                         &dev->ready_cmd_list);
1032                 do_wake = 1;
1033         } else if ((ucmd->cmd != NULL) &&
1034            unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1035                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1036                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1037         } else {
1038                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1039                 list_add_tail(&ucmd->ready_cmd_list_entry,
1040                               &dev->ready_cmd_list);
1041         }
1042
1043         if (do_wake) {
1044                 TRACE_DBG("Waking up dev %p", dev);
1045                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1046         }
1047
1048         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1049
1050         TRACE_EXIT();
1051         return;
1052 }
1053
1054 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1055         int num_pg)
1056 {
1057         int res = 0, rc;
1058         int i;
1059         struct task_struct *tsk = current;
1060
1061         TRACE_ENTRY();
1062
1063         if (unlikely(ubuff == 0))
1064                 goto out_nomem;
1065
1066         sBUG_ON(ucmd->data_pages != NULL);
1067
1068         ucmd->num_data_pages = num_pg;
1069
1070         ucmd->data_pages =
1071                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1072                           GFP_KERNEL);
1073         if (ucmd->data_pages == NULL) {
1074                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1075                         "(num_data_pages=%d)", ucmd->num_data_pages);
1076                 res = -ENOMEM;
1077                 goto out_nomem;
1078         }
1079
1080         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1081                 " first_page_offset %d, len %d)", ucmd, ubuff,
1082                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1083                 ucmd->cmd->bufflen);
1084
1085         down_read(&tsk->mm->mmap_sem);
1086         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1087                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1088         up_read(&tsk->mm->mmap_sem);
1089
1090         /* get_user_pages() flushes dcache */
1091
1092         if (rc < ucmd->num_data_pages)
1093                 goto out_unmap;
1094
1095         ucmd->ubuff = ubuff;
1096         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1097
1098 out:
1099         TRACE_EXIT_RES(res);
1100         return res;
1101
1102 out_nomem:
1103         scst_set_busy(ucmd->cmd);
1104         /* go through */
1105
1106 out_err:
1107         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1108         goto out;
1109
1110 out_unmap:
1111         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1112                 ucmd->num_data_pages, rc);
1113         if (rc > 0) {
1114                 for (i = 0; i < rc; i++)
1115                         page_cache_release(ucmd->data_pages[i]);
1116         }
1117         kfree(ucmd->data_pages);
1118         ucmd->data_pages = NULL;
1119         res = -EFAULT;
1120         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1121         goto out_err;
1122 }
1123
1124 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1125         struct scst_user_reply_cmd *reply)
1126 {
1127         int res = 0;
1128         struct scst_cmd *cmd = ucmd->cmd;
1129
1130         TRACE_ENTRY();
1131
1132         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1133
1134         if (likely(reply->alloc_reply.pbuf != 0)) {
1135                 int pages;
1136                 if (ucmd->buff_cached) {
1137                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1138                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1139                                         "page aligned",
1140                                         reply->alloc_reply.pbuf);
1141                                 goto out_hwerr;
1142                         }
1143                         pages = cmd->sg_cnt;
1144                 } else
1145                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1146                                             cmd->bufflen);
1147                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1148         } else {
1149                 scst_set_busy(ucmd->cmd);
1150                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1151         }
1152
1153 out_process:
1154         scst_process_active_cmd(cmd, false);
1155
1156         TRACE_EXIT_RES(res);
1157         return res;
1158
1159 out_hwerr:
1160         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1161         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1162         res = -EINVAL;
1163         goto out_process;
1164 }
1165
1166 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1167         struct scst_user_reply_cmd *reply)
1168 {
1169         int res = 0;
1170         struct scst_user_scsi_cmd_reply_parse *preply =
1171                 &reply->parse_reply;
1172         struct scst_cmd *cmd = ucmd->cmd;
1173
1174         TRACE_ENTRY();
1175
1176         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1177                 goto out_inval;
1178
1179         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1180                      (preply->data_direction != SCST_DATA_READ) &&
1181                      (preply->data_direction != SCST_DATA_NONE)))
1182                 goto out_inval;
1183
1184         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1185                      (preply->bufflen == 0)))
1186                 goto out_inval;
1187
1188         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1189                 goto out_inval;
1190
1191         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1192                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1193                 preply->data_direction, preply->bufflen, preply->data_len,
1194                 reply->alloc_reply.pbuf);
1195
1196         cmd->queue_type = preply->queue_type;
1197         cmd->data_direction = preply->data_direction;
1198         cmd->bufflen = preply->bufflen;
1199         cmd->data_len = preply->data_len;
1200
1201 out_process:
1202         scst_process_active_cmd(cmd, false);
1203
1204         TRACE_EXIT_RES(res);
1205         return res;
1206
1207 out_inval:
1208         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1209                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1210         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1211         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1212         scst_set_cmd_abnormal_done_state(cmd);
1213         res = -EINVAL;
1214         goto out_process;
1215 }
1216
1217 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1218 {
1219         int res = 0;
1220
1221         TRACE_ENTRY();
1222
1223         TRACE_DBG("ON FREE ucmd %p", ucmd);
1224
1225         dev_user_free_sgv(ucmd);
1226         ucmd_put(ucmd);
1227
1228         TRACE_EXIT_RES(res);
1229         return res;
1230 }
1231
1232 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1233 {
1234         int res = 0;
1235
1236         TRACE_ENTRY();
1237
1238         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1239
1240         ucmd_put(ucmd);
1241
1242         TRACE_EXIT_RES(res);
1243         return res;
1244 }
1245
1246 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1247         struct scst_user_reply_cmd *reply)
1248 {
1249         int res = 0;
1250         struct scst_user_scsi_cmd_reply_exec *ereply =
1251                 &reply->exec_reply;
1252         struct scst_cmd *cmd = ucmd->cmd;
1253
1254         TRACE_ENTRY();
1255
1256         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1257                 if (ucmd->background_exec) {
1258                         TRACE_DBG("Background ucmd %p finished", ucmd);
1259                         ucmd_put(ucmd);
1260                         goto out;
1261                 }
1262                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1263                         goto out_inval;
1264                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1265                              (ereply->resp_data_len != 0)))
1266                         goto out_inval;
1267         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1268                 if (unlikely(ucmd->background_exec))
1269                         goto out_inval;
1270                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1271                              (cmd->resp_data_len != 0)))
1272                         goto out_inval;
1273                 ucmd_get_ordered(ucmd);
1274                 ucmd->background_exec = 1;
1275                 TRACE_DBG("Background ucmd %p", ucmd);
1276                 goto out_compl;
1277         } else
1278                 goto out_inval;
1279
1280         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1281                 ereply->status, ereply->resp_data_len);
1282
1283          if (ereply->resp_data_len != 0) {
1284                 if (ucmd->ubuff == 0) {
1285                         int pages, rc;
1286                         if (unlikely(ereply->pbuf == 0))
1287                                 goto out_busy;
1288                         if (ucmd->buff_cached) {
1289                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1290                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1291                                                 "page aligned", ereply->pbuf);
1292                                         goto out_hwerr;
1293                                 }
1294                                 pages = cmd->sg_cnt;
1295                         } else
1296                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1297                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1298                         if ((rc != 0) || (ucmd->ubuff == 0))
1299                                 goto out_compl;
1300
1301                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1302                         if (unlikely(rc != 0))
1303                                 goto out_busy;
1304                 } else
1305                         dev_user_flush_dcache(ucmd);
1306                 cmd->may_need_dma_sync = 1;
1307                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1308         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1309                 if (ucmd->ubuff == 0)
1310                         cmd->resp_data_len = ereply->resp_data_len;
1311                 else
1312                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1313         }
1314
1315         cmd->status = ereply->status;
1316         if (ereply->sense_len != 0) {
1317                 res = scst_alloc_sense(cmd, 0);
1318                 if (res != 0)
1319                         goto out_compl;
1320                 res = copy_from_user(cmd->sense,
1321                         (void __user *)(unsigned long)ereply->psense_buffer,
1322                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1323                                 (unsigned int)ereply->sense_len));
1324                 if (res < 0) {
1325                         PRINT_ERROR("%s", "Unable to get sense data");
1326                         goto out_hwerr_res_set;
1327                 }
1328         }
1329
1330 out_compl:
1331         cmd->completed = 1;
1332         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1333         /* !! At this point cmd can be already freed !! */
1334
1335 out:
1336         TRACE_EXIT_RES(res);
1337         return res;
1338
1339 out_inval:
1340         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1341                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1342         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1343
1344 out_hwerr:
1345         res = -EINVAL;
1346
1347 out_hwerr_res_set:
1348         if (ucmd->background_exec) {
1349                 ucmd_put(ucmd);
1350                 goto out;
1351         } else {
1352                 scst_set_cmd_error(cmd,
1353                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1354                 goto out_compl;
1355         }
1356
1357 out_busy:
1358         scst_set_busy(cmd);
1359         goto out_compl;
1360 }
1361
1362 static int dev_user_process_reply(struct scst_user_dev *dev,
1363         struct scst_user_reply_cmd *reply)
1364 {
1365         int res = 0;
1366         struct scst_user_cmd *ucmd;
1367         int state;
1368
1369         TRACE_ENTRY();
1370
1371         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1372
1373         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1374         if (unlikely(ucmd == NULL)) {
1375                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1376                 res = -ESRCH;
1377                 goto out_unlock;
1378         }
1379
1380         if (unlikely(ucmd_get_check(ucmd))) {
1381                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1382                 res = -ESRCH;
1383                 goto out_unlock;
1384         }
1385
1386         if (ucmd->background_exec) {
1387                 state = UCMD_STATE_EXECING;
1388                 goto unlock_process;
1389         }
1390
1391         if (unlikely(ucmd->this_state_unjammed)) {
1392                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1393                         ucmd);
1394                 goto out_unlock_put;
1395         }
1396
1397         if (unlikely(!ucmd->sent_to_user)) {
1398                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1399                         "state %x", ucmd, ucmd->state);
1400                 res = -EINVAL;
1401                 goto out_unlock_put;
1402         }
1403
1404         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1405                 goto out_wrong_state;
1406
1407         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1408                 goto out_wrong_state;
1409
1410         state = ucmd->state;
1411         ucmd->sent_to_user = 0;
1412
1413 unlock_process:
1414         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1415
1416         switch (state) {
1417         case UCMD_STATE_PARSING:
1418                 res = dev_user_process_reply_parse(ucmd, reply);
1419                 break;
1420
1421         case UCMD_STATE_BUF_ALLOCING:
1422                 res = dev_user_process_reply_alloc(ucmd, reply);
1423                 break;
1424
1425         case UCMD_STATE_EXECING:
1426                 res = dev_user_process_reply_exec(ucmd, reply);
1427                 break;
1428
1429         case UCMD_STATE_ON_FREEING:
1430                 res = dev_user_process_reply_on_free(ucmd);
1431                 break;
1432
1433         case UCMD_STATE_ON_CACHE_FREEING:
1434                 res = dev_user_process_reply_on_cache_free(ucmd);
1435                 break;
1436
1437         case UCMD_STATE_TM_EXECING:
1438                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1439                 break;
1440
1441         case UCMD_STATE_ATTACH_SESS:
1442         case UCMD_STATE_DETACH_SESS:
1443                 res = dev_user_process_reply_sess(ucmd, reply->result);
1444                 break;
1445
1446         default:
1447                 sBUG();
1448                 break;
1449         }
1450
1451 out_put:
1452         ucmd_put(ucmd);
1453
1454 out:
1455         TRACE_EXIT_RES(res);
1456         return res;
1457
1458 out_wrong_state:
1459         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1460                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1461                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1462                 reply->subcode, ucmd->user_cmd.subcode);
1463         res = -EINVAL;
1464         dev_user_unjam_cmd(ucmd, 0, NULL);
1465
1466 out_unlock_put:
1467         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1468         goto out_put;
1469
1470 out_unlock:
1471         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1472         goto out;
1473 }
1474
1475 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1476 {
1477         int res = 0;
1478         struct scst_user_dev *dev;
1479         struct scst_user_reply_cmd reply;
1480
1481         TRACE_ENTRY();
1482
1483         mutex_lock(&dev_priv_mutex);
1484         dev = (struct scst_user_dev *)file->private_data;
1485         res = dev_user_check_reg(dev);
1486         if (res != 0) {
1487                 mutex_unlock(&dev_priv_mutex);
1488                 goto out;
1489         }
1490         down_read(&dev->dev_rwsem);
1491         mutex_unlock(&dev_priv_mutex);
1492
1493         res = copy_from_user(&reply, arg, sizeof(reply));
1494         if (res < 0)
1495                 goto out_up;
1496
1497         TRACE_MGMT_DBG("Reply for dev %s", dev->name);
1498
1499         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1500
1501         res = dev_user_process_reply(dev, &reply);
1502         if (res < 0)
1503                 goto out_up;
1504
1505 out_up:
1506         up_read(&dev->dev_rwsem);
1507
1508 out:
1509         TRACE_EXIT_RES(res);
1510         return res;
1511 }
1512
1513 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1514         __releases(&dev->cmd_lists.cmd_list_lock)
1515         __acquires(&dev->cmd_lists.cmd_list_lock)
1516 {
1517         int res = 0;
1518
1519         TRACE_ENTRY();
1520
1521         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1522                 struct scst_cmd *cmd = list_entry(
1523                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1524                         cmd_list_entry);
1525                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1526                 list_del(&cmd->cmd_list_entry);
1527                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1528                 scst_process_active_cmd(cmd, false);
1529                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1530                 res++;
1531         }
1532
1533         TRACE_EXIT_RES(res);
1534         return res;
1535 }
1536
1537 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1538 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1539         __releases(&dev->cmd_lists.cmd_list_lock)
1540         __acquires(&dev->cmd_lists.cmd_list_lock)
1541 {
1542         struct scst_user_cmd *u;
1543
1544 again:
1545         u = NULL;
1546         if (!list_empty(cmd_list)) {
1547                 u = list_entry(cmd_list->next, typeof(*u),
1548                                ready_cmd_list_entry);
1549
1550                 TRACE_DBG("Found ready ucmd %p", u);
1551                 list_del(&u->ready_cmd_list_entry);
1552
1553                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1554
1555                 if (u->cmd != NULL) {
1556                         if (u->state == UCMD_STATE_EXECING) {
1557                                 struct scst_user_dev *dev = u->dev;
1558                                 int rc;
1559
1560                                 EXTRACHECKS_BUG_ON(u->jammed);
1561
1562                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1563
1564                                 rc = scst_check_local_events(u->cmd);
1565                                 if (unlikely(rc != 0)) {
1566                                         u->cmd->scst_cmd_done(u->cmd,
1567                                                 SCST_CMD_STATE_DEFAULT,
1568                                                 SCST_CONTEXT_DIRECT);
1569                                         /*
1570                                          * !! At this point cmd & u can be !!
1571                                          * !! already freed                !!
1572                                          */
1573                                         spin_lock_irq(
1574                                                 &dev->cmd_lists.cmd_list_lock);
1575                                         goto again;
1576                                 }
1577
1578                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1579                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1580                                         &u->cmd->cmd_flags))) {
1581                                 switch (u->state) {
1582                                 case UCMD_STATE_PARSING:
1583                                 case UCMD_STATE_BUF_ALLOCING:
1584                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1585                                         dev_user_unjam_cmd(u, 0, NULL);
1586                                         goto again;
1587                                 case UCMD_STATE_EXECING:
1588                                         EXTRACHECKS_BUG_ON(1);
1589                                 }
1590                         }
1591                 }
1592                 u->sent_to_user = 1;
1593                 u->seen_by_user = 1;
1594         }
1595         return u;
1596 }
1597
1598 static inline int test_cmd_lists(struct scst_user_dev *dev)
1599 {
1600         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1601                   !list_empty(&dev->ready_cmd_list) ||
1602                   !dev->blocking || dev->cleanup_done ||
1603                   signal_pending(current);
1604         return res;
1605 }
1606
1607 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1608 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1609         struct scst_user_cmd **ucmd)
1610 {
1611         int res = 0;
1612         wait_queue_t wait;
1613
1614         TRACE_ENTRY();
1615
1616         init_waitqueue_entry(&wait, current);
1617
1618         while (1) {
1619                 if (!test_cmd_lists(dev)) {
1620                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1621                                 &wait);
1622                         for (;;) {
1623                                 set_current_state(TASK_INTERRUPTIBLE);
1624                                 if (test_cmd_lists(dev))
1625                                         break;
1626                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1627                                 schedule();
1628                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1629                         }
1630                         set_current_state(TASK_RUNNING);
1631                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1632                                 &wait);
1633                 }
1634
1635                 dev_user_process_scst_commands(dev);
1636
1637                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1638                 if (*ucmd != NULL)
1639                         break;
1640
1641                 if (!dev->blocking || dev->cleanup_done) {
1642                         res = -EAGAIN;
1643                         TRACE_DBG("No ready commands, returning %d", res);
1644                         break;
1645                 }
1646
1647                 if (signal_pending(current)) {
1648                         res = -EINTR;
1649                         TRACE_DBG("Signal pending, returning %d", res);
1650                         break;
1651                 }
1652         }
1653
1654         TRACE_EXIT_RES(res);
1655         return res;
1656 }
1657
1658 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1659 {
1660         int res = 0;
1661         struct scst_user_dev *dev;
1662         struct scst_user_get_cmd *cmd;
1663         struct scst_user_reply_cmd *reply;
1664         struct scst_user_cmd *ucmd;
1665         uint64_t ureply;
1666
1667         TRACE_ENTRY();
1668
1669         mutex_lock(&dev_priv_mutex);
1670         dev = (struct scst_user_dev *)file->private_data;
1671         res = dev_user_check_reg(dev);
1672         if (res != 0) {
1673                 mutex_unlock(&dev_priv_mutex);
1674                 goto out;
1675         }
1676         down_read(&dev->dev_rwsem);
1677         mutex_unlock(&dev_priv_mutex);
1678
1679         res = copy_from_user(&ureply, arg, sizeof(ureply));
1680         if (res < 0)
1681                 goto out_up;
1682
1683         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1684                 dev->name);
1685
1686         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1687         if (cmd == NULL) {
1688                 res = -ENOMEM;
1689                 goto out_up;
1690         }
1691
1692         if (ureply != 0) {
1693                 unsigned long u = (unsigned long)ureply;
1694                 reply = (struct scst_user_reply_cmd *)cmd;
1695                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1696                 if (res < 0)
1697                         goto out_free;
1698
1699                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1700
1701                 res = dev_user_process_reply(dev, reply);
1702                 if (res < 0)
1703                         goto out_free;
1704         }
1705
1706         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1707         res = dev_user_get_next_cmd(dev, &ucmd);
1708         if (res == 0) {
1709                 *cmd = ucmd->user_cmd;
1710                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1711                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1712                 res = copy_to_user(arg, cmd, sizeof(*cmd));
1713         } else
1714                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1715
1716 out_free:
1717         kmem_cache_free(user_get_cmd_cachep, cmd);
1718
1719 out_up:
1720         up_read(&dev->dev_rwsem);
1721
1722 out:
1723         TRACE_EXIT_RES(res);
1724         return res;
1725 }
1726
1727 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1728         unsigned long arg)
1729 {
1730         long res;
1731
1732         TRACE_ENTRY();
1733
1734         switch (cmd) {
1735         case SCST_USER_REPLY_AND_GET_CMD:
1736                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1737                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1738                 break;
1739
1740         case SCST_USER_REPLY_CMD:
1741                 TRACE_DBG("%s", "REPLY_CMD");
1742                 res = dev_user_reply_cmd(file, (void __user *)arg);
1743                 break;
1744
1745         case SCST_USER_REGISTER_DEVICE:
1746         {
1747                 struct scst_user_dev_desc *dev_desc;
1748                 TRACE_DBG("%s", "REGISTER_DEVICE");
1749                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1750                 if (dev_desc == NULL) {
1751                         res = -ENOMEM;
1752                         goto out;
1753                 }
1754                 res = copy_from_user(dev_desc, (void __user *)arg,
1755                                      sizeof(*dev_desc));
1756                 if (res < 0) {
1757                         kfree(dev_desc);
1758                         goto out;
1759                 }
1760                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1761                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1762                 res = dev_user_register_dev(file, dev_desc);
1763                 kfree(dev_desc);
1764                 break;
1765         }
1766
1767         case SCST_USER_SET_OPTIONS:
1768         {
1769                 struct scst_user_opt opt;
1770                 TRACE_DBG("%s", "SET_OPTIONS");
1771                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1772                 if (res < 0)
1773                         goto out;
1774                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1775                 res = dev_user_set_opt(file, &opt);
1776                 break;
1777         }
1778
1779         case SCST_USER_GET_OPTIONS:
1780                 TRACE_DBG("%s", "GET_OPTIONS");
1781                 res = dev_user_get_opt(file, (void __user *)arg);
1782                 break;
1783
1784         default:
1785                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1786                 res = -EINVAL;
1787                 goto out;
1788         }
1789
1790 out:
1791         TRACE_EXIT_RES(res);
1792         return res;
1793 }
1794
1795 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1796 {
1797         int res = 0;
1798         struct scst_user_dev *dev;
1799
1800         TRACE_ENTRY();
1801
1802         mutex_lock(&dev_priv_mutex);
1803         dev = (struct scst_user_dev *)file->private_data;
1804         res = dev_user_check_reg(dev);
1805         if (res != 0) {
1806                 mutex_unlock(&dev_priv_mutex);
1807                 goto out;
1808         }
1809         down_read(&dev->dev_rwsem);
1810         mutex_unlock(&dev_priv_mutex);
1811
1812         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1813
1814         if (!list_empty(&dev->ready_cmd_list) ||
1815             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1816                 res |= POLLIN | POLLRDNORM;
1817                 goto out_unlock;
1818         }
1819
1820         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1821
1822         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
1823         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1824         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
1825
1826         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1827
1828         if (!list_empty(&dev->ready_cmd_list) ||
1829             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1830                 res |= POLLIN | POLLRDNORM;
1831                 goto out_unlock;
1832         }
1833
1834 out_unlock:
1835         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1836
1837         up_read(&dev->dev_rwsem);
1838
1839 out:
1840         TRACE_EXIT_HRES(res);
1841         return res;
1842 }
1843
1844 /*
1845  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
1846  */
1847 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1848         unsigned long *flags)
1849         __releases(&dev->cmd_lists.cmd_list_lock)
1850         __acquires(&dev->cmd_lists.cmd_list_lock)
1851 {
1852         int state = ucmd->state;
1853         struct scst_user_dev *dev = ucmd->dev;
1854
1855         TRACE_ENTRY();
1856
1857         if (ucmd->this_state_unjammed)
1858                 goto out;
1859
1860         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1861                 state);
1862
1863         ucmd->jammed = 1;
1864         ucmd->this_state_unjammed = 1;
1865         ucmd->sent_to_user = 0;
1866
1867         switch (state) {
1868         case UCMD_STATE_PARSING:
1869         case UCMD_STATE_BUF_ALLOCING:
1870                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1871                         ucmd->aborted = 1;
1872                 else {
1873                         if (busy)
1874                                 scst_set_busy(ucmd->cmd);
1875                         else
1876                                 scst_set_cmd_error(ucmd->cmd,
1877                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
1878                 }
1879                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1880
1881                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1882                 list_add(&ucmd->cmd->cmd_list_entry,
1883                         &ucmd->cmd->cmd_lists->active_cmd_list);
1884                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1885                 break;
1886
1887         case UCMD_STATE_EXECING:
1888                 if (flags != NULL)
1889                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
1890                                                *flags);
1891                 else
1892                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1893
1894                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1895
1896                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1897                         ucmd->aborted = 1;
1898                 else {
1899                         if (busy)
1900                                 scst_set_busy(ucmd->cmd);
1901                         else
1902                                 scst_set_cmd_error(ucmd->cmd,
1903                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
1904                 }
1905
1906                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
1907                         SCST_CONTEXT_DIRECT);
1908                 /* !! At this point cmd and ucmd can be already freed !! */
1909
1910                 if (flags != NULL)
1911                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
1912                                           *flags);
1913                 else
1914                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1915                 break;
1916
1917         case UCMD_STATE_ON_FREEING:
1918         case UCMD_STATE_ON_CACHE_FREEING:
1919         case UCMD_STATE_TM_EXECING:
1920         case UCMD_STATE_ATTACH_SESS:
1921         case UCMD_STATE_DETACH_SESS:
1922         {
1923                 if (flags != NULL)
1924                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
1925                                                *flags);
1926                 else
1927                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1928
1929                 switch (state) {
1930                 case UCMD_STATE_ON_FREEING:
1931                         dev_user_process_reply_on_free(ucmd);
1932                         break;
1933
1934                 case UCMD_STATE_ON_CACHE_FREEING:
1935                         dev_user_process_reply_on_cache_free(ucmd);
1936                         break;
1937
1938                 case UCMD_STATE_TM_EXECING:
1939                         dev_user_process_reply_tm_exec(ucmd,
1940                                                        SCST_MGMT_STATUS_FAILED);
1941                         break;
1942
1943                 case UCMD_STATE_ATTACH_SESS:
1944                 case UCMD_STATE_DETACH_SESS:
1945                         dev_user_process_reply_sess(ucmd, -EFAULT);
1946                         break;
1947                 }
1948
1949                 if (flags != NULL)
1950                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
1951                                           *flags);
1952                 else
1953                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1954                 break;
1955         }
1956
1957         default:
1958                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1959                 sBUG();
1960                 break;
1961         }
1962
1963 out:
1964         TRACE_EXIT();
1965         return;
1966 }
1967
1968 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1969         __releases(&dev->cmd_lists.cmd_list_lock)
1970         __acquires(&dev->cmd_lists.cmd_list_lock)
1971 {
1972         int i;
1973         struct scst_user_cmd *ucmd;
1974
1975         TRACE_ENTRY();
1976
1977         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1978
1979         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1980
1981 repeat:
1982         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1983                 struct list_head *head = &dev->ucmd_hash[i];
1984
1985                 list_for_each_entry(ucmd, head, hash_list_entry) {
1986                         if (!ucmd->sent_to_user)
1987                                 continue;
1988
1989                         if (ucmd_get_check(ucmd))
1990                                 continue;
1991
1992                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1993                                 ucmd->state, ucmd->cmd);
1994
1995                         dev_user_unjam_cmd(ucmd, 0, NULL);
1996
1997                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1998                         ucmd_put(ucmd);
1999                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2000
2001                         goto repeat;
2002                 }
2003         }
2004
2005         if (dev_user_process_scst_commands(dev) != 0)
2006                 goto repeat;
2007
2008         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2009
2010         TRACE_EXIT();
2011         return;
2012 }
2013
2014 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2015         int status)
2016 {
2017         int res = 0;
2018
2019         TRACE_ENTRY();
2020
2021         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2022                 ucmd->user_cmd.tm_cmd.fn, status);
2023
2024         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2025                 /*
2026                  * It is possible that user space seen TM cmd before cmd
2027                  * to abort or will never see it at all, because it was
2028                  * aborted on the way there. So, it is safe to return
2029                  * success instead, because, if there is the TM cmd at this
2030                  * point, then the cmd to abort apparrently does exist.
2031                  */
2032                 status = SCST_MGMT_STATUS_SUCCESS;
2033         }
2034
2035         scst_async_mcmd_completed(ucmd->mcmd, status);
2036
2037         ucmd_put(ucmd);
2038
2039         TRACE_EXIT_RES(res);
2040         return res;
2041 }
2042
2043 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2044 {
2045         struct scst_user_cmd *ucmd;
2046         unsigned long flags;
2047
2048         TRACE_ENTRY();
2049
2050         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2051 again:
2052         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2053                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2054                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2055                         switch (ucmd->state) {
2056                         case UCMD_STATE_PARSING:
2057                         case UCMD_STATE_BUF_ALLOCING:
2058                         case UCMD_STATE_EXECING:
2059                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2060                                 list_del(&ucmd->ready_cmd_list_entry);
2061                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2062                                 goto again;
2063                         }
2064                 }
2065         }
2066
2067         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2068
2069         TRACE_EXIT();
2070         return;
2071 }
2072
2073 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2074         struct scst_tgt_dev *tgt_dev)
2075 {
2076         struct scst_user_cmd *ucmd;
2077         struct scst_user_dev *dev =
2078                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2079         struct scst_user_cmd *ucmd_to_abort = NULL;
2080
2081         TRACE_ENTRY();
2082
2083         /*
2084          * In the used approach we don't do anything with hung devices, which
2085          * stopped responding and/or have stuck commands. We forcedly abort such
2086          * commands only if they not yet sent to the user space or if the device
2087          * is getting unloaded, e.g. if its handler program gets killed. This is
2088          * because it's pretty hard to distinguish between stuck and temporary
2089          * overloaded states of the device. There are several reasons for that:
2090          *
2091          * 1. Some commands need a lot of time to complete (several hours),
2092          *    so for an impatient user such command(s) will always look as
2093          *    stuck.
2094          *
2095          * 2. If we forcedly abort, i.e. abort before it's actually completed
2096          *    in the user space, just one command, we will have to put the whole
2097          *    device offline until we are sure that no more previously aborted
2098          *    commands will get executed. Otherwise, we might have a possibility
2099          *    for data corruption, when aborted and reported as completed
2100          *    command actually gets executed *after* new commands sent
2101          *    after the force abort was done. Many journaling file systems and
2102          *    databases use "provide required commands order via queue draining"
2103          *    approach and not putting the whole device offline after the forced
2104          *    abort will break it. This makes our decision, if a command stuck
2105          *    or not, cost a lot.
2106          *
2107          * So, we leave policy definition if a device stuck or not to
2108          * the user space and simply let all commands live until they are
2109          * completed or their devices get closed/killed. This approach is very
2110          * much OK, but can affect management commands, which need activity
2111          * suspending via scst_suspend_activity() function such as devices or
2112          * targets registration/removal. But during normal life such commands
2113          * should be rare. Plus, when possible, scst_suspend_activity() will
2114          * return after timeout EBUSY status to allow caller to not stuck
2115          * forever as well.
2116          *
2117          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2118          * stuck commands would affect only related devices.
2119          */
2120
2121         dev_user_abort_ready_commands(dev);
2122
2123         /* We can't afford missing TM command due to memory shortage */
2124         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2125
2126         ucmd->user_cmd.cmd_h = ucmd->h;
2127         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2128         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2129         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2130         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2131         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2132
2133         if (mcmd->cmd_to_abort != NULL) {
2134                 ucmd_to_abort =
2135                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2136                 if (ucmd_to_abort != NULL)
2137                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2138         }
2139
2140         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2141                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2142                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2143                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2144
2145         ucmd->mcmd = mcmd;
2146         ucmd->state = UCMD_STATE_TM_EXECING;
2147
2148         scst_prepare_async_mcmd(mcmd);
2149
2150         dev_user_add_to_ready(ucmd);
2151
2152         TRACE_EXIT();
2153         return SCST_DEV_TM_NOT_COMPLETED;
2154 }
2155
2156 static int dev_user_attach(struct scst_device *sdev)
2157 {
2158         int res = 0;
2159         struct scst_user_dev *dev = NULL, *d;
2160
2161         TRACE_ENTRY();
2162
2163         spin_lock(&dev_list_lock);
2164         list_for_each_entry(d, &dev_list, dev_list_entry) {
2165                 if (strcmp(d->name, sdev->virt_name) == 0) {
2166                         dev = d;
2167                         break;
2168                 }
2169         }
2170         spin_unlock(&dev_list_lock);
2171         if (dev == NULL) {
2172                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2173                 res = -EINVAL;
2174                 goto out;
2175         }
2176
2177         sdev->p_cmd_lists = &dev->cmd_lists;
2178         sdev->dh_priv = dev;
2179         sdev->tst = dev->tst;
2180         sdev->queue_alg = dev->queue_alg;
2181         sdev->swp = dev->swp;
2182         sdev->tas = dev->tas;
2183         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2184
2185         dev->sdev = sdev;
2186
2187         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2188                 dev->name);
2189
2190 out:
2191         TRACE_EXIT();
2192         return res;
2193 }
2194
2195 static void dev_user_detach(struct scst_device *sdev)
2196 {
2197         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2198
2199         TRACE_ENTRY();
2200
2201         TRACE_DBG("virt_id %d", sdev->virt_id);
2202
2203         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2204                 dev->name);
2205
2206         /* dev will be freed by the caller */
2207         sdev->dh_priv = NULL;
2208         dev->sdev = NULL;
2209
2210         TRACE_EXIT();
2211         return;
2212 }
2213
2214 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2215 {
2216         int res = 0;
2217         unsigned long flags;
2218
2219         TRACE_ENTRY();
2220
2221         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2222
2223         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2224
2225         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2226                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2227                 ucmd->result = status;
2228         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2229                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2230         } else
2231                 sBUG();
2232
2233         if (ucmd->cmpl != NULL)
2234                 complete_all(ucmd->cmpl);
2235
2236         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2237
2238         ucmd_put(ucmd);
2239
2240         TRACE_EXIT_RES(res);
2241         return res;
2242 }
2243
2244 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2245 {
2246         struct scst_user_dev *dev =
2247                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2248         int res = 0, rc;
2249         struct scst_user_cmd *ucmd;
2250         DECLARE_COMPLETION_ONSTACK(cmpl);
2251
2252         TRACE_ENTRY();
2253
2254         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2255         if (ucmd == NULL)
2256                 goto out_nomem;
2257
2258         ucmd->cmpl = &cmpl;
2259
2260         ucmd->user_cmd.cmd_h = ucmd->h;
2261         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2262         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2263         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2264         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2265         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2266         strncpy(ucmd->user_cmd.sess.initiator_name,
2267                 tgt_dev->sess->initiator_name,
2268                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2269         ucmd->user_cmd.sess.initiator_name[
2270                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2271
2272         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2273                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2274                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2275                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2276                 ucmd->user_cmd.sess.initiator_name);
2277
2278         ucmd->state = UCMD_STATE_ATTACH_SESS;
2279
2280         ucmd_get(ucmd);
2281
2282         dev_user_add_to_ready(ucmd);
2283
2284         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2285         if (rc > 0)
2286                 res = ucmd->result;
2287         else {
2288                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2289                 res = -EFAULT;
2290         }
2291
2292         sBUG_ON(irqs_disabled());
2293
2294         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2295         ucmd->cmpl = NULL;
2296         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2297
2298         ucmd_put(ucmd);
2299
2300 out:
2301         TRACE_EXIT_RES(res);
2302         return res;
2303
2304 out_nomem:
2305         res = -ENOMEM;
2306         goto out;
2307 }
2308
2309 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2310 {
2311         struct scst_user_dev *dev =
2312                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2313         struct scst_user_cmd *ucmd;
2314
2315         TRACE_ENTRY();
2316
2317         /*
2318          * We can't miss TM command due to memory shortage, because it might
2319          * lead to a memory leak in the user space handler.
2320          */
2321         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2322         if (ucmd == NULL)
2323                 goto out;
2324
2325         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2326                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2327
2328         ucmd->user_cmd.cmd_h = ucmd->h;
2329         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2330         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2331
2332         ucmd->state = UCMD_STATE_DETACH_SESS;
2333
2334         dev_user_add_to_ready(ucmd);
2335
2336 out:
2337         TRACE_EXIT();
2338         return;
2339 }
2340
2341 /* No locks are needed, but the activity must be suspended */
2342 static void dev_user_setup_functions(struct scst_user_dev *dev)
2343 {
2344         TRACE_ENTRY();
2345
2346         dev->devtype.parse = dev_user_parse;
2347         dev->devtype.dev_done = NULL;
2348
2349         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2350                 switch (dev->devtype.type) {
2351                 case TYPE_DISK:
2352                         dev->generic_parse = scst_sbc_generic_parse;
2353                         dev->devtype.dev_done = dev_user_disk_done;
2354                         break;
2355
2356                 case TYPE_TAPE:
2357                         dev->generic_parse = scst_tape_generic_parse;
2358                         dev->devtype.dev_done = dev_user_tape_done;
2359                         break;
2360
2361                 case TYPE_MOD:
2362                         dev->generic_parse = scst_modisk_generic_parse;
2363                         dev->devtype.dev_done = dev_user_disk_done;
2364                         break;
2365
2366                 case TYPE_ROM:
2367                         dev->generic_parse = scst_cdrom_generic_parse;
2368                         dev->devtype.dev_done = dev_user_disk_done;
2369                         break;
2370
2371                 case TYPE_MEDIUM_CHANGER:
2372                         dev->generic_parse = scst_changer_generic_parse;
2373                         break;
2374
2375                 case TYPE_PROCESSOR:
2376                         dev->generic_parse = scst_processor_generic_parse;
2377                         break;
2378
2379                 case TYPE_RAID:
2380                         dev->generic_parse = scst_raid_generic_parse;
2381                         break;
2382
2383                 default:
2384                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2385                                 "for it", dev->devtype.type);
2386                         dev->parse_type = SCST_USER_PARSE_CALL;
2387                         break;
2388                 }
2389         } else {
2390                 dev->generic_parse = NULL;
2391                 dev->devtype.dev_done = NULL;
2392         }
2393
2394         TRACE_EXIT();
2395         return;
2396 }
2397
2398 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2399 {
2400         char ver[sizeof(DEV_USER_VERSION)+1];
2401         int res;
2402
2403         res = copy_from_user(ver,
2404                         (void __user *)(unsigned long)dev_desc->version_str,
2405                         sizeof(ver));
2406         if (res < 0) {
2407                 PRINT_ERROR("%s", "Unable to get version string");
2408                 goto out;
2409         }
2410         ver[sizeof(ver)-1] = '\0';
2411
2412         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2413                 /* ->name already 0-terminated in dev_user_ioctl() */
2414                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2415                         dev_desc->name, ver);
2416                 res = -EINVAL;
2417                 goto out;
2418         }
2419
2420 out:
2421         return res;
2422 }
2423
2424 static int dev_user_register_dev(struct file *file,
2425         const struct scst_user_dev_desc *dev_desc)
2426 {
2427         int res = -ENOMEM, i;
2428         struct scst_user_dev *dev, *d;
2429         int block;
2430
2431         TRACE_ENTRY();
2432
2433         res = dev_user_check_version(dev_desc);
2434         if (res != 0)
2435                 goto out;
2436
2437         switch (dev_desc->type) {
2438         case TYPE_DISK:
2439         case TYPE_ROM:
2440         case TYPE_MOD:
2441                 if (dev_desc->block_size == 0) {
2442                         PRINT_ERROR("Wrong block size %d",
2443                                     dev_desc->block_size);
2444                         res = -EINVAL;
2445                         goto out;
2446                 }
2447                 block = scst_calc_block_shift(dev_desc->block_size);
2448                 if (block == -1) {
2449                         res = -EINVAL;
2450                         goto out;
2451                 }
2452                 break;
2453         default:
2454                 block = dev_desc->block_size;
2455                 break;
2456         }
2457
2458         if (!try_module_get(THIS_MODULE)) {
2459                 PRINT_ERROR("%s", "Fail to get module");
2460                 goto out;
2461         }
2462
2463         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2464         if (dev == NULL)
2465                 goto out_put;
2466
2467         init_rwsem(&dev->dev_rwsem);
2468         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2469         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2470         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2471         INIT_LIST_HEAD(&dev->ready_cmd_list);
2472         if (file->f_flags & O_NONBLOCK) {
2473                 TRACE_DBG("%s", "Non-blocking operations");
2474                 dev->blocking = 0;
2475         } else
2476                 dev->blocking = 1;
2477         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2478                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2479
2480         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2481         dev->name[sizeof(dev->name)-1] = '\0';
2482
2483         scst_init_mem_lim(&dev->udev_mem_lim);
2484
2485         /*
2486          * We don't use clustered pool, since it implies pages reordering,
2487          * which isn't possible with user space supplied buffers. Although
2488          * it's still possible to cluster pages by the tail of each other,
2489          * seems it doesn't worth the effort.
2490          */
2491         dev->pool = sgv_pool_create(dev->name, 0);
2492         if (dev->pool == NULL)
2493                 goto out_put;
2494         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2495                 dev_user_free_sg_entries);
2496
2497         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2498                 dev->name);
2499         dev->devtype.type = dev_desc->type;
2500         dev->devtype.threads_num = -1;
2501         dev->devtype.parse_atomic = 1;
2502         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2503         dev->devtype.dev_done_atomic = 1;
2504         dev->devtype.no_proc = 1;
2505         dev->devtype.attach = dev_user_attach;
2506         dev->devtype.detach = dev_user_detach;
2507         dev->devtype.attach_tgt = dev_user_attach_tgt;
2508         dev->devtype.detach_tgt = dev_user_detach_tgt;
2509         dev->devtype.exec = dev_user_exec;
2510         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2511         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2512
2513         init_completion(&dev->cleanup_cmpl);
2514         dev->block = block;
2515         dev->def_block = block;
2516
2517         res = __dev_user_set_opt(dev, &dev_desc->opt);
2518
2519         TRACE_MEM("dev %p, name %s", dev, dev->name);
2520
2521         spin_lock(&dev_list_lock);
2522
2523         list_for_each_entry(d, &dev_list, dev_list_entry) {
2524                 if (strcmp(d->name, dev->name) == 0) {
2525                         PRINT_ERROR("Device %s already exist",
2526                                 dev->name);
2527                         res = -EEXIST;
2528                         spin_unlock(&dev_list_lock);
2529                         goto out_free;
2530                 }
2531         }
2532
2533         list_add_tail(&dev->dev_list_entry, &dev_list);
2534
2535         spin_unlock(&dev_list_lock);
2536
2537         if (res != 0)
2538                 goto out_del_free;
2539
2540         res = scst_register_virtual_dev_driver(&dev->devtype);
2541         if (res < 0)
2542                 goto out_del_free;
2543
2544         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2545         if (dev->virt_id < 0) {
2546                 res = dev->virt_id;
2547                 goto out_unreg_handler;
2548         }
2549
2550         mutex_lock(&dev_priv_mutex);
2551         if (file->private_data != NULL) {
2552                 mutex_unlock(&dev_priv_mutex);
2553                 PRINT_ERROR("%s", "Device already registered");
2554                 res = -EINVAL;
2555                 goto out_unreg_drv;
2556         }
2557         file->private_data = dev;
2558         mutex_unlock(&dev_priv_mutex);
2559
2560 out:
2561         TRACE_EXIT_RES(res);
2562         return res;
2563
2564 out_unreg_drv:
2565         scst_unregister_virtual_device(dev->virt_id);
2566
2567 out_unreg_handler:
2568         scst_unregister_virtual_dev_driver(&dev->devtype);
2569
2570 out_del_free:
2571         spin_lock(&dev_list_lock);
2572         list_del(&dev->dev_list_entry);
2573         spin_unlock(&dev_list_lock);
2574
2575 out_free:
2576         sgv_pool_destroy(dev->pool);
2577         kfree(dev);
2578         goto out_put;
2579
2580 out_put:
2581         module_put(THIS_MODULE);
2582         goto out;
2583 }
2584
2585 static int __dev_user_set_opt(struct scst_user_dev *dev,
2586         const struct scst_user_opt *opt)
2587 {
2588         int res = 0;
2589
2590         TRACE_ENTRY();
2591
2592         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2593                 "memory_reuse_type %x, partial_transfers_type %x, "
2594                 "partial_len %d", dev->name, opt->parse_type,
2595                 opt->on_free_cmd_type, opt->memory_reuse_type,
2596                 opt->partial_transfers_type, opt->partial_len);
2597
2598         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
2599             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
2600             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
2601             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
2602                 PRINT_ERROR("%s", "Invalid option");
2603                 res = -EINVAL;
2604                 goto out;
2605         }
2606
2607         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2608              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2609             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2610              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2611             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2612                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
2613                         " tas %x, has_own_order_mgmt %x)", opt->tst,
2614                         opt->queue_alg, opt->swp, opt->tas,
2615                         opt->has_own_order_mgmt);
2616                 res = -EINVAL;
2617                 goto out;
2618         }
2619
2620         dev->parse_type = opt->parse_type;
2621         dev->on_free_cmd_type = opt->on_free_cmd_type;
2622         dev->memory_reuse_type = opt->memory_reuse_type;
2623         dev->partial_transfers_type = opt->partial_transfers_type;
2624         dev->partial_len = opt->partial_len;
2625
2626         dev->tst = opt->tst;
2627         dev->queue_alg = opt->queue_alg;
2628         dev->swp = opt->swp;
2629         dev->tas = opt->tas;
2630         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2631         if (dev->sdev != NULL) {
2632                 dev->sdev->tst = opt->tst;
2633                 dev->sdev->queue_alg = opt->queue_alg;
2634                 dev->sdev->swp = opt->swp;
2635                 dev->sdev->tas = opt->tas;
2636                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2637         }
2638
2639         dev_user_setup_functions(dev);
2640
2641 out:
2642         TRACE_EXIT_RES(res);
2643         return res;
2644 }
2645
2646 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2647 {
2648         int res = 0;
2649         struct scst_user_dev *dev;
2650
2651         TRACE_ENTRY();
2652
2653         mutex_lock(&dev_priv_mutex);
2654         dev = (struct scst_user_dev *)file->private_data;
2655         res = dev_user_check_reg(dev);
2656         if (res != 0) {
2657                 mutex_unlock(&dev_priv_mutex);
2658                 goto out;
2659         }
2660         down_write(&dev->dev_rwsem);
2661         mutex_unlock(&dev_priv_mutex);
2662
2663         res = scst_suspend_activity(true);
2664         if (res != 0)
2665                 goto out;
2666
2667         res = __dev_user_set_opt(dev, opt);
2668
2669         scst_resume_activity();
2670
2671         up_write(&dev->dev_rwsem);
2672
2673 out:
2674         TRACE_EXIT_RES(res);
2675         return res;
2676 }
2677
2678 static int dev_user_get_opt(struct file *file, void __user *arg)
2679 {
2680         int res = 0;
2681         struct scst_user_dev *dev;
2682         struct scst_user_opt opt;
2683
2684         TRACE_ENTRY();
2685
2686         mutex_lock(&dev_priv_mutex);
2687         dev = (struct scst_user_dev *)file->private_data;
2688         res = dev_user_check_reg(dev);
2689         if (res != 0) {
2690                 mutex_unlock(&dev_priv_mutex);
2691                 goto out;
2692         }
2693         down_read(&dev->dev_rwsem);
2694         mutex_unlock(&dev_priv_mutex);
2695
2696         opt.parse_type = dev->parse_type;
2697         opt.on_free_cmd_type = dev->on_free_cmd_type;
2698         opt.memory_reuse_type = dev->memory_reuse_type;
2699         opt.partial_transfers_type = dev->partial_transfers_type;
2700         opt.partial_len = dev->partial_len;
2701         opt.tst = dev->tst;
2702         opt.queue_alg = dev->queue_alg;
2703         opt.tas = dev->tas;
2704         opt.swp = dev->swp;
2705         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2706
2707         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2708                 "memory_reuse_type %x, partial_transfers_type %x, "
2709                 "partial_len %d", dev->name, opt.parse_type,
2710                 opt.on_free_cmd_type, opt.memory_reuse_type,
2711                 opt.partial_transfers_type, opt.partial_len);
2712
2713         res = copy_to_user(arg, &opt, sizeof(opt));
2714
2715         up_read(&dev->dev_rwsem);
2716 out:
2717         TRACE_EXIT_RES(res);
2718         return res;
2719 }
2720
2721 static int dev_usr_parse(struct scst_cmd *cmd)
2722 {
2723         sBUG();
2724         return SCST_CMD_STATE_DEFAULT;
2725 }
2726
2727 /* Needed only for /proc support */
2728 #define USR_TYPE {                      \
2729         .name =         DEV_USER_NAME,  \
2730         .type =         -1,             \
2731         .parse =        dev_usr_parse,  \
2732 }
2733
2734 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2735
2736 static int dev_user_release(struct inode *inode, struct file *file)
2737 {
2738         int res = 0;
2739         struct scst_user_dev *dev;
2740
2741         TRACE_ENTRY();
2742
2743         mutex_lock(&dev_priv_mutex);
2744         dev = (struct scst_user_dev *)file->private_data;
2745         if (dev == NULL) {
2746                 mutex_unlock(&dev_priv_mutex);
2747                 goto out;
2748         }
2749         file->private_data = NULL;
2750
2751         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2752
2753         spin_lock(&dev_list_lock);
2754         list_del(&dev->dev_list_entry);
2755         spin_unlock(&dev_list_lock);
2756
2757         mutex_unlock(&dev_priv_mutex);
2758
2759         down_write(&dev->dev_rwsem);
2760
2761         spin_lock(&cleanup_lock);
2762         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2763         spin_unlock(&cleanup_lock);
2764
2765         wake_up(&cleanup_list_waitQ);
2766         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2767
2768         scst_unregister_virtual_device(dev->virt_id);
2769         scst_unregister_virtual_dev_driver(&dev->devtype);
2770
2771         sgv_pool_destroy(dev->pool);
2772
2773         TRACE_DBG("Unregistering finished (dev %p)", dev);
2774
2775         dev->cleanup_done = 1;
2776
2777         wake_up(&cleanup_list_waitQ);
2778         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2779
2780         wait_for_completion(&dev->cleanup_cmpl);
2781
2782         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2783
2784         TRACE_DBG("Releasing completed (dev %p)", dev);
2785
2786         kfree(dev);
2787
2788         module_put(THIS_MODULE);
2789
2790 out:
2791         TRACE_EXIT_RES(res);
2792         return res;
2793 }
2794
2795 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2796 {
2797         struct scst_user_cmd *ucmd;
2798         int rc, res = 1;
2799
2800         TRACE_ENTRY();
2801
2802         dev->blocking = 0;
2803
2804         while (1) {
2805                 TRACE_DBG("Cleanuping dev %p", dev);
2806
2807                 dev_user_unjam_dev(dev);
2808
2809                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2810
2811                 rc = dev_user_get_next_cmd(dev, &ucmd);
2812                 if (rc == 0)
2813                         dev_user_unjam_cmd(ucmd, 1, NULL);
2814
2815                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2816
2817                 if (rc == -EAGAIN) {
2818                         if (dev->cleanup_done)
2819                                 break;
2820                         else {
2821                                 TRACE_DBG("No more commands (dev %p)", dev);
2822                                 goto out;
2823                         }
2824                 }
2825         }
2826
2827 #ifdef CONFIG_SCST_EXTRACHECKS
2828 {
2829         int i;
2830         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2831                 struct list_head *head = &dev->ucmd_hash[i];
2832                 struct scst_user_cmd *ucmd2;
2833 again:
2834                 list_for_each_entry(ucmd2, head, hash_list_entry) {
2835                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
2836                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
2837                         ucmd_put(ucmd2);
2838                         goto again;
2839                 }
2840         }
2841 }
2842 #endif
2843
2844         TRACE_DBG("Cleanuping done (dev %p)", dev);
2845         complete_all(&dev->cleanup_cmpl);
2846         res = 0;
2847
2848 out:
2849         TRACE_EXIT_RES(res);
2850         return res;
2851 }
2852
2853 static inline int test_cleanup_list(void)
2854 {
2855         int res = !list_empty(&cleanup_list) ||
2856                   unlikely(kthread_should_stop());
2857         return res;
2858 }
2859
2860 static int dev_user_cleanup_thread(void *arg)
2861 {
2862         TRACE_ENTRY();
2863
2864         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2865
2866         current->flags |= PF_NOFREEZE;
2867
2868         spin_lock(&cleanup_lock);
2869         while (!kthread_should_stop()) {
2870                 wait_queue_t wait;
2871                 init_waitqueue_entry(&wait, current);
2872
2873                 if (!test_cleanup_list()) {
2874                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2875                         for (;;) {
2876                                 set_current_state(TASK_INTERRUPTIBLE);
2877                                 if (test_cleanup_list())
2878                                         break;
2879                                 spin_unlock(&cleanup_lock);
2880                                 schedule();
2881                                 spin_lock(&cleanup_lock);
2882                         }
2883                         set_current_state(TASK_RUNNING);
2884                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2885                 }
2886
2887                 /*
2888                  * We have to poll devices, because commands can go from SCST
2889                  * core on cmd_list_waitQ and we have no practical way to
2890                  * detect them.
2891                  */
2892
2893                 while (1) {
2894                         struct scst_user_dev *dev;
2895                         LIST_HEAD(cl_devs);
2896
2897                         while (!list_empty(&cleanup_list)) {
2898                                 int rc;
2899
2900                                 dev = list_entry(cleanup_list.next,
2901                                         typeof(*dev), cleanup_list_entry);
2902                                 list_del(&dev->cleanup_list_entry);
2903
2904                                 spin_unlock(&cleanup_lock);
2905                                 rc = dev_user_process_cleanup(dev);
2906                                 spin_lock(&cleanup_lock);
2907
2908                                 if (rc != 0)
2909                                         list_add_tail(&dev->cleanup_list_entry,
2910                                                 &cl_devs);
2911                         }
2912
2913                         if (list_empty(&cl_devs))
2914                                 break;
2915
2916                         spin_unlock(&cleanup_lock);
2917                         msleep(100);
2918                         spin_lock(&cleanup_lock);
2919
2920                         while (!list_empty(&cl_devs)) {
2921                                 dev = list_entry(cl_devs.next, typeof(*dev),
2922                                         cleanup_list_entry);
2923                                 list_move_tail(&dev->cleanup_list_entry,
2924                                         &cleanup_list);
2925                         }
2926                 }
2927         }
2928         spin_unlock(&cleanup_lock);
2929
2930         /*
2931          * If kthread_should_stop() is true, we are guaranteed to be
2932          * on the module unload, so cleanup_list must be empty.
2933          */
2934         sBUG_ON(!list_empty(&cleanup_list));
2935
2936         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2937
2938         TRACE_EXIT();
2939         return 0;
2940 }
2941
2942 static int __init init_scst_user(void)
2943 {
2944         int res = 0;
2945         struct max_get_reply {
2946                 union {
2947                         struct scst_user_get_cmd g;
2948                         struct scst_user_reply_cmd r;
2949                 };
2950         };
2951 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
2952         struct class_device *class_member;
2953 #else
2954         struct device *dev;
2955 #endif
2956
2957         TRACE_ENTRY();
2958
2959 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2960         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2961                 "Consider change VMSPLIT option or use 64-bit "
2962                 "configuration instead. See README file for details.");
2963         res = -EINVAL;
2964         goto out;
2965 #endif
2966
2967         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2968         if (user_cmd_cachep == NULL) {
2969                 res = -ENOMEM;
2970                 goto out;
2971         }
2972
2973         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
2974         if (user_get_cmd_cachep == NULL) {
2975                 res = -ENOMEM;
2976                 goto out_cache;
2977         }
2978
2979         dev_user_devtype.module = THIS_MODULE;
2980
2981         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2982         if (res < 0)
2983                 goto out_cache1;
2984
2985         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2986         if (res != 0)
2987                 goto out_unreg;
2988
2989         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2990         if (IS_ERR(dev_user_sysfs_class)) {
2991                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2992                         "space handler");
2993                 res = PTR_ERR(dev_user_sysfs_class);
2994                 goto out_proc;
2995         }
2996
2997         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2998         if (res) {
2999                 PRINT_ERROR("Unable to get major %d for SCSI tapes",
3000                             DEV_USER_MAJOR);
3001                 goto out_class;
3002         }
3003
3004 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3005         class_member = class_device_create(dev_user_sysfs_class, NULL,
3006                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
3007         if (IS_ERR(class_member)) {
3008                 res = PTR_ERR(class_member);
3009                 goto out_chrdev;
3010         }
3011 #else
3012         dev = device_create(dev_user_sysfs_class, NULL,
3013                             MKDEV(DEV_USER_MAJOR, 0),
3014 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
3015                                 NULL,
3016 #endif
3017                                 DEV_USER_NAME);
3018         if (IS_ERR(dev)) {
3019                 res = PTR_ERR(dev);
3020                 goto out_chrdev;
3021         }
3022 #endif
3023
3024         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
3025                 "scst_usr_cleanupd");
3026         if (IS_ERR(cleanup_thread)) {
3027                 res = PTR_ERR(cleanup_thread);
3028                 PRINT_ERROR("kthread_create() failed: %d", res);
3029                 goto out_dev;
3030         }
3031
3032 out:
3033         TRACE_EXIT_RES(res);
3034         return res;
3035
3036 out_dev:
3037 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3038         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3039 #else
3040         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3041 #endif
3042
3043 out_chrdev:
3044         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3045
3046 out_class:
3047         class_destroy(dev_user_sysfs_class);
3048
3049 out_proc:
3050         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3051
3052 out_unreg:
3053         scst_unregister_dev_driver(&dev_user_devtype);
3054
3055 out_cache1:
3056         kmem_cache_destroy(user_get_cmd_cachep);
3057
3058 out_cache:
3059         kmem_cache_destroy(user_cmd_cachep);
3060         goto out;
3061 }
3062
3063 static void __exit exit_scst_user(void)
3064 {
3065         int rc;
3066
3067         TRACE_ENTRY();
3068
3069         rc = kthread_stop(cleanup_thread);
3070         if (rc < 0)
3071                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3072
3073         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3074 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3075         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3076 #else
3077         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3078 #endif
3079         class_destroy(dev_user_sysfs_class);
3080
3081         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3082         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3083
3084         kmem_cache_destroy(user_get_cmd_cachep);
3085         kmem_cache_destroy(user_cmd_cachep);
3086
3087         TRACE_EXIT();
3088         return;
3089 }
3090
3091 module_init(init_scst_user);
3092 module_exit(exit_scst_user);
3093
3094 MODULE_AUTHOR("Vladislav Bolkhovitin");
3095 MODULE_LICENSE("GPL");
3096 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3097 MODULE_VERSION(SCST_VERSION_STRING);
3098 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);