scst/src/dev_handlers/scst_user.c

   1 /*
   2  *  scst_user.c
   3  *
   4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   6  *
   7  *  SCSI virtual user space device handler
   8  *
   9  *  This program is free software; you can redistribute it and/or
  10  *  modify it under the terms of the GNU General Public License
  11  *  as published by the Free Software Foundation, version 2
  12  *  of the License.
  13  *
  14  *  This program is distributed in the hope that it will be useful,
  15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  *  GNU General Public License for more details.
  18  */
  19
  20 #include <linux/kthread.h>
  21 #include <linux/delay.h>
  22 #include <linux/poll.h>
  23
  24 #define LOG_PREFIX              DEV_USER_NAME
  25
  26 #include "scst.h"
  27 #include "scst_user.h"
  28 #include "scst_dev_handler.h"
  29
  30 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
  31 #warning "HIGHMEM kernel configurations are not supported by this module, \
  32         because nowadays it doesn't worth the effort. Consider change \
  33         VMSPLIT option or use 64-bit configuration instead. See README file \
  34         for details."
  35 #endif
  36
  37 #define DEV_USER_MAJOR                  237
  38
  39 #define DEV_USER_CMD_HASH_ORDER         6
  40
  41 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
  42
  43 struct scst_user_dev {
  44         struct rw_semaphore dev_rwsem;
  45
  46         struct scst_cmd_lists cmd_lists;
  47
  48         /* Protected by cmd_lists.cmd_list_lock */
  49         struct list_head ready_cmd_list;
  50
  51         /* Protected by dev_rwsem or don't need any protection */
  52         unsigned int blocking:1;
  53         unsigned int cleanup_done:1;
  54         unsigned int tst:3;
  55         unsigned int queue_alg:4;
  56         unsigned int tas:1;
  57         unsigned int swp:1;
  58         unsigned int has_own_order_mgmt:1;
  59
  60         int (*generic_parse)(struct scst_cmd *cmd,
  61                 int (*get_block)(struct scst_cmd *cmd));
  62
  63         int block;
  64         int def_block;
  65
  66         struct scst_mem_lim udev_mem_lim;
  67         struct sgv_pool *pool;
  68
  69         uint8_t parse_type;
  70         uint8_t on_free_cmd_type;
  71         uint8_t memory_reuse_type;
  72         uint8_t partial_transfers_type;
  73         uint32_t partial_len;
  74
  75         struct scst_dev_type devtype;
  76
  77         /* Both protected by cmd_lists.cmd_list_lock */
  78         unsigned int handle_counter;
  79         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
  80
  81         struct scst_device *sdev;
  82
  83         int virt_id;
  84         struct list_head dev_list_entry;
  85         char name[SCST_MAX_NAME];
  86
  87         struct list_head cleanup_list_entry;
  88         /* ToDo: make it on-stack */
  89         struct completion cleanup_cmpl;
  90 };
  91
  92 /* Most fields are unprotected, since only one thread at time can access them */
  93 struct scst_user_cmd {
  94         struct scst_cmd *cmd;
  95         struct scst_user_dev *dev;
  96
  97         atomic_t ucmd_ref;
  98
  99         unsigned int buff_cached:1;
 100         unsigned int buf_dirty:1;
 101         unsigned int background_exec:1;
 102         unsigned int aborted:1;
 103
 104         struct scst_user_cmd *buf_ucmd;
 105
 106         int cur_data_page;
 107         int num_data_pages;
 108         int first_page_offset;
 109         unsigned long ubuff;
 110         struct page **data_pages;
 111         struct sgv_pool_obj *sgv;
 112
 113         /*
 114          * Special flags, which can be accessed asynchronously (hence "long").
 115          * Protected by cmd_lists.cmd_list_lock.
 116          */
 117         unsigned long sent_to_user:1;
 118         unsigned long jammed:1;
 119         unsigned long this_state_unjammed:1;
 120         unsigned long seen_by_user:1; /* here only as a small optimization */
 121
 122         unsigned int state;
 123
 124         struct list_head ready_cmd_list_entry;
 125
 126         unsigned int h;
 127         struct list_head hash_list_entry;
 128
 129         struct scst_user_get_cmd user_cmd;
 130
 131         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
 132         union {
 133                 struct completion *cmpl;
 134                 struct scst_mgmt_cmd *mcmd;
 135         };
 136         int result;
 137 };
 138
 139 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 140         gfp_t gfp_mask);
 141 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
 142
 143 static int dev_user_parse(struct scst_cmd *cmd);
 144 static int dev_user_exec(struct scst_cmd *cmd);
 145 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
 146 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
 147         struct scst_tgt_dev *tgt_dev);
 148
 149 static int dev_user_disk_done(struct scst_cmd *cmd);
 150 static int dev_user_tape_done(struct scst_cmd *cmd);
 151
 152 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 153         gfp_t gfp_mask, void *priv);
 154 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 155                                      void *priv);
 156
 157 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
 158
 159 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
 160         unsigned long *flags);
 161
 162 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
 163 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
 164         int status);
 165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
 166 static int dev_user_register_dev(struct file *file,
 167         const struct scst_user_dev_desc *dev_desc);
 168 static int __dev_user_set_opt(struct scst_user_dev *dev,
 169         const struct scst_user_opt *opt);
 170 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
 171 static int dev_user_get_opt(struct file *file, void __user *arg);
 172
 173 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
 174 static long dev_user_ioctl(struct file *file, unsigned int cmd,
 175         unsigned long arg);
 176 static int dev_user_release(struct inode *inode, struct file *file);
 177
 178 /** Data **/
 179
 180 static struct kmem_cache *user_cmd_cachep;
 181 static struct kmem_cache *user_get_cmd_cachep;
 182
 183 static DEFINE_MUTEX(dev_priv_mutex);
 184
 185 static struct file_operations dev_user_fops = {
 186         .poll           = dev_user_poll,
 187         .unlocked_ioctl = dev_user_ioctl,
 188 #ifdef CONFIG_COMPAT
 189         .compat_ioctl   = dev_user_ioctl,
 190 #endif
 191         .release        = dev_user_release,
 192 };
 193
 194 static struct class *dev_user_sysfs_class;
 195
 196 static DEFINE_SPINLOCK(dev_list_lock);
 197 static LIST_HEAD(dev_list);
 198
 199 static DEFINE_SPINLOCK(cleanup_lock);
 200 static LIST_HEAD(cleanup_list);
 201 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
 202 static struct task_struct *cleanup_thread;
 203
 204 /*
 205  * Skip this command if result is not 0. Must be called under
 206  * cmd_lists.cmd_list_lock and IRQ off.
 207  */
 208 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
 209 {
 210         int r = atomic_inc_return(&ucmd->ucmd_ref);
 211         int res;
 212         if (unlikely(r == 1)) {
 213                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
 214                 atomic_dec(&ucmd->ucmd_ref);
 215                 res = true;
 216                 /*
 217                  * Necessary code is serialized by cmd_list_lock in
 218                  * cmd_remove_hash()
 219                  */
 220         } else {
 221                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
 222                         atomic_read(&ucmd->ucmd_ref));
 223                 res = false;
 224         }
 225         return res;
 226 }
 227
 228 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
 229 {
 230         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 231         atomic_inc(&ucmd->ucmd_ref);
 232         if (barrier)
 233                 smp_mb__after_atomic_inc();
 234 }
 235
 236 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
 237 {
 238         __ucmd_get(ucmd, true);
 239 }
 240
 241 static inline void ucmd_get(struct scst_user_cmd *ucmd)
 242 {
 243         __ucmd_get(ucmd, false);
 244 }
 245
 246 /* Must not be called under cmd_list_lock!! */
 247 static inline void ucmd_put(struct scst_user_cmd *ucmd)
 248 {
 249         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 250
 251         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
 252
 253         if (atomic_dec_and_test(&ucmd->ucmd_ref))
 254                 dev_user_free_ucmd(ucmd);
 255 }
 256
 257 static inline int calc_num_pg(unsigned long buf, int len)
 258 {
 259         len += buf & ~PAGE_MASK;
 260         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
 261 }
 262
 263 static inline int is_need_offs_page(unsigned long buf, int len)
 264 {
 265         return ((buf & ~PAGE_MASK) != 0) &&
 266                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
 267 }
 268
 269 static void __dev_user_not_reg(void)
 270 {
 271         PRINT_ERROR("%s", "Device not registered");
 272         return;
 273 }
 274
 275 static inline int dev_user_check_reg(struct scst_user_dev *dev)
 276 {
 277         if (dev == NULL) {
 278                 __dev_user_not_reg();
 279                 return -EINVAL;
 280         }
 281         return 0;
 282 }
 283
 284 static inline int scst_user_cmd_hashfn(int h)
 285 {
 286         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
 287 }
 288
 289 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
 290         unsigned int h)
 291 {
 292         struct list_head *head;
 293         struct scst_user_cmd *ucmd;
 294
 295         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
 296         list_for_each_entry(ucmd, head, hash_list_entry) {
 297                 if (ucmd->h == h) {
 298                         TRACE_DBG("Found ucmd %p", ucmd);
 299                         return ucmd;
 300                 }
 301         }
 302         return NULL;
 303 }
 304
 305 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
 306 {
 307         struct list_head *head;
 308         struct scst_user_dev *dev = ucmd->dev;
 309         struct scst_user_cmd *u;
 310         unsigned long flags;
 311
 312         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
 313         do {
 314                 ucmd->h = dev->handle_counter++;
 315                 u = __ucmd_find_hash(dev, ucmd->h);
 316         } while (u != NULL);
 317         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
 318         list_add_tail(&ucmd->hash_list_entry, head);
 319         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
 320
 321         TRACE_DBG("Inserted ucmd %p, h=%d", ucmd, ucmd->h);
 322         return;
 323 }
 324
 325 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
 326 {
 327         unsigned long flags;
 328
 329         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 330         list_del(&ucmd->hash_list_entry);
 331         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 332
 333         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
 334         return;
 335 }
 336
 337 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
 338 {
 339         TRACE_ENTRY();
 340
 341         TRACE_MEM("Freeing ucmd %p", ucmd);
 342
 343         cmd_remove_hash(ucmd);
 344         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
 345
 346         kmem_cache_free(user_cmd_cachep, ucmd);
 347
 348         TRACE_EXIT();
 349         return;
 350 }
 351
 352 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 353         gfp_t gfp_mask, void *priv)
 354 {
 355         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 356         int offset = 0;
 357
 358         TRACE_ENTRY();
 359
 360         /* *sg supposed to be zeroed */
 361
 362         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
 363                 ucmd->ubuff, ucmd->cur_data_page);
 364
 365         if (ucmd->cur_data_page == 0) {
 366                 TRACE_MEM("ucmd->first_page_offset %d",
 367                         ucmd->first_page_offset);
 368                 offset = ucmd->first_page_offset;
 369                 ucmd_get(ucmd);
 370         }
 371
 372         if (ucmd->cur_data_page >= ucmd->num_data_pages)
 373                 goto out;
 374
 375         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
 376                 PAGE_SIZE - offset, offset);
 377         ucmd->cur_data_page++;
 378
 379         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
 380                 sg->offset);
 381         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
 382
 383 out:
 384         TRACE_EXIT();
 385         return sg_page(sg);
 386 }
 387
 388 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
 389 {
 390         TRACE_ENTRY();
 391
 392         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
 393                 ucmd, ucmd->h, ucmd->ubuff);
 394
 395         ucmd->user_cmd.cmd_h = ucmd->h;
 396         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
 397         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
 398
 399         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
 400
 401         dev_user_add_to_ready(ucmd);
 402
 403         TRACE_EXIT();
 404         return;
 405 }
 406
 407 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
 408 {
 409         int i;
 410
 411         TRACE_ENTRY();
 412
 413         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
 414                 ucmd->ubuff, ucmd->num_data_pages);
 415
 416         for (i = 0; i < ucmd->num_data_pages; i++) {
 417                 struct page *page = ucmd->data_pages[i];
 418
 419                 if (ucmd->buf_dirty)
 420                         SetPageDirty(page);
 421
 422                 page_cache_release(page);
 423         }
 424
 425         kfree(ucmd->data_pages);
 426         ucmd->data_pages = NULL;
 427
 428         TRACE_EXIT();
 429         return;
 430 }
 431
 432 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
 433 {
 434         TRACE_ENTRY();
 435
 436         sBUG_ON(ucmd->data_pages == NULL);
 437
 438         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
 439                 ucmd, ucmd->ubuff, ucmd->buff_cached);
 440
 441         dev_user_unmap_buf(ucmd);
 442
 443         if (ucmd->buff_cached)
 444                 dev_user_on_cached_mem_free(ucmd);
 445         else
 446                 ucmd_put(ucmd);
 447
 448         TRACE_EXIT();
 449         return;
 450 }
 451
 452 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 453         void *priv)
 454 {
 455         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 456
 457         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
 458                 sg_count, ucmd);
 459
 460         __dev_user_free_sg_entries(ucmd);
 461
 462         return;
 463 }
 464
 465 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
 466 {
 467         int mem_reuse_type = ucmd->dev->memory_reuse_type;
 468
 469         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
 470             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
 471              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
 472             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
 473              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
 474                 return 1;
 475         else
 476                 return 0;
 477 }
 478
 479 /*
 480  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
 481  * Unmaps the buffer, if needed in case of error
 482  */
 483 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 484 {
 485         int res = 0;
 486         struct scst_cmd *cmd = ucmd->cmd;
 487         struct scst_user_dev *dev = ucmd->dev;
 488         gfp_t gfp_mask;
 489         int flags = 0;
 490         int bufflen = cmd->bufflen;
 491         int last_len = 0;
 492
 493         TRACE_ENTRY();
 494
 495         sBUG_ON(bufflen == 0);
 496
 497         gfp_mask = __GFP_NOWARN;
 498         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
 499
 500         if (cached_buff) {
 501                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
 502                 if (ucmd->ubuff == 0)
 503                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
 504         } else {
 505                 TRACE_MEM("%s", "Not cached buff");
 506                 flags |= SCST_POOL_ALLOC_NO_CACHED;
 507                 if (ucmd->ubuff == 0) {
 508                         res = 1;
 509                         goto out;
 510                 }
 511                 bufflen += ucmd->first_page_offset;
 512                 if (is_need_offs_page(ucmd->ubuff, cmd->bufflen))
 513                         last_len = bufflen & ~PAGE_MASK;
 514                 else
 515                         last_len = cmd->bufflen & ~PAGE_MASK;
 516                 if (last_len == 0)
 517                         last_len = PAGE_SIZE;
 518         }
 519         ucmd->buff_cached = cached_buff;
 520
 521         cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
 522                         &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
 523         if (cmd->sg != NULL) {
 524                 struct scst_user_cmd *buf_ucmd =
 525                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
 526
 527                 TRACE_MEM("Buf ucmd %p", buf_ucmd);
 528
 529                 ucmd->ubuff = buf_ucmd->ubuff;
 530                 ucmd->buf_ucmd = buf_ucmd;
 531
 532                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
 533                                    (ucmd != buf_ucmd));
 534
 535                 if (last_len != 0) {
 536                         /* We don't use clustering, so the assignment is safe */
 537                         cmd->sg[cmd->sg_cnt-1].length = last_len;
 538                 }
 539
 540                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
 541                         "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
 542                         last_len, cmd->sg[cmd->sg_cnt-1].length);
 543
 544                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
 545                         static int ll;
 546                         if (ll < 10) {
 547                                 PRINT_INFO("Unable to complete command due to "
 548                                         "SG IO count limitation (requested %d, "
 549                                         "available %d, tgt lim %d)", cmd->sg_cnt,
 550                                         cmd->tgt_dev->max_sg_cnt,
 551                                         cmd->tgt->sg_tablesize);
 552                                 ll++;
 553                         }
 554                         cmd->sg = NULL;
 555                         /* sgv will be freed in dev_user_free_sgv() */
 556                         res = -1;
 557                 }
 558         } else {
 559                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
 560                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
 561                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
 562                 if (unlikely(cmd->sg_cnt == 0)) {
 563                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
 564                         sBUG_ON(ucmd->sgv != NULL);
 565                         res = -1;
 566                 } else {
 567                         switch (ucmd->state) {
 568                         case UCMD_STATE_BUF_ALLOCING:
 569                                 res = 1;
 570                                 break;
 571                         case UCMD_STATE_EXECING:
 572                                 res = -1;
 573                                 break;
 574                         default:
 575                                 sBUG();
 576                                 break;
 577                         }
 578                 }
 579         }
 580
 581 out:
 582         TRACE_EXIT_RES(res);
 583         return res;
 584 }
 585
 586 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
 587 {
 588         int rc, res = SCST_CMD_STATE_DEFAULT;
 589         struct scst_cmd *cmd = ucmd->cmd;
 590
 591         TRACE_ENTRY();
 592
 593         if (unlikely(ucmd->cmd->data_buf_tgt_alloc)) {
 594                 PRINT_ERROR("Target driver %s requested own memory "
 595                         "allocation", ucmd->cmd->tgtt->name);
 596                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 597                 res = scst_get_cmd_abnormal_done_state(cmd);
 598                 goto out;
 599         }
 600
 601         ucmd->state = UCMD_STATE_BUF_ALLOCING;
 602         cmd->data_buf_alloced = 1;
 603
 604         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
 605         if (rc == 0)
 606                 goto out;
 607         else if (rc < 0) {
 608                 scst_set_busy(cmd);
 609                 res = scst_get_cmd_abnormal_done_state(cmd);
 610                 goto out;
 611         }
 612
 613         if ((cmd->data_direction != SCST_DATA_WRITE) &&
 614             !scst_is_cmd_local(cmd)) {
 615                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
 616                 goto out;
 617         }
 618
 619         ucmd->user_cmd.cmd_h = ucmd->h;
 620         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
 621         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 622         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb,
 623                 min(sizeof(ucmd->user_cmd.alloc_cmd.cdb), sizeof(cmd->cdb)));
 624         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
 625         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
 626                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 627         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
 628         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
 629         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
 630
 631         dev_user_add_to_ready(ucmd);
 632
 633         res = SCST_CMD_STATE_STOP;
 634
 635 out:
 636         TRACE_EXIT_RES(res);
 637         return res;
 638 }
 639
 640 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 641         gfp_t gfp_mask)
 642 {
 643         struct scst_user_cmd *ucmd = NULL;
 644
 645         TRACE_ENTRY();
 646
 647 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
 648         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
 649         if (ucmd != NULL)
 650                 memset(ucmd, 0, sizeof(*ucmd));
 651 #else
 652         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
 653 #endif
 654         if (unlikely(ucmd == NULL)) {
 655                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
 656                         "user cmd (gfp_mask %x)", gfp_mask);
 657                 goto out;
 658         }
 659         ucmd->dev = dev;
 660         atomic_set(&ucmd->ucmd_ref, 1);
 661
 662         cmd_insert_hash(ucmd);
 663
 664         TRACE_MEM("ucmd %p allocated", ucmd);
 665
 666 out:
 667         TRACE_EXIT_HRES((unsigned long)ucmd);
 668         return ucmd;
 669 }
 670
 671 static int dev_user_get_block(struct scst_cmd *cmd)
 672 {
 673         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 674         /*
 675          * No need for locks here, since *_detach() can not be
 676          * called, when there are existing commands.
 677          */
 678         TRACE_EXIT_RES(dev->block);
 679         return dev->block;
 680 }
 681
 682 static int dev_user_parse(struct scst_cmd *cmd)
 683 {
 684         int rc, res = SCST_CMD_STATE_DEFAULT;
 685         struct scst_user_cmd *ucmd;
 686         int atomic = scst_cmd_atomic(cmd);
 687         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 688         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
 689
 690         TRACE_ENTRY();
 691
 692         if (cmd->dh_priv == NULL) {
 693                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
 694                 if (unlikely(ucmd == NULL)) {
 695                         if (atomic) {
 696                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
 697                                 goto out;
 698                         } else {
 699                                 scst_set_busy(cmd);
 700                                 goto out_error;
 701                         }
 702                 }
 703                 ucmd->cmd = cmd;
 704                 cmd->dh_priv = ucmd;
 705         } else {
 706                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 707                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
 708         }
 709
 710         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
 711
 712         if (ucmd->state != UCMD_STATE_NEW)
 713                 goto alloc;
 714
 715         switch (dev->parse_type) {
 716         case SCST_USER_PARSE_STANDARD:
 717                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
 718                 rc = dev->generic_parse(cmd, dev_user_get_block);
 719                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
 720                         goto out_invalid;
 721                 break;
 722
 723         case SCST_USER_PARSE_EXCEPTION:
 724                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
 725                 rc = dev->generic_parse(cmd, dev_user_get_block);
 726                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
 727                         break;
 728                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
 729                         TRACE_MEM("Restarting PARSE to thread context "
 730                                 "(ucmd %p)", ucmd);
 731                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
 732                         goto out;
 733                 }
 734                 /* else go through */
 735
 736         case SCST_USER_PARSE_CALL:
 737                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
 738                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
 739                 ucmd->user_cmd.cmd_h = ucmd->h;
 740                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
 741                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 742                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb,
 743                         min(sizeof(ucmd->user_cmd.parse_cmd.cdb),
 744                             sizeof(cmd->cdb)));
 745                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
 746                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
 747                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
 748                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
 749                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
 750                 ucmd->user_cmd.parse_cmd.expected_values_set =
 751                                         cmd->expected_values_set;
 752                 ucmd->user_cmd.parse_cmd.expected_data_direction =
 753                                         cmd->expected_data_direction;
 754                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
 755                                         cmd->expected_transfer_len;
 756                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
 757                 ucmd->state = UCMD_STATE_PARSING;
 758                 dev_user_add_to_ready(ucmd);
 759                 res = SCST_CMD_STATE_STOP;
 760                 goto out;
 761
 762         default:
 763                 sBUG();
 764                 goto out;
 765         }
 766
 767 alloc:
 768         if (cmd->data_direction != SCST_DATA_NONE)
 769                 res = dev_user_alloc_space(ucmd);
 770
 771 out:
 772         TRACE_EXIT_RES(res);
 773         return res;
 774
 775 out_invalid:
 776         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
 777                 cmd->op_flags & SCST_INFO_INVALID);
 778         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 779
 780 out_error:
 781         res = scst_get_cmd_abnormal_done_state(cmd);
 782         goto out;
 783 }
 784
 785 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
 786 {
 787         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
 788         unsigned long start = buf_ucmd->ubuff;
 789         int i;
 790
 791         TRACE_ENTRY();
 792
 793         if (start == 0)
 794                 goto out;
 795
 796         for (i = 0; i < buf_ucmd->num_data_pages; i++) {
 797                 struct page *page;
 798                 page = buf_ucmd->data_pages[i];
 799 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
 800                 struct vm_area_struct *vma = find_vma(current->mm, start);
 801                 if (vma != NULL)
 802                         flush_anon_page(vma, page, start);
 803 #endif
 804                 flush_dcache_page(page);
 805                 start += PAGE_SIZE;
 806         }
 807
 808 out:
 809         TRACE_EXIT();
 810         return;
 811 }
 812
 813 static int dev_user_exec(struct scst_cmd *cmd)
 814 {
 815         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 816         int res = SCST_EXEC_COMPLETED;
 817
 818         TRACE_ENTRY();
 819
 820 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
 821        * context to complete the necessary actions, but all we are going to
 822        * do in this function is, in fact, atomic, so let's skip this check.
 823        */
 824         if (scst_cmd_atomic(cmd)) {
 825                 TRACE_DBG("%s", "User exec() can not be called in atomic "
 826                         "context, rescheduling to the thread");
 827                 res = SCST_EXEC_NEED_THREAD;
 828                 goto out;
 829         }
 830 #endif
 831
 832         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
 833                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
 834                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
 835
 836         if (cmd->data_direction == SCST_DATA_WRITE)
 837                 dev_user_flush_dcache(ucmd);
 838
 839         ucmd->user_cmd.cmd_h = ucmd->h;
 840         ucmd->user_cmd.subcode = SCST_USER_EXEC;
 841         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 842         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb,
 843                 min(sizeof(ucmd->user_cmd.exec_cmd.cdb),
 844                     sizeof(cmd->cdb)));
 845         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
 846         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
 847         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
 848         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
 849         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
 850                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
 851                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 852         }
 853         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
 854         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
 855         ucmd->user_cmd.exec_cmd.partial = 0;
 856         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
 857         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
 858
 859         ucmd->state = UCMD_STATE_EXECING;
 860
 861         dev_user_add_to_ready(ucmd);
 862
 863         TRACE_EXIT_RES(res);
 864         return res;
 865 }
 866
 867 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
 868 {
 869         if (ucmd->sgv != NULL) {
 870                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
 871                 ucmd->sgv = NULL;
 872         } else if (ucmd->data_pages != NULL) {
 873                 /* We mapped pages, but for some reason didn't allocate them */
 874                 ucmd_get(ucmd);
 875                 __dev_user_free_sg_entries(ucmd);
 876         }
 877         return;
 878 }
 879
 880 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
 881 {
 882         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 883
 884         TRACE_ENTRY();
 885
 886         if (unlikely(ucmd == NULL))
 887                 goto out;
 888
 889         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
 890                 ucmd->buff_cached, ucmd->ubuff);
 891
 892         ucmd->cmd = NULL;
 893         if ((cmd->data_direction == SCST_DATA_WRITE) && (ucmd->buf_ucmd != NULL))
 894                 ucmd->buf_ucmd->buf_dirty = 1;
 895
 896         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
 897                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
 898                 /* The state assignment must be before freeing sgv! */
 899                 goto out_reply;
 900         }
 901
 902         if (unlikely(!ucmd->seen_by_user)) {
 903                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
 904                 sBUG_ON((ucmd->sgv != NULL) || (ucmd->data_pages != NULL));
 905                 goto out_reply;
 906         }
 907
 908         ucmd->user_cmd.cmd_h = ucmd->h;
 909         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
 910
 911         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
 912         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
 913         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
 914         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
 915         ucmd->user_cmd.on_free_cmd.status = cmd->status;
 916         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
 917
 918         ucmd->state = UCMD_STATE_ON_FREEING;
 919
 920         dev_user_add_to_ready(ucmd);
 921
 922 out:
 923         TRACE_EXIT();
 924         return;
 925
 926 out_reply:
 927         dev_user_process_reply_on_free(ucmd);
 928         goto out;
 929 }
 930
 931 static void dev_user_set_block(struct scst_cmd *cmd, int block)
 932 {
 933         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 934         /*
 935          * No need for locks here, since *_detach() can not be
 936          * called, when there are existing commands.
 937          */
 938         TRACE_DBG("dev %p, new block %d", dev, block);
 939         if (block != 0)
 940                 dev->block = block;
 941         else
 942                 dev->block = dev->def_block;
 943         return;
 944 }
 945
 946 static int dev_user_disk_done(struct scst_cmd *cmd)
 947 {
 948         int res = SCST_CMD_STATE_DEFAULT;
 949
 950         TRACE_ENTRY();
 951
 952         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
 953
 954         TRACE_EXIT_RES(res);
 955         return res;
 956 }
 957
 958 static int dev_user_tape_done(struct scst_cmd *cmd)
 959 {
 960         int res = SCST_CMD_STATE_DEFAULT;
 961
 962         TRACE_ENTRY();
 963
 964         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
 965
 966         TRACE_EXIT_RES(res);
 967         return res;
 968 }
 969
 970 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
 971 {
 972         struct scst_user_dev *dev = ucmd->dev;
 973         unsigned long flags;
 974         int do_wake;
 975
 976         TRACE_ENTRY();
 977
 978         do_wake = (in_interrupt() ||
 979                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
 980         if (ucmd->cmd)
 981                 do_wake |= ucmd->cmd->preprocessing_only;
 982
 983         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
 984
 985         ucmd->this_state_unjammed = 0;
 986
 987         if ((ucmd->state == UCMD_STATE_PARSING) ||
 988             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
 989                 /*
 990                  * If we don't put such commands in the queue head, then under
 991                  * high load we might delay threads, waiting for memory
 992                  * allocations, for too long and start loosing NOPs, which
 993                  * would lead to consider us by remote initiators as
 994                  * unresponsive and stuck => broken connections, etc. If none
 995                  * of our commands completed in NOP timeout to allow the head
 996                  * commands to go, then we are really overloaded and/or stuck.
 997                  */
 998                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
 999                         "cmd list", ucmd, ucmd->state);
1000                 list_add(&ucmd->ready_cmd_list_entry,
1001                         &dev->ready_cmd_list);
1002                 do_wake = 1;
1003         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1004                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1005                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1006                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1007                         "ready cmd list", ucmd, ucmd->state);
1008                 list_add(&ucmd->ready_cmd_list_entry,
1009                         &dev->ready_cmd_list);
1010                 do_wake = 1;
1011         } else if ((ucmd->cmd != NULL) &&
1012                    unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1013                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1014                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1015         } else {
1016                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1017                 list_add_tail(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1018         }
1019
1020         if (do_wake) {
1021                 TRACE_DBG("Waking up dev %p", dev);
1022                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1023         }
1024
1025         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1026
1027         TRACE_EXIT();
1028         return;
1029 }
1030
1031 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1032         int num_pg)
1033 {
1034         int res = 0, rc;
1035         int i;
1036
1037         TRACE_ENTRY();
1038
1039         if (unlikely(ubuff == 0))
1040                 goto out_nomem;
1041
1042         sBUG_ON(ucmd->data_pages != NULL);
1043
1044         ucmd->num_data_pages = num_pg;
1045
1046         ucmd->data_pages = kmalloc(sizeof(*ucmd->data_pages)*ucmd->num_data_pages,
1047                 GFP_KERNEL);
1048         if (ucmd->data_pages == NULL) {
1049                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1050                         "(num_data_pages=%d)", ucmd->num_data_pages);
1051                 res = -ENOMEM;
1052                 goto out_nomem;
1053         }
1054
1055         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d, "
1056                 "first_page_offset %d, len %d)", ucmd, ubuff,
1057                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1058                 ucmd->cmd->bufflen);
1059
1060         down_read(&current->mm->mmap_sem);
1061         rc = get_user_pages(current, current->mm, ubuff, ucmd->num_data_pages,
1062                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1063         up_read(&current->mm->mmap_sem);
1064
1065         /* get_user_pages() flushes dcache */
1066
1067         if (rc < ucmd->num_data_pages)
1068                 goto out_unmap;
1069
1070         ucmd->ubuff = ubuff;
1071         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1072
1073 out:
1074         TRACE_EXIT_RES(res);
1075         return res;
1076
1077 out_nomem:
1078         scst_set_busy(ucmd->cmd);
1079         /* go through */
1080
1081 out_err:
1082         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1083         goto out;
1084
1085 out_unmap:
1086         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1087                 ucmd->num_data_pages, rc);
1088         if (rc > 0) {
1089                 for (i = 0; i < rc; i++)
1090                         page_cache_release(ucmd->data_pages[i]);
1091         }
1092         kfree(ucmd->data_pages);
1093         ucmd->data_pages = NULL;
1094         res = -EFAULT;
1095         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1096         goto out_err;
1097 }
1098
1099 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1100         struct scst_user_reply_cmd *reply)
1101 {
1102         int res = 0;
1103         struct scst_cmd *cmd = ucmd->cmd;
1104
1105         TRACE_ENTRY();
1106
1107         TRACE_DBG("ucmd %p, pbuf %Lx", ucmd, reply->alloc_reply.pbuf);
1108
1109         if (likely(reply->alloc_reply.pbuf != 0)) {
1110                 int pages;
1111                 if (ucmd->buff_cached) {
1112                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1113                                 PRINT_ERROR("Supplied pbuf %Lx isn't "
1114                                         "page aligned", reply->alloc_reply.pbuf);
1115                                 goto out_hwerr;
1116                         }
1117                         pages = cmd->sg_cnt;
1118                 } else
1119                         pages = calc_num_pg(reply->alloc_reply.pbuf, cmd->bufflen);
1120                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1121         } else {
1122                 scst_set_busy(ucmd->cmd);
1123                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1124         }
1125
1126 out_process:
1127         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1128
1129         TRACE_EXIT_RES(res);
1130         return res;
1131
1132 out_hwerr:
1133         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1134         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1135         res = -EINVAL;
1136         goto out_process;
1137 }
1138
1139 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1140         struct scst_user_reply_cmd *reply)
1141 {
1142         int res = 0;
1143         struct scst_user_scsi_cmd_reply_parse *preply =
1144                 &reply->parse_reply;
1145         struct scst_cmd *cmd = ucmd->cmd;
1146
1147         TRACE_ENTRY();
1148
1149         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1150                 goto out_inval;
1151
1152         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1153                      (preply->data_direction != SCST_DATA_READ) &&
1154                      (preply->data_direction != SCST_DATA_NONE)))
1155                 goto out_inval;
1156
1157         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1158                      (preply->bufflen == 0)))
1159                 goto out_inval;
1160
1161         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1162                 goto out_inval;
1163
1164         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1165                 "data_len %d, pbuf %Lx", ucmd, preply->queue_type,
1166                 preply->data_direction, preply->bufflen, preply->data_len,
1167                 reply->alloc_reply.pbuf);
1168
1169         cmd->queue_type = preply->queue_type;
1170         cmd->data_direction = preply->data_direction;
1171         cmd->bufflen = preply->bufflen;
1172         cmd->data_len = preply->data_len;
1173
1174 out_process:
1175         scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT);
1176
1177         TRACE_EXIT_RES(res);
1178         return res;
1179
1180 out_inval:
1181         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1182                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1183         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1184         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1185         scst_set_cmd_abnormal_done_state(cmd);
1186         res = -EINVAL;
1187         goto out_process;
1188 }
1189
1190 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1191 {
1192         int res = 0;
1193
1194         TRACE_ENTRY();
1195
1196         TRACE_DBG("ON FREE ucmd %p", ucmd);
1197
1198         dev_user_free_sgv(ucmd);
1199         ucmd_put(ucmd);
1200
1201         TRACE_EXIT_RES(res);
1202         return res;
1203 }
1204
1205 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1206 {
1207         int res = 0;
1208
1209         TRACE_ENTRY();
1210
1211         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1212
1213         ucmd_put(ucmd);
1214
1215         TRACE_EXIT_RES(res);
1216         return res;
1217 }
1218
1219 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1220         struct scst_user_reply_cmd *reply)
1221 {
1222         int res = 0;
1223         struct scst_user_scsi_cmd_reply_exec *ereply =
1224                 &reply->exec_reply;
1225         struct scst_cmd *cmd = ucmd->cmd;
1226
1227         TRACE_ENTRY();
1228
1229         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1230                 if (ucmd->background_exec) {
1231                         TRACE_DBG("Background ucmd %p finished", ucmd);
1232                         ucmd_put(ucmd);
1233                         goto out;
1234                 }
1235                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1236                         goto out_inval;
1237                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1238                              (ereply->resp_data_len != 0)))
1239                         goto out_inval;
1240         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1241                 if (unlikely(ucmd->background_exec))
1242                         goto out_inval;
1243                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1244                              (cmd->resp_data_len != 0)))
1245                         goto out_inval;
1246                 ucmd_get_ordered(ucmd);
1247                 ucmd->background_exec = 1;
1248                 TRACE_DBG("Background ucmd %p", ucmd);
1249                 goto out_compl;
1250         } else
1251                 goto out_inval;
1252
1253         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1254                 ereply->status, ereply->resp_data_len);
1255
1256          if (ereply->resp_data_len != 0) {
1257                 if (ucmd->ubuff == 0) {
1258                         int pages, rc;
1259                         if (unlikely(ereply->pbuf == 0))
1260                                 goto out_busy;
1261                         if (ucmd->buff_cached) {
1262                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1263                                         PRINT_ERROR("Supplied pbuf %Lx isn't "
1264                                                 "page aligned", ereply->pbuf);
1265                                         goto out_hwerr;
1266                                 }
1267                                 pages = cmd->sg_cnt;
1268                         } else
1269                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1270                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1271                         if ((rc != 0) || (ucmd->ubuff == 0))
1272                                 goto out_compl;
1273
1274                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1275                         if (unlikely(rc != 0))
1276                                 goto out_busy;
1277                 } else
1278                         dev_user_flush_dcache(ucmd);
1279                 cmd->may_need_dma_sync = 1;
1280                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1281         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1282                 if (ucmd->ubuff == 0)
1283                         cmd->resp_data_len = ereply->resp_data_len;
1284                 else
1285                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1286         }
1287
1288         cmd->status = ereply->status;
1289         if (ereply->sense_len != 0) {
1290                 res = scst_alloc_sense(cmd, 0);
1291                 if (res != 0)
1292                         goto out_compl;
1293                 res = copy_from_user(cmd->sense,
1294                         (void __user *)(unsigned long)ereply->psense_buffer,
1295                         min((unsigned int)SCST_SENSE_BUFFERSIZE,
1296                                 (unsigned int)ereply->sense_len));
1297                 if (res < 0) {
1298                         PRINT_ERROR("%s", "Unable to get sense data");
1299                         goto out_hwerr_res_set;
1300                 }
1301         }
1302
1303 out_compl:
1304         cmd->completed = 1;
1305         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1306         /* !! At this point cmd can be already freed !! */
1307
1308 out:
1309         TRACE_EXIT_RES(res);
1310         return res;
1311
1312 out_inval:
1313         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1314                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1315         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1316
1317 out_hwerr:
1318         res = -EINVAL;
1319
1320 out_hwerr_res_set:
1321         if (ucmd->background_exec) {
1322                 ucmd_put(ucmd);
1323                 goto out;
1324         } else {
1325                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1326                 goto out_compl;
1327         }
1328
1329 out_busy:
1330         scst_set_busy(cmd);
1331         goto out_compl;
1332 }
1333
1334 static int dev_user_process_reply(struct scst_user_dev *dev,
1335         struct scst_user_reply_cmd *reply)
1336 {
1337         int res = 0;
1338         struct scst_user_cmd *ucmd;
1339         int state;
1340
1341         TRACE_ENTRY();
1342
1343         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1344
1345         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1346         if (unlikely(ucmd == NULL)) {
1347                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1348                 res = -ESRCH;
1349                 goto out_unlock;
1350         }
1351
1352         if (unlikely(ucmd_get_check(ucmd))) {
1353                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1354                 res = -ESRCH;
1355                 goto out_unlock;
1356         }
1357
1358         if (ucmd->background_exec) {
1359                 state = UCMD_STATE_EXECING;
1360                 goto unlock_process;
1361         }
1362
1363         if (unlikely(ucmd->this_state_unjammed)) {
1364                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1365                         ucmd);
1366                 goto out_unlock_put;
1367         }
1368
1369         if (unlikely(!ucmd->sent_to_user)) {
1370                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1371                         "state %x", ucmd, ucmd->state);
1372                 res = -EINVAL;
1373                 goto out_unlock_put;
1374         }
1375
1376         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1377                 goto out_wrong_state;
1378
1379         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1380                 goto out_wrong_state;
1381
1382         state = ucmd->state;
1383         ucmd->sent_to_user = 0;
1384
1385 unlock_process:
1386         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1387
1388         switch (state) {
1389         case UCMD_STATE_PARSING:
1390                 res = dev_user_process_reply_parse(ucmd, reply);
1391                 break;
1392
1393         case UCMD_STATE_BUF_ALLOCING:
1394                 res = dev_user_process_reply_alloc(ucmd, reply);
1395                 break;
1396
1397         case UCMD_STATE_EXECING:
1398                 res = dev_user_process_reply_exec(ucmd, reply);
1399                 break;
1400
1401         case UCMD_STATE_ON_FREEING:
1402                 res = dev_user_process_reply_on_free(ucmd);
1403                 break;
1404
1405         case UCMD_STATE_ON_CACHE_FREEING:
1406                 res = dev_user_process_reply_on_cache_free(ucmd);
1407                 break;
1408
1409         case UCMD_STATE_TM_EXECING:
1410                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1411                 break;
1412
1413         case UCMD_STATE_ATTACH_SESS:
1414         case UCMD_STATE_DETACH_SESS:
1415                 res = dev_user_process_reply_sess(ucmd, reply->result);
1416                 break;
1417
1418         default:
1419                 sBUG();
1420                 break;
1421         }
1422
1423 out_put:
1424         ucmd_put(ucmd);
1425
1426 out:
1427         TRACE_EXIT_RES(res);
1428         return res;
1429
1430 out_wrong_state:
1431         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1432                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1433                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1434                 reply->subcode, ucmd->user_cmd.subcode);
1435         res = -EINVAL;
1436         dev_user_unjam_cmd(ucmd, 0, NULL);
1437
1438 out_unlock_put:
1439         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1440         goto out_put;
1441
1442 out_unlock:
1443         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1444         goto out;
1445 }
1446
1447 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1448 {
1449         int res = 0;
1450         struct scst_user_dev *dev;
1451         struct scst_user_reply_cmd reply;
1452
1453         TRACE_ENTRY();
1454
1455         mutex_lock(&dev_priv_mutex);
1456         dev = (struct scst_user_dev *)file->private_data;
1457         res = dev_user_check_reg(dev);
1458         if (res != 0) {
1459                 mutex_unlock(&dev_priv_mutex);
1460                 goto out;
1461         }
1462         down_read(&dev->dev_rwsem);
1463         mutex_unlock(&dev_priv_mutex);
1464
1465         res = copy_from_user(&reply, arg, sizeof(reply));
1466         if (res < 0)
1467                 goto out_up;
1468
1469         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1470
1471         res = dev_user_process_reply(dev, &reply);
1472         if (res < 0)
1473                 goto out_up;
1474
1475 out_up:
1476         up_read(&dev->dev_rwsem);
1477
1478 out:
1479         TRACE_EXIT_RES(res);
1480         return res;
1481 }
1482
1483 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1484 {
1485         int res = 0;
1486
1487         TRACE_ENTRY();
1488
1489         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1490                 struct scst_cmd *cmd = list_entry(
1491                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1492                         cmd_list_entry);
1493                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1494                 list_del(&cmd->cmd_list_entry);
1495                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1496                 scst_process_active_cmd(cmd, SCST_CONTEXT_DIRECT |
1497                                                  SCST_CONTEXT_PROCESSABLE);
1498                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1499                 res++;
1500         }
1501
1502         TRACE_EXIT_RES(res);
1503         return res;
1504 }
1505
1506 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1507 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1508 {
1509         struct scst_user_cmd *u;
1510
1511 again:
1512         u = NULL;
1513         if (!list_empty(cmd_list)) {
1514                 u = list_entry(cmd_list->next, typeof(*u), ready_cmd_list_entry);
1515
1516                 TRACE_DBG("Found ready ucmd %p", u);
1517                 list_del(&u->ready_cmd_list_entry);
1518
1519                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1520
1521                 if (u->cmd != NULL) {
1522                         if (u->state == UCMD_STATE_EXECING) {
1523                                 struct scst_user_dev *dev = u->dev;
1524                                 int rc;
1525
1526                                 EXTRACHECKS_BUG_ON(u->jammed);
1527
1528                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1529
1530                                 rc = scst_check_local_events(u->cmd);
1531                                 if (unlikely(rc != 0)) {
1532                                         u->cmd->scst_cmd_done(u->cmd,
1533                                                 SCST_CMD_STATE_DEFAULT);
1534                                         /*
1535                                          * !! At this point cmd & u can be !!
1536                                          * !! already freed                !!
1537                                          */
1538                                         spin_lock_irq(
1539                                                 &dev->cmd_lists.cmd_list_lock);
1540                                         goto again;
1541                                 }
1542
1543                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1544                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1545                                         &u->cmd->cmd_flags))) {
1546                                 switch (u->state) {
1547                                 case UCMD_STATE_PARSING:
1548                                 case UCMD_STATE_BUF_ALLOCING:
1549                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1550                                         dev_user_unjam_cmd(u, 0, NULL);
1551                                         goto again;
1552                                 case UCMD_STATE_EXECING:
1553                                         EXTRACHECKS_BUG_ON(1);
1554                                 }
1555                         }
1556                 }
1557                 u->sent_to_user = 1;
1558                 u->seen_by_user = 1;
1559         }
1560         return u;
1561 }
1562
1563 static inline int test_cmd_lists(struct scst_user_dev *dev)
1564 {
1565         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1566                   !list_empty(&dev->ready_cmd_list) ||
1567                   !dev->blocking || dev->cleanup_done ||
1568                   signal_pending(current);
1569         return res;
1570 }
1571
1572 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1573 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1574         struct scst_user_cmd **ucmd)
1575 {
1576         int res = 0;
1577         wait_queue_t wait;
1578
1579         TRACE_ENTRY();
1580
1581         init_waitqueue_entry(&wait, current);
1582
1583         while (1) {
1584                 if (!test_cmd_lists(dev)) {
1585                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1586                                 &wait);
1587                         for (;;) {
1588                                 set_current_state(TASK_INTERRUPTIBLE);
1589                                 if (test_cmd_lists(dev))
1590                                         break;
1591                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1592                                 schedule();
1593                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1594                         }
1595                         set_current_state(TASK_RUNNING);
1596                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1597                                 &wait);
1598                 }
1599
1600                 dev_user_process_scst_commands(dev);
1601
1602                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1603                 if (*ucmd != NULL)
1604                         break;
1605
1606                 if (!dev->blocking || dev->cleanup_done) {
1607                         res = -EAGAIN;
1608                         TRACE_DBG("No ready commands, returning %d", res);
1609                         break;
1610                 }
1611
1612                 if (signal_pending(current)) {
1613                         res = -EINTR;
1614                         TRACE_DBG("Signal pending, returning %d", res);
1615                         break;
1616                 }
1617         }
1618
1619         TRACE_EXIT_RES(res);
1620         return res;
1621 }
1622
1623 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1624 {
1625         int res = 0;
1626         struct scst_user_dev *dev;
1627         struct scst_user_get_cmd *cmd;
1628         struct scst_user_reply_cmd *reply;
1629         struct scst_user_cmd *ucmd;
1630         uint64_t ureply;
1631
1632         TRACE_ENTRY();
1633
1634         mutex_lock(&dev_priv_mutex);
1635         dev = (struct scst_user_dev *)file->private_data;
1636         res = dev_user_check_reg(dev);
1637         if (res != 0) {
1638                 mutex_unlock(&dev_priv_mutex);
1639                 goto out;
1640         }
1641         down_read(&dev->dev_rwsem);
1642         mutex_unlock(&dev_priv_mutex);
1643
1644         res = copy_from_user(&ureply, arg, sizeof(ureply));
1645         if (res < 0)
1646                 goto out_up;
1647
1648         TRACE_DBG("ureply %Ld", (long long unsigned int)ureply);
1649
1650         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1651         if (cmd == NULL) {
1652                 res = -ENOMEM;
1653                 goto out_up;
1654         }
1655
1656         if (ureply != 0) {
1657                 unsigned long u = (unsigned long)ureply;
1658                 reply = (struct scst_user_reply_cmd *)cmd;
1659                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1660                 if (res < 0)
1661                         goto out_free;
1662
1663                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1664
1665                 res = dev_user_process_reply(dev, reply);
1666                 if (res < 0)
1667                         goto out_free;
1668         }
1669
1670         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1671         res = dev_user_get_next_cmd(dev, &ucmd);
1672         if (res == 0) {
1673                 *cmd = ucmd->user_cmd;
1674                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1675                 TRACE_BUFFER("UCMD", cmd, sizeof(*cmd));
1676                 res = copy_to_user(arg, cmd, sizeof(*cmd));
1677         } else
1678                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1679
1680 out_free:
1681         kmem_cache_free(user_get_cmd_cachep, cmd);
1682
1683 out_up:
1684         up_read(&dev->dev_rwsem);
1685
1686 out:
1687         TRACE_EXIT_RES(res);
1688         return res;
1689 }
1690
1691 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1692         unsigned long arg)
1693 {
1694         long res;
1695
1696         TRACE_ENTRY();
1697
1698         switch (cmd) {
1699         case SCST_USER_REPLY_AND_GET_CMD:
1700                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1701                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1702                 break;
1703
1704         case SCST_USER_REPLY_CMD:
1705                 TRACE_DBG("%s", "REPLY_CMD");
1706                 res = dev_user_reply_cmd(file, (void __user *)arg);
1707                 break;
1708
1709         case SCST_USER_REGISTER_DEVICE:
1710         {
1711                 struct scst_user_dev_desc *dev_desc;
1712                 TRACE_DBG("%s", "REGISTER_DEVICE");
1713                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1714                 if (dev_desc == NULL) {
1715                         res = -ENOMEM;
1716                         goto out;
1717                 }
1718                 res = copy_from_user(dev_desc, (void __user *)arg, sizeof(*dev_desc));
1719                 if (res < 0) {
1720                         kfree(dev_desc);
1721                         goto out;
1722                 }
1723                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1724                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1725                 res = dev_user_register_dev(file, dev_desc);
1726                 kfree(dev_desc);
1727                 break;
1728         }
1729
1730         case SCST_USER_SET_OPTIONS:
1731         {
1732                 struct scst_user_opt opt;
1733                 TRACE_DBG("%s", "SET_OPTIONS");
1734                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1735                 if (res < 0)
1736                         goto out;
1737                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1738                 res = dev_user_set_opt(file, &opt);
1739                 break;
1740         }
1741
1742         case SCST_USER_GET_OPTIONS:
1743                 TRACE_DBG("%s", "GET_OPTIONS");
1744                 res = dev_user_get_opt(file, (void __user *)arg);
1745                 break;
1746
1747         default:
1748                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1749                 res = -EINVAL;
1750                 goto out;
1751         }
1752
1753 out:
1754         TRACE_EXIT_RES(res);
1755         return res;
1756 }
1757
1758 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1759 {
1760         int res = 0;
1761         struct scst_user_dev *dev;
1762
1763         TRACE_ENTRY();
1764
1765         mutex_lock(&dev_priv_mutex);
1766         dev = (struct scst_user_dev *)file->private_data;
1767         res = dev_user_check_reg(dev);
1768         if (res != 0) {
1769                 mutex_unlock(&dev_priv_mutex);
1770                 goto out;
1771         }
1772         down_read(&dev->dev_rwsem);
1773         mutex_unlock(&dev_priv_mutex);
1774
1775         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1776
1777         if (!list_empty(&dev->ready_cmd_list) ||
1778             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1779                 res |= POLLIN | POLLRDNORM;
1780                 goto out_unlock;
1781         }
1782
1783         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1784
1785         TRACE_DBG("Before poll_wait() (dev %p)", dev);
1786         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
1787         TRACE_DBG("After poll_wait() (dev %p)", dev);
1788
1789         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1790
1791         if (!list_empty(&dev->ready_cmd_list) ||
1792             !list_empty(&dev->cmd_lists.active_cmd_list)) {
1793                 res |= POLLIN | POLLRDNORM;
1794                 goto out_unlock;
1795         }
1796
1797 out_unlock:
1798         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1799
1800         up_read(&dev->dev_rwsem);
1801
1802 out:
1803         TRACE_EXIT_HRES(res);
1804         return res;
1805 }
1806
1807 /*
1808  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reaquire.
1809  */
1810 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
1811         unsigned long *flags)
1812 {
1813         int state = ucmd->state;
1814         struct scst_user_dev *dev = ucmd->dev;
1815
1816         TRACE_ENTRY();
1817
1818         if (ucmd->this_state_unjammed)
1819                 goto out;
1820
1821         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
1822                 state);
1823
1824         ucmd->jammed = 1;
1825         ucmd->this_state_unjammed = 1;
1826         ucmd->sent_to_user = 0;
1827
1828         switch (state) {
1829         case UCMD_STATE_PARSING:
1830         case UCMD_STATE_BUF_ALLOCING:
1831                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
1832                         ucmd->aborted = 1;
1833                 else {
1834                         if (busy)
1835                                 scst_set_busy(ucmd->cmd);
1836                         else
1837                                 scst_set_cmd_error(ucmd->cmd,
1838                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1839                 }
1840                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1841
1842                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
1843                 list_add(&ucmd->cmd->cmd_list_entry,
1844                         &ucmd->cmd->cmd_lists->active_cmd_list);
1845                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
1846                 break;
1847
1848         case UCMD_STATE_EXECING:
1849                 if (flags != NULL)
1850                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1851                 else
1852                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1853
1854                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
1855
1856                 if (test_bit(SCST_CMD_ABORTED,  &ucmd->cmd->cmd_flags))
1857                         ucmd->aborted = 1;
1858                 else {
1859                         if (busy)
1860                                 scst_set_busy(ucmd->cmd);
1861                         else
1862                                 scst_set_cmd_error(ucmd->cmd,
1863                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
1864                 }
1865
1866                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT);
1867                 /* !! At this point cmd and ucmd can be already freed !! */
1868
1869                 if (flags != NULL)
1870                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1871                 else
1872                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1873                 break;
1874
1875         case UCMD_STATE_ON_FREEING:
1876         case UCMD_STATE_ON_CACHE_FREEING:
1877         case UCMD_STATE_TM_EXECING:
1878         case UCMD_STATE_ATTACH_SESS:
1879         case UCMD_STATE_DETACH_SESS:
1880         {
1881                 if (flags != NULL)
1882                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, *flags);
1883                 else
1884                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1885
1886                 switch (state) {
1887                 case UCMD_STATE_ON_FREEING:
1888                         dev_user_process_reply_on_free(ucmd);
1889                         break;
1890
1891                 case UCMD_STATE_ON_CACHE_FREEING:
1892                         dev_user_process_reply_on_cache_free(ucmd);
1893                         break;
1894
1895                 case UCMD_STATE_TM_EXECING:
1896                         dev_user_process_reply_tm_exec(ucmd, SCST_MGMT_STATUS_FAILED);
1897                         break;
1898
1899                 case UCMD_STATE_ATTACH_SESS:
1900                 case UCMD_STATE_DETACH_SESS:
1901                         dev_user_process_reply_sess(ucmd, -EFAULT);
1902                         break;
1903                 }
1904
1905                 if (flags != NULL)
1906                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, *flags);
1907                 else
1908                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1909                 break;
1910         }
1911
1912         default:
1913                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
1914                 sBUG();
1915                 break;
1916         }
1917
1918 out:
1919         TRACE_EXIT();
1920         return;
1921 }
1922
1923 static void dev_user_unjam_dev(struct scst_user_dev *dev)
1924 {
1925         int i;
1926         struct scst_user_cmd *ucmd;
1927
1928         TRACE_ENTRY();
1929
1930         TRACE_MGMT_DBG("Unjamming dev %p", dev);
1931
1932         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1933
1934 repeat:
1935         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
1936                 struct list_head *head = &dev->ucmd_hash[i];
1937
1938                 list_for_each_entry(ucmd, head, hash_list_entry) {
1939                         if (!ucmd->sent_to_user)
1940                                 continue;
1941
1942                         if (ucmd_get_check(ucmd))
1943                                 continue;
1944
1945                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
1946                                 ucmd->state, ucmd->cmd);
1947
1948                         dev_user_unjam_cmd(ucmd, 0, NULL);
1949
1950                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1951                         ucmd_put(ucmd);
1952                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1953
1954                         goto repeat;
1955                 }
1956         }
1957
1958         if (dev_user_process_scst_commands(dev) != 0)
1959                 goto repeat;
1960
1961         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1962
1963         TRACE_EXIT();
1964         return;
1965 }
1966
1967 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
1968         int status)
1969 {
1970         int res = 0;
1971
1972         TRACE_ENTRY();
1973
1974         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
1975                 ucmd->user_cmd.tm_cmd.fn, status);
1976
1977         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
1978                 /*
1979                  * It is possible that user space seen TM cmd before cmd
1980                  * to abort or will never see it at all, because it was
1981                  * aborted on the way there. So, it is safe to return
1982                  * success instead, because, if there is the TM cmd at this
1983                  * point, then the cmd to abort apparrently does exist.
1984                  */
1985                 status = SCST_MGMT_STATUS_SUCCESS;
1986         }
1987
1988         scst_async_mcmd_completed(ucmd->mcmd, status);
1989
1990         ucmd_put(ucmd);
1991
1992         TRACE_EXIT_RES(res);
1993         return res;
1994 }
1995
1996 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
1997 {
1998         struct scst_user_cmd *ucmd;
1999         unsigned long flags;
2000
2001         TRACE_ENTRY();
2002
2003         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2004 again:
2005         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2006                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2007                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2008                         switch (ucmd->state) {
2009                         case UCMD_STATE_PARSING:
2010                         case UCMD_STATE_BUF_ALLOCING:
2011                         case UCMD_STATE_EXECING:
2012                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2013                                 list_del(&ucmd->ready_cmd_list_entry);
2014                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2015                                 goto again;
2016                         }
2017                 }
2018         }
2019
2020         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2021
2022         TRACE_EXIT();
2023         return;
2024 }
2025
2026 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2027         struct scst_tgt_dev *tgt_dev)
2028 {
2029         struct scst_user_cmd *ucmd;
2030         struct scst_user_dev *dev = (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2031         struct scst_user_cmd *ucmd_to_abort = NULL;
2032
2033         TRACE_ENTRY();
2034
2035         /*
2036          * In the used approach we don't do anything with hung devices, which
2037          * stopped responding and/or have stuck commands. We forcedly abort such
2038          * commands only if they not yet sent to the user space or if the device
2039          * is getting unloaded, e.g. if its handler program gets killed. This is
2040          * because it's pretty hard to distinguish between stuck and temporary
2041          * overloaded states of the device. There are several reasons for that:
2042          *
2043          * 1. Some commands need a lot of time to complete (several hours),
2044          *    so for an impatient user such command(s) will always look as
2045          *    stuck.
2046          *
2047          * 2. If we forcedly abort, i.e. abort before it's actually completed
2048          *    in the user space, just one command, we will have to put the whole
2049          *    device offline until we are sure that no more previously aborted
2050          *    commands will get executed. Otherwise, we might have a possibility
2051          *    for data corruption, when aborted and reported as completed
2052          *    command actually gets executed *after* new commands sent
2053          *    after the force abort was done. Many journaling file systems and
2054          *    databases use "provide required commands order via queue draining"
2055          *    approach and not putting the whole device offline after the forced
2056          *    abort will break it. This makes our decision, if a command stuck
2057          *    or not, cost a lot.
2058          *
2059          * So, we leave policy definition if a device stuck or not to
2060          * the user space and simply let all commands live until they are
2061          * completed or their devices get closed/killed. This approach is very
2062          * much OK, but can affect management commands, which need activity
2063          * suspending via scst_suspend_activity() function such as devices or
2064          * targets registration/removal. But during normal life such commands
2065          * should be rare. Plus, when possible, scst_suspend_activity() will
2066          * return after timeout EBUSY status to allow caller to not stuck
2067          * forever as well.
2068          *
2069          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2070          * stuck commands would affect only related devices.
2071          */
2072
2073         dev_user_abort_ready_commands(dev);
2074
2075         /* We can't afford missing TM command due to memory shortage */
2076         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2077
2078         ucmd->user_cmd.cmd_h = ucmd->h;
2079         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2080         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2081         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2082         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2083         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2084
2085         if (mcmd->cmd_to_abort != NULL) {
2086                 ucmd_to_abort = (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2087                 if (ucmd_to_abort != NULL)
2088                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2089         }
2090
2091         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2092                 "ucmd_to_abort %p, cmd_h_to_abort %d)", ucmd, ucmd->h,
2093                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2094                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort);
2095
2096         ucmd->mcmd = mcmd;
2097         ucmd->state = UCMD_STATE_TM_EXECING;
2098
2099         scst_prepare_async_mcmd(mcmd);
2100
2101         dev_user_add_to_ready(ucmd);
2102
2103         TRACE_EXIT();
2104         return SCST_DEV_TM_NOT_COMPLETED;
2105 }
2106
2107 static int dev_user_attach(struct scst_device *sdev)
2108 {
2109         int res = 0;
2110         struct scst_user_dev *dev = NULL, *d;
2111
2112         TRACE_ENTRY();
2113
2114         spin_lock(&dev_list_lock);
2115         list_for_each_entry(d, &dev_list, dev_list_entry) {
2116                 if (strcmp(d->name, sdev->virt_name) == 0) {
2117                         dev = d;
2118                         break;
2119                 }
2120         }
2121         spin_unlock(&dev_list_lock);
2122         if (dev == NULL) {
2123                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2124                 res = -EINVAL;
2125                 goto out;
2126         }
2127
2128         sdev->p_cmd_lists = &dev->cmd_lists;
2129         sdev->dh_priv = dev;
2130         sdev->tst = dev->tst;
2131         sdev->queue_alg = dev->queue_alg;
2132         sdev->swp = dev->swp;
2133         sdev->tas = dev->tas;
2134         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2135
2136         dev->sdev = sdev;
2137
2138         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2139                 dev->name);
2140
2141 out:
2142         TRACE_EXIT();
2143         return res;
2144 }
2145
2146 static void dev_user_detach(struct scst_device *sdev)
2147 {
2148         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2149
2150         TRACE_ENTRY();
2151
2152         TRACE_DBG("virt_id %d", sdev->virt_id);
2153
2154         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2155                 dev->name);
2156
2157         /* dev will be freed by the caller */
2158         sdev->dh_priv = NULL;
2159         dev->sdev = NULL;
2160
2161         TRACE_EXIT();
2162         return;
2163 }
2164
2165 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2166 {
2167         int res = 0;
2168         unsigned long flags;
2169
2170         TRACE_ENTRY();
2171
2172         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2173
2174         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2175
2176         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2177                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2178                 ucmd->result = status;
2179         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2180                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2181         } else
2182                 sBUG();
2183
2184         if (ucmd->cmpl != NULL)
2185                 complete_all(ucmd->cmpl);
2186
2187         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2188
2189         ucmd_put(ucmd);
2190
2191         TRACE_EXIT_RES(res);
2192         return res;
2193 }
2194
2195 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2196 {
2197         struct scst_user_dev *dev =
2198                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2199         int res = 0, rc;
2200         struct scst_user_cmd *ucmd;
2201         DECLARE_COMPLETION_ONSTACK(cmpl);
2202
2203         TRACE_ENTRY();
2204
2205         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2206         if (ucmd == NULL)
2207                 goto out_nomem;
2208
2209         ucmd->cmpl = &cmpl;
2210
2211         ucmd->user_cmd.cmd_h = ucmd->h;
2212         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2213         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2214         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2215         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2216         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2217         strncpy(ucmd->user_cmd.sess.initiator_name,
2218                 tgt_dev->sess->initiator_name,
2219                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2220         ucmd->user_cmd.sess.initiator_name[
2221                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2222
2223         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %Lx, LUN %Lx, "
2224                 "threads_num %d, rd_only_flag %d, initiator %s)", ucmd, ucmd->h,
2225                 ucmd->user_cmd.sess.sess_h, ucmd->user_cmd.sess.lun,
2226                 ucmd->user_cmd.sess.threads_num, ucmd->user_cmd.sess.rd_only,
2227                 ucmd->user_cmd.sess.initiator_name);
2228
2229         ucmd->state = UCMD_STATE_ATTACH_SESS;
2230
2231         ucmd_get(ucmd);
2232
2233         dev_user_add_to_ready(ucmd);
2234
2235         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2236         if (rc > 0)
2237                 res = ucmd->result;
2238         else {
2239                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2240                 res = -EFAULT;
2241         }
2242
2243         sBUG_ON(irqs_disabled());
2244
2245         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2246         ucmd->cmpl = NULL;
2247         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2248
2249         ucmd_put(ucmd);
2250
2251 out:
2252         TRACE_EXIT_RES(res);
2253         return res;
2254
2255 out_nomem:
2256         res = -ENOMEM;
2257         goto out;
2258 }
2259
2260 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2261 {
2262         struct scst_user_dev *dev =
2263                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2264         struct scst_user_cmd *ucmd;
2265
2266         TRACE_ENTRY();
2267
2268         /*
2269          * We can't miss TM command due to memory shortage, because it might
2270          * lead to a memory leak in the user space handler.
2271          */
2272         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2273         if (ucmd == NULL)
2274                 goto out;
2275
2276         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %Lx)", ucmd,
2277                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2278
2279         ucmd->user_cmd.cmd_h = ucmd->h;
2280         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2281         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2282
2283         ucmd->state = UCMD_STATE_DETACH_SESS;
2284
2285         dev_user_add_to_ready(ucmd);
2286
2287 out:
2288         TRACE_EXIT();
2289         return;
2290 }
2291
2292 /* No locks are needed, but the activity must be suspended */
2293 static void dev_user_setup_functions(struct scst_user_dev *dev)
2294 {
2295         TRACE_ENTRY();
2296
2297         dev->devtype.parse = dev_user_parse;
2298         dev->devtype.dev_done = NULL;
2299
2300         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2301                 switch (dev->devtype.type) {
2302                 case TYPE_DISK:
2303                         dev->generic_parse = scst_sbc_generic_parse;
2304                         dev->devtype.dev_done = dev_user_disk_done;
2305                         break;
2306
2307                 case TYPE_TAPE:
2308                         dev->generic_parse = scst_tape_generic_parse;
2309                         dev->devtype.dev_done = dev_user_tape_done;
2310                         break;
2311
2312                 case TYPE_MOD:
2313                         dev->generic_parse = scst_modisk_generic_parse;
2314                         dev->devtype.dev_done = dev_user_disk_done;
2315                         break;
2316
2317                 case TYPE_ROM:
2318                         dev->generic_parse = scst_cdrom_generic_parse;
2319                         dev->devtype.dev_done = dev_user_disk_done;
2320                         break;
2321
2322                 case TYPE_MEDIUM_CHANGER:
2323                         dev->generic_parse = scst_changer_generic_parse;
2324                         break;
2325
2326                 case TYPE_PROCESSOR:
2327                         dev->generic_parse = scst_processor_generic_parse;
2328                         break;
2329
2330                 case TYPE_RAID:
2331                         dev->generic_parse = scst_raid_generic_parse;
2332                         break;
2333
2334                 default:
2335                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2336                                 "for it", dev->devtype.type);
2337                         dev->parse_type = SCST_USER_PARSE_CALL;
2338                         break;
2339                 }
2340         } else {
2341                 dev->generic_parse = NULL;
2342                 dev->devtype.dev_done = NULL;
2343         }
2344
2345         TRACE_EXIT();
2346         return;
2347 }
2348
2349 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2350 {
2351         char ver[sizeof(DEV_USER_VERSION)+1];
2352         int res;
2353
2354         res = copy_from_user(ver, (void __user *)(unsigned long)dev_desc->version_str,
2355                                 sizeof(ver));
2356         if (res < 0) {
2357                 PRINT_ERROR("%s", "Unable to get version string");
2358                 goto out;
2359         }
2360         ver[sizeof(ver)-1] = '\0';
2361
2362         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2363                 /* ->name already 0-terminated in dev_user_ioctl() */
2364                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2365                         dev_desc->name, ver);
2366                 res = -EINVAL;
2367                 goto out;
2368         }
2369
2370 out:
2371         return res;
2372 }
2373
2374 static int dev_user_register_dev(struct file *file,
2375         const struct scst_user_dev_desc *dev_desc)
2376 {
2377         int res = -ENOMEM, i;
2378         struct scst_user_dev *dev, *d;
2379         int block;
2380
2381         TRACE_ENTRY();
2382
2383         res = dev_user_check_version(dev_desc);
2384         if (res != 0)
2385                 goto out;
2386
2387         switch (dev_desc->type) {
2388         case TYPE_DISK:
2389         case TYPE_ROM:
2390         case TYPE_MOD:
2391                 if (dev_desc->block_size == 0) {
2392                         PRINT_ERROR("Wrong block size %d", dev_desc->block_size);
2393                         res = -EINVAL;
2394                         goto out;
2395                 }
2396                 block = scst_calc_block_shift(dev_desc->block_size);
2397                 if (block == -1) {
2398                         res = -EINVAL;
2399                         goto out;
2400                 }
2401                 break;
2402         default:
2403                 block = dev_desc->block_size;
2404                 break;
2405         }
2406
2407         if (!try_module_get(THIS_MODULE)) {
2408                 PRINT_ERROR("%s", "Fail to get module");
2409                 goto out;
2410         }
2411
2412         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2413         if (dev == NULL)
2414                 goto out_put;
2415
2416         init_rwsem(&dev->dev_rwsem);
2417         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2418         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2419         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2420         INIT_LIST_HEAD(&dev->ready_cmd_list);
2421         if (file->f_flags & O_NONBLOCK) {
2422                 TRACE_DBG("%s", "Non-blocking operations");
2423                 dev->blocking = 0;
2424         } else
2425                 dev->blocking = 1;
2426         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2427                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2428
2429         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2430         dev->name[sizeof(dev->name)-1] = '\0';
2431
2432         scst_init_mem_lim(&dev->udev_mem_lim);
2433
2434         /*
2435          * We don't use clustered pool, since it implies pages reordering,
2436          * which isn't possible with user space supplied buffers. Although
2437          * it's still possible to cluster pages by the tail of each other,
2438          * seems it doesn't worth the effort.
2439          */
2440         dev->pool = sgv_pool_create(dev->name, 0);
2441         if (dev->pool == NULL)
2442                 goto out_put;
2443         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2444                 dev_user_free_sg_entries);
2445
2446         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2447                 dev->name);
2448         dev->devtype.type = dev_desc->type;
2449         dev->devtype.threads_num = -1;
2450         dev->devtype.parse_atomic = 1;
2451         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2452         dev->devtype.dev_done_atomic = 1;
2453         dev->devtype.no_proc = 1;
2454         dev->devtype.attach = dev_user_attach;
2455         dev->devtype.detach = dev_user_detach;
2456         dev->devtype.attach_tgt = dev_user_attach_tgt;
2457         dev->devtype.detach_tgt = dev_user_detach_tgt;
2458         dev->devtype.exec = dev_user_exec;
2459         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2460         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2461
2462         init_completion(&dev->cleanup_cmpl);
2463         dev->block = block;
2464         dev->def_block = block;
2465
2466         res = __dev_user_set_opt(dev, &dev_desc->opt);
2467
2468         TRACE_MEM("dev %p, name %s", dev, dev->name);
2469
2470         spin_lock(&dev_list_lock);
2471
2472         list_for_each_entry(d, &dev_list, dev_list_entry) {
2473                 if (strcmp(d->name, dev->name) == 0) {
2474                         PRINT_ERROR("Device %s already exist",
2475                                 dev->name);
2476                         res = -EEXIST;
2477                         spin_unlock(&dev_list_lock);
2478                         goto out_free;
2479                 }
2480         }
2481
2482         list_add_tail(&dev->dev_list_entry, &dev_list);
2483
2484         spin_unlock(&dev_list_lock);
2485
2486         if (res != 0)
2487                 goto out_del_free;
2488
2489         res = scst_register_virtual_dev_driver(&dev->devtype);
2490         if (res < 0)
2491                 goto out_del_free;
2492
2493         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2494         if (dev->virt_id < 0) {
2495                 res = dev->virt_id;
2496                 goto out_unreg_handler;
2497         }
2498
2499         mutex_lock(&dev_priv_mutex);
2500         if (file->private_data != NULL) {
2501                 mutex_unlock(&dev_priv_mutex);
2502                 PRINT_ERROR("%s", "Device already registered");
2503                 res = -EINVAL;
2504                 goto out_unreg_drv;
2505         }
2506         file->private_data = dev;
2507         mutex_unlock(&dev_priv_mutex);
2508
2509 out:
2510         TRACE_EXIT_RES(res);
2511         return res;
2512
2513 out_unreg_drv:
2514         scst_unregister_virtual_device(dev->virt_id);
2515
2516 out_unreg_handler:
2517         scst_unregister_virtual_dev_driver(&dev->devtype);
2518
2519 out_del_free:
2520         spin_lock(&dev_list_lock);
2521         list_del(&dev->dev_list_entry);
2522         spin_unlock(&dev_list_lock);
2523
2524 out_free:
2525         sgv_pool_destroy(dev->pool);
2526         kfree(dev);
2527         goto out_put;
2528
2529 out_put:
2530         module_put(THIS_MODULE);
2531         goto out;
2532 }
2533
2534 static int __dev_user_set_opt(struct scst_user_dev *dev,
2535         const struct scst_user_opt *opt)
2536 {
2537         int res = 0;
2538
2539         TRACE_ENTRY();
2540
2541         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2542                 "partial_transfers_type %x, partial_len %d", opt->parse_type,
2543                 opt->on_free_cmd_type, opt->memory_reuse_type,
2544                 opt->partial_transfers_type, opt->partial_len);
2545
2546         if ((opt->parse_type > SCST_USER_MAX_PARSE_OPT) ||
2547             (opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT) ||
2548             (opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT) ||
2549             (opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT)) {
2550                 PRINT_ERROR("%s", "Invalid option");
2551                 res = -EINVAL;
2552                 goto out;
2553         }
2554
2555         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2556              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2557             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2558              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2559             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1)) {
2560                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x, "
2561                         "tas %x, has_own_order_mgmt %x)", opt->tst,
2562                         opt->queue_alg, opt->swp, opt->tas, opt->has_own_order_mgmt);
2563                 res = -EINVAL;
2564                 goto out;
2565         }
2566
2567         dev->parse_type = opt->parse_type;
2568         dev->on_free_cmd_type = opt->on_free_cmd_type;
2569         dev->memory_reuse_type = opt->memory_reuse_type;
2570         dev->partial_transfers_type = opt->partial_transfers_type;
2571         dev->partial_len = opt->partial_len;
2572
2573         dev->tst = opt->tst;
2574         dev->queue_alg = opt->queue_alg;
2575         dev->swp = opt->swp;
2576         dev->tas = opt->tas;
2577         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2578         if (dev->sdev != NULL) {
2579                 dev->sdev->tst = opt->tst;
2580                 dev->sdev->queue_alg = opt->queue_alg;
2581                 dev->sdev->swp = opt->swp;
2582                 dev->sdev->tas = opt->tas;
2583                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2584         }
2585
2586         dev_user_setup_functions(dev);
2587
2588 out:
2589         TRACE_EXIT_RES(res);
2590         return res;
2591 }
2592
2593 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2594 {
2595         int res = 0;
2596         struct scst_user_dev *dev;
2597
2598         TRACE_ENTRY();
2599
2600         mutex_lock(&dev_priv_mutex);
2601         dev = (struct scst_user_dev *)file->private_data;
2602         res = dev_user_check_reg(dev);
2603         if (res != 0) {
2604                 mutex_unlock(&dev_priv_mutex);
2605                 goto out;
2606         }
2607         down_write(&dev->dev_rwsem);
2608         mutex_unlock(&dev_priv_mutex);
2609
2610         res = scst_suspend_activity(true);
2611         if (res != 0)
2612                 goto out;
2613
2614         res = __dev_user_set_opt(dev, opt);
2615
2616         scst_resume_activity();
2617
2618         up_write(&dev->dev_rwsem);
2619
2620 out:
2621         TRACE_EXIT_RES(res);
2622         return res;
2623 }
2624
2625 static int dev_user_get_opt(struct file *file, void __user *arg)
2626 {
2627         int res = 0;
2628         struct scst_user_dev *dev;
2629         struct scst_user_opt opt;
2630
2631         TRACE_ENTRY();
2632
2633         mutex_lock(&dev_priv_mutex);
2634         dev = (struct scst_user_dev *)file->private_data;
2635         res = dev_user_check_reg(dev);
2636         if (res != 0) {
2637                 mutex_unlock(&dev_priv_mutex);
2638                 goto out;
2639         }
2640         down_read(&dev->dev_rwsem);
2641         mutex_unlock(&dev_priv_mutex);
2642
2643         opt.parse_type = dev->parse_type;
2644         opt.on_free_cmd_type = dev->on_free_cmd_type;
2645         opt.memory_reuse_type = dev->memory_reuse_type;
2646         opt.partial_transfers_type = dev->partial_transfers_type;
2647         opt.partial_len = dev->partial_len;
2648         opt.tst = dev->tst;
2649         opt.queue_alg = dev->queue_alg;
2650         opt.tas = dev->tas;
2651         opt.swp = dev->swp;
2652         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
2653
2654         TRACE_DBG("parse_type %x, on_free_cmd_type %x, memory_reuse_type %x, "
2655                 "partial_transfers_type %x, partial_len %d", opt.parse_type,
2656                 opt.on_free_cmd_type, opt.memory_reuse_type,
2657                 opt.partial_transfers_type, opt.partial_len);
2658
2659         res = copy_to_user(arg, &opt, sizeof(opt));
2660
2661         up_read(&dev->dev_rwsem);
2662 out:
2663         TRACE_EXIT_RES(res);
2664         return res;
2665 }
2666
2667 static int dev_usr_parse(struct scst_cmd *cmd)
2668 {
2669         sBUG();
2670         return SCST_CMD_STATE_DEFAULT;
2671 }
2672
2673 /* Needed only for /proc support */
2674 #define USR_TYPE {                      \
2675         .name =         DEV_USER_NAME,  \
2676         .type =         -1,             \
2677         .parse =        dev_usr_parse,  \
2678 }
2679
2680 static struct scst_dev_type dev_user_devtype = USR_TYPE;
2681
2682 static int dev_user_release(struct inode *inode, struct file *file)
2683 {
2684         int res = 0;
2685         struct scst_user_dev *dev;
2686
2687         TRACE_ENTRY();
2688
2689         mutex_lock(&dev_priv_mutex);
2690         dev = (struct scst_user_dev *)file->private_data;
2691         if (dev == NULL) {
2692                 mutex_unlock(&dev_priv_mutex);
2693                 goto out;
2694         }
2695         file->private_data = NULL;
2696
2697         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
2698
2699         spin_lock(&dev_list_lock);
2700         list_del(&dev->dev_list_entry);
2701         spin_unlock(&dev_list_lock);
2702
2703         mutex_unlock(&dev_priv_mutex);
2704
2705         down_write(&dev->dev_rwsem);
2706
2707         spin_lock(&cleanup_lock);
2708         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
2709         spin_unlock(&cleanup_lock);
2710
2711         wake_up(&cleanup_list_waitQ);
2712         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2713
2714         scst_unregister_virtual_device(dev->virt_id);
2715         scst_unregister_virtual_dev_driver(&dev->devtype);
2716
2717         sgv_pool_destroy(dev->pool);
2718
2719         TRACE_DBG("Unregistering finished (dev %p)", dev);
2720
2721         dev->cleanup_done = 1;
2722
2723         wake_up(&cleanup_list_waitQ);
2724         wake_up(&dev->cmd_lists.cmd_list_waitQ);
2725
2726         wait_for_completion(&dev->cleanup_cmpl);
2727
2728         up_write(&dev->dev_rwsem); /* to make the debug check happy */
2729
2730         TRACE_DBG("Releasing completed (dev %p)", dev);
2731
2732         kfree(dev);
2733
2734         module_put(THIS_MODULE);
2735
2736 out:
2737         TRACE_EXIT_RES(res);
2738         return res;
2739 }
2740
2741 static int dev_user_process_cleanup(struct scst_user_dev *dev)
2742 {
2743         struct scst_user_cmd *ucmd;
2744         int rc, res = 1;
2745
2746         TRACE_ENTRY();
2747
2748         dev->blocking = 0;
2749
2750         while (1) {
2751                 TRACE_DBG("Cleanuping dev %p", dev);
2752
2753                 dev_user_unjam_dev(dev);
2754
2755                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2756
2757                 rc = dev_user_get_next_cmd(dev, &ucmd);
2758                 if (rc == 0)
2759                         dev_user_unjam_cmd(ucmd, 1, NULL);
2760
2761                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2762
2763                 if (rc == -EAGAIN) {
2764                         if (dev->cleanup_done)
2765                                 break;
2766                         else {
2767                                 TRACE_DBG("No more commands (dev %p)", dev);
2768                                 goto out;
2769                         }
2770                 }
2771         }
2772
2773 #ifdef CONFIG_SCST_EXTRACHECKS
2774 {
2775         int i;
2776         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2777                 struct list_head *head = &dev->ucmd_hash[i];
2778                 struct scst_user_cmd *ucmd;
2779 again:
2780                 list_for_each_entry(ucmd, head, hash_list_entry) {
2781                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd,
2782                                 ucmd->state, atomic_read(&ucmd->ucmd_ref));
2783                         ucmd_put(ucmd);
2784                         goto again;
2785                 }
2786         }
2787 }
2788 #endif
2789
2790         TRACE_DBG("Cleanuping done (dev %p)", dev);
2791         complete_all(&dev->cleanup_cmpl);
2792         res = 0;
2793
2794 out:
2795         TRACE_EXIT_RES(res);
2796         return res;
2797 }
2798
2799 static inline int test_cleanup_list(void)
2800 {
2801         int res = !list_empty(&cleanup_list) ||
2802                   unlikely(kthread_should_stop());
2803         return res;
2804 }
2805
2806 static int dev_user_cleanup_thread(void *arg)
2807 {
2808         TRACE_ENTRY();
2809
2810         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
2811
2812         current->flags |= PF_NOFREEZE;
2813
2814         spin_lock(&cleanup_lock);
2815         while (!kthread_should_stop()) {
2816                 wait_queue_t wait;
2817                 init_waitqueue_entry(&wait, current);
2818
2819                 if (!test_cleanup_list()) {
2820                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
2821                         for (;;) {
2822                                 set_current_state(TASK_INTERRUPTIBLE);
2823                                 if (test_cleanup_list())
2824                                         break;
2825                                 spin_unlock(&cleanup_lock);
2826                                 schedule();
2827                                 spin_lock(&cleanup_lock);
2828                         }
2829                         set_current_state(TASK_RUNNING);
2830                         remove_wait_queue(&cleanup_list_waitQ, &wait);
2831                 }
2832
2833                 /*
2834                  * We have to poll devices, because commands can go from SCST
2835                  * core on cmd_list_waitQ and we have no practical way to
2836                  * detect them.
2837                  */
2838
2839                 while (1) {
2840                         struct scst_user_dev *dev;
2841                         LIST_HEAD(cl_devs);
2842
2843                         while (!list_empty(&cleanup_list)) {
2844                                 int rc;
2845
2846                                 dev = list_entry(cleanup_list.next,
2847                                         typeof(*dev), cleanup_list_entry);
2848                                 list_del(&dev->cleanup_list_entry);
2849
2850                                 spin_unlock(&cleanup_lock);
2851                                 rc = dev_user_process_cleanup(dev);
2852                                 spin_lock(&cleanup_lock);
2853
2854                                 if (rc != 0)
2855                                         list_add_tail(&dev->cleanup_list_entry,
2856                                                 &cl_devs);
2857                         }
2858
2859                         if (list_empty(&cl_devs))
2860                                 break;
2861
2862                         spin_unlock(&cleanup_lock);
2863                         msleep(100);
2864                         spin_lock(&cleanup_lock);
2865
2866                         while (!list_empty(&cl_devs)) {
2867                                 dev = list_entry(cl_devs.next, typeof(*dev),
2868                                         cleanup_list_entry);
2869                                 list_move_tail(&dev->cleanup_list_entry,
2870                                         &cleanup_list);
2871                         }
2872                 }
2873         }
2874         spin_unlock(&cleanup_lock);
2875
2876         /*
2877          * If kthread_should_stop() is true, we are guaranteed to be
2878          * on the module unload, so cleanup_list must be empty.
2879          */
2880         sBUG_ON(!list_empty(&cleanup_list));
2881
2882         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
2883
2884         TRACE_EXIT();
2885         return 0;
2886 }
2887
2888 static int __init init_scst_user(void)
2889 {
2890         int res = 0;
2891         struct class_device *class_member;
2892         struct max_get_reply {
2893                 union {
2894                         struct scst_user_get_cmd g;
2895                         struct scst_user_reply_cmd r;
2896                 };
2897         };
2898
2899         TRACE_ENTRY();
2900
2901 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
2902         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
2903                 "Consider change VMSPLIT option or use 64-bit "
2904                 "configuration instead. See README file for details.");
2905         res = -EINVAL;
2906         goto out;
2907 #endif
2908
2909         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
2910         if (user_cmd_cachep == NULL) {
2911                 res = -ENOMEM;
2912                 goto out;
2913         }
2914
2915         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
2916         if (user_get_cmd_cachep == NULL) {
2917                 res = -ENOMEM;
2918                 goto out_cache;
2919         }
2920
2921         dev_user_devtype.module = THIS_MODULE;
2922
2923         res = scst_register_virtual_dev_driver(&dev_user_devtype);
2924         if (res < 0)
2925                 goto out_cache1;
2926
2927         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
2928         if (res != 0)
2929                 goto out_unreg;
2930
2931         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
2932         if (IS_ERR(dev_user_sysfs_class)) {
2933                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
2934                         "space handler");
2935                 res = PTR_ERR(dev_user_sysfs_class);
2936                 goto out_proc;
2937         }
2938
2939         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
2940         if (res) {
2941                 PRINT_ERROR("Unable to get major %d for SCSI tapes", DEV_USER_MAJOR);
2942                 goto out_class;
2943         }
2944
2945         class_member = class_device_create(dev_user_sysfs_class, NULL,
2946                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
2947         if (IS_ERR(class_member)) {
2948                 res = PTR_ERR(class_member);
2949                 goto out_chrdev;
2950         }
2951
2952         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
2953                 "scst_usr_cleanupd");
2954         if (IS_ERR(cleanup_thread)) {
2955                 res = PTR_ERR(cleanup_thread);
2956                 PRINT_ERROR("kthread_create() failed: %d", res);
2957                 goto out_dev;
2958         }
2959
2960 out:
2961         TRACE_EXIT_RES(res);
2962         return res;
2963
2964 out_dev:
2965         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2966
2967 out_chrdev:
2968         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2969
2970 out_class:
2971         class_destroy(dev_user_sysfs_class);
2972
2973 out_proc:
2974         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
2975
2976 out_unreg:
2977         scst_unregister_dev_driver(&dev_user_devtype);
2978
2979 out_cache1:
2980         kmem_cache_destroy(user_get_cmd_cachep);
2981
2982 out_cache:
2983         kmem_cache_destroy(user_cmd_cachep);
2984         goto out;
2985 }
2986
2987 static void __exit exit_scst_user(void)
2988 {
2989         int rc;
2990
2991         TRACE_ENTRY();
2992
2993         rc = kthread_stop(cleanup_thread);
2994         if (rc < 0)
2995                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2996
2997         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
2998         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
2999         class_destroy(dev_user_sysfs_class);
3000
3001         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3002         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3003
3004         kmem_cache_destroy(user_get_cmd_cachep);
3005         kmem_cache_destroy(user_cmd_cachep);
3006
3007         TRACE_EXIT();
3008         return;
3009 }
3010
3011 module_init(init_scst_user);
3012 module_exit(exit_scst_user);
3013
3014 MODULE_AUTHOR("Vladislav Bolkhovitin");
3015 MODULE_LICENSE("GPL");
3016 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3017 MODULE_VERSION(SCST_VERSION_STRING);
3018 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);