scst/src/dev_handlers/scst_user.c

   1 /*
   2  *  scst_user.c
   3  *
   4  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   6  *
   7  *  SCSI virtual user space device handler
   8  *
   9  *  This program is free software; you can redistribute it and/or
  10  *  modify it under the terms of the GNU General Public License
  11  *  as published by the Free Software Foundation, version 2
  12  *  of the License.
  13  *
  14  *  This program is distributed in the hope that it will be useful,
  15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  *  GNU General Public License for more details.
  18  */
  19
  20 #include <linux/kthread.h>
  21 #include <linux/delay.h>
  22 #include <linux/poll.h>
  23 #include <linux/stddef.h>
  24
  25 #define LOG_PREFIX              DEV_USER_NAME
  26
  27 #include "scst.h"
  28 #include "scst_user.h"
  29 #include "scst_dev_handler.h"
  30
  31 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
  32 #warning "HIGHMEM kernel configurations are not supported by this module,\
  33  because nowadays it doesn't worth the effort. Consider changing\
  34  VMSPLIT option or use a 64-bit configuration instead. See README file\
  35  for details."
  36 #endif
  37
  38 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18))
  39 #define COMPLETION_INITIALIZER_ONSTACK(work) \
  40         ({ init_completion(&work); work; })
  41
  42 /*
  43  * Lockdep needs to run a non-constant initializer for on-stack
  44  * completions - so we use the _ONSTACK() variant for those that
  45  * are on the kernel stack:
  46  */
  47 #ifdef CONFIG_LOCKDEP
  48 # define DECLARE_COMPLETION_ONSTACK(work) \
  49         struct completion work = COMPLETION_INITIALIZER_ONSTACK(work)
  50 #else
  51 # define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work)
  52 #endif
  53
  54 #endif
  55
  56 #define DEV_USER_MAJOR                  237
  57 #define DEV_USER_CMD_HASH_ORDER         6
  58 #define DEV_USER_ATTACH_TIMEOUT         (5*HZ)
  59
  60 struct scst_user_dev {
  61         struct rw_semaphore dev_rwsem;
  62
  63         struct scst_cmd_lists cmd_lists;
  64
  65         /* Protected by cmd_lists.cmd_list_lock */
  66         struct list_head ready_cmd_list;
  67
  68         /* Protected by dev_rwsem or don't need any protection */
  69         unsigned int blocking:1;
  70         unsigned int cleanup_done:1;
  71         unsigned int tst:3;
  72         unsigned int queue_alg:4;
  73         unsigned int tas:1;
  74         unsigned int swp:1;
  75         unsigned int d_sense:1;
  76         unsigned int has_own_order_mgmt:1;
  77
  78         int (*generic_parse)(struct scst_cmd *cmd,
  79                 int (*get_block)(struct scst_cmd *cmd));
  80
  81         int block;
  82         int def_block;
  83
  84         struct scst_mem_lim udev_mem_lim;
  85         struct sgv_pool *pool;
  86         struct sgv_pool *pool_clust;
  87
  88         uint8_t parse_type;
  89         uint8_t on_free_cmd_type;
  90         uint8_t memory_reuse_type;
  91         uint8_t partial_transfers_type;
  92         uint32_t partial_len;
  93
  94         struct scst_dev_type devtype;
  95
  96         /* Both protected by cmd_lists.cmd_list_lock */
  97         unsigned int handle_counter;
  98         struct list_head ucmd_hash[1 << DEV_USER_CMD_HASH_ORDER];
  99
 100         struct scst_device *sdev;
 101
 102         int virt_id;
 103         struct list_head dev_list_entry;
 104         char name[SCST_MAX_NAME];
 105
 106         struct list_head cleanup_list_entry;
 107         struct completion cleanup_cmpl;
 108 };
 109
 110 /* Most fields are unprotected, since only one thread at time can access them */
 111 struct scst_user_cmd {
 112         struct scst_cmd *cmd;
 113         struct scst_user_dev *dev;
 114
 115         atomic_t ucmd_ref;
 116
 117         unsigned int buff_cached:1;
 118         unsigned int buf_dirty:1;
 119         unsigned int background_exec:1;
 120         unsigned int aborted:1;
 121
 122         struct scst_user_cmd *buf_ucmd;
 123
 124         int cur_data_page;
 125         int num_data_pages;
 126         int first_page_offset;
 127         unsigned long ubuff;
 128         struct page **data_pages;
 129         struct sgv_pool_obj *sgv;
 130
 131         /*
 132          * Special flags, which can be accessed asynchronously (hence "long").
 133          * Protected by cmd_lists.cmd_list_lock.
 134          */
 135         unsigned long sent_to_user:1;
 136         unsigned long jammed:1;
 137         unsigned long this_state_unjammed:1;
 138         unsigned long seen_by_user:1; /* here only as a small optimization */
 139
 140         unsigned int state;
 141
 142         struct list_head ready_cmd_list_entry;
 143
 144         unsigned int h;
 145         struct list_head hash_list_entry;
 146
 147         int user_cmd_payload_len;
 148         struct scst_user_get_cmd user_cmd;
 149
 150         /* cmpl used only by ATTACH_SESS, mcmd used only by TM */
 151         union {
 152                 struct completion *cmpl;
 153                 struct scst_mgmt_cmd *mcmd;
 154         };
 155         int result;
 156 };
 157
 158 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 159         gfp_t gfp_mask);
 160 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd);
 161
 162 static int dev_user_parse(struct scst_cmd *cmd);
 163 static int dev_user_exec(struct scst_cmd *cmd);
 164 static void dev_user_on_free_cmd(struct scst_cmd *cmd);
 165 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
 166         struct scst_tgt_dev *tgt_dev);
 167
 168 static int dev_user_disk_done(struct scst_cmd *cmd);
 169 static int dev_user_tape_done(struct scst_cmd *cmd);
 170
 171 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 172         gfp_t gfp_mask, void *priv);
 173 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 174                                      void *priv);
 175
 176 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
 177
 178 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
 179         unsigned long *flags);
 180
 181 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd);
 182 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
 183         int status);
 184 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
 185 static int dev_user_register_dev(struct file *file,
 186         const struct scst_user_dev_desc *dev_desc);
 187 static int dev_user_unregister_dev(struct file *file);
 188 static int dev_user_flush_cache(struct file *file);
 189 static int dev_user_capacity_changed(struct file *file);
 190 static int __dev_user_set_opt(struct scst_user_dev *dev,
 191         const struct scst_user_opt *opt);
 192 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
 193 static int dev_user_get_opt(struct file *file, void __user *arg);
 194
 195 static unsigned int dev_user_poll(struct file *filp, poll_table *wait);
 196 static long dev_user_ioctl(struct file *file, unsigned int cmd,
 197         unsigned long arg);
 198 static int dev_user_release(struct inode *inode, struct file *file);
 199
 200 /** Data **/
 201
 202 static struct kmem_cache *user_cmd_cachep;
 203 static struct kmem_cache *user_get_cmd_cachep;
 204
 205 static DEFINE_MUTEX(dev_priv_mutex);
 206
 207 static const struct file_operations dev_user_fops = {
 208         .poll           = dev_user_poll,
 209         .unlocked_ioctl = dev_user_ioctl,
 210 #ifdef CONFIG_COMPAT
 211         .compat_ioctl   = dev_user_ioctl,
 212 #endif
 213         .release        = dev_user_release,
 214 };
 215
 216 static struct class *dev_user_sysfs_class;
 217
 218 static DEFINE_SPINLOCK(dev_list_lock);
 219 static LIST_HEAD(dev_list);
 220
 221 static DEFINE_SPINLOCK(cleanup_lock);
 222 static LIST_HEAD(cleanup_list);
 223 static DECLARE_WAIT_QUEUE_HEAD(cleanup_list_waitQ);
 224 static struct task_struct *cleanup_thread;
 225
 226 /*
 227  * Skip this command if result is not 0. Must be called under
 228  * cmd_lists.cmd_list_lock and IRQ off.
 229  */
 230 static inline bool ucmd_get_check(struct scst_user_cmd *ucmd)
 231 {
 232         int r = atomic_inc_return(&ucmd->ucmd_ref);
 233         int res;
 234         if (unlikely(r == 1)) {
 235                 TRACE_DBG("ucmd %p is being destroyed", ucmd);
 236                 atomic_dec(&ucmd->ucmd_ref);
 237                 res = true;
 238                 /*
 239                  * Necessary code is serialized by cmd_list_lock in
 240                  * cmd_remove_hash()
 241                  */
 242         } else {
 243                 TRACE_DBG("ucmd %p, new ref_cnt %d", ucmd,
 244                         atomic_read(&ucmd->ucmd_ref));
 245                 res = false;
 246         }
 247         return res;
 248 }
 249
 250 static inline void __ucmd_get(struct scst_user_cmd *ucmd, bool barrier)
 251 {
 252         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 253         atomic_inc(&ucmd->ucmd_ref);
 254         if (barrier)
 255                 smp_mb__after_atomic_inc();
 256 }
 257
 258 static inline void ucmd_get_ordered(struct scst_user_cmd *ucmd)
 259 {
 260         __ucmd_get(ucmd, true);
 261 }
 262
 263 static inline void ucmd_get(struct scst_user_cmd *ucmd)
 264 {
 265         __ucmd_get(ucmd, false);
 266 }
 267
 268 /* Must not be called under cmd_list_lock!! */
 269 static inline void ucmd_put(struct scst_user_cmd *ucmd)
 270 {
 271         TRACE_DBG("ucmd %p, ucmd_ref %d", ucmd, atomic_read(&ucmd->ucmd_ref));
 272
 273         EXTRACHECKS_BUG_ON(atomic_read(&ucmd->ucmd_ref) == 0);
 274
 275         if (atomic_dec_and_test(&ucmd->ucmd_ref))
 276                 dev_user_free_ucmd(ucmd);
 277 }
 278
 279 static inline int calc_num_pg(unsigned long buf, int len)
 280 {
 281         len += buf & ~PAGE_MASK;
 282         return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
 283 }
 284
 285 static void __dev_user_not_reg(void)
 286 {
 287         PRINT_ERROR("%s", "Device not registered");
 288         return;
 289 }
 290
 291 static inline int dev_user_check_reg(struct scst_user_dev *dev)
 292 {
 293         if (dev == NULL) {
 294                 __dev_user_not_reg();
 295                 return -EINVAL;
 296         }
 297         return 0;
 298 }
 299
 300 static inline int scst_user_cmd_hashfn(int h)
 301 {
 302         return h & ((1 << DEV_USER_CMD_HASH_ORDER) - 1);
 303 }
 304
 305 static inline struct scst_user_cmd *__ucmd_find_hash(struct scst_user_dev *dev,
 306         unsigned int h)
 307 {
 308         struct list_head *head;
 309         struct scst_user_cmd *ucmd;
 310
 311         head = &dev->ucmd_hash[scst_user_cmd_hashfn(h)];
 312         list_for_each_entry(ucmd, head, hash_list_entry) {
 313                 if (ucmd->h == h) {
 314                         TRACE_DBG("Found ucmd %p", ucmd);
 315                         return ucmd;
 316                 }
 317         }
 318         return NULL;
 319 }
 320
 321 static void cmd_insert_hash(struct scst_user_cmd *ucmd)
 322 {
 323         struct list_head *head;
 324         struct scst_user_dev *dev = ucmd->dev;
 325         struct scst_user_cmd *u;
 326         unsigned long flags;
 327
 328         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
 329         do {
 330                 ucmd->h = dev->handle_counter++;
 331                 u = __ucmd_find_hash(dev, ucmd->h);
 332         } while (u != NULL);
 333         head = &dev->ucmd_hash[scst_user_cmd_hashfn(ucmd->h)];
 334         list_add_tail(&ucmd->hash_list_entry, head);
 335         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
 336
 337         TRACE_DBG("Inserted ucmd %p, h=%d (dev %s)", ucmd, ucmd->h, dev->name);
 338         return;
 339 }
 340
 341 static inline void cmd_remove_hash(struct scst_user_cmd *ucmd)
 342 {
 343         unsigned long flags;
 344
 345         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 346         list_del(&ucmd->hash_list_entry);
 347         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
 348
 349         TRACE_DBG("Removed ucmd %p, h=%d", ucmd, ucmd->h);
 350         return;
 351 }
 352
 353 static void dev_user_free_ucmd(struct scst_user_cmd *ucmd)
 354 {
 355         TRACE_ENTRY();
 356
 357         TRACE_MEM("Freeing ucmd %p", ucmd);
 358
 359         cmd_remove_hash(ucmd);
 360         EXTRACHECKS_BUG_ON(ucmd->cmd != NULL);
 361
 362         kmem_cache_free(user_cmd_cachep, ucmd);
 363
 364         TRACE_EXIT();
 365         return;
 366 }
 367
 368 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
 369         gfp_t gfp_mask, void *priv)
 370 {
 371         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 372         int offset = 0;
 373
 374         TRACE_ENTRY();
 375
 376         /* *sg supposed to be zeroed */
 377
 378         TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
 379                 ucmd->ubuff, ucmd->cur_data_page);
 380
 381         if (ucmd->cur_data_page == 0) {
 382                 TRACE_MEM("ucmd->first_page_offset %d",
 383                         ucmd->first_page_offset);
 384                 offset = ucmd->first_page_offset;
 385                 ucmd_get(ucmd);
 386         }
 387
 388         if (ucmd->cur_data_page >= ucmd->num_data_pages)
 389                 goto out;
 390
 391         sg_set_page(sg, ucmd->data_pages[ucmd->cur_data_page],
 392                 PAGE_SIZE - offset, offset);
 393         ucmd->cur_data_page++;
 394
 395         TRACE_MEM("page=%p, length=%d, offset=%d", sg_page(sg), sg->length,
 396                 sg->offset);
 397         TRACE_BUFFER("Page data", sg_virt(sg), sg->length);
 398
 399 out:
 400         TRACE_EXIT();
 401         return sg_page(sg);
 402 }
 403
 404 static void dev_user_on_cached_mem_free(struct scst_user_cmd *ucmd)
 405 {
 406         TRACE_ENTRY();
 407
 408         TRACE_MEM("Preparing ON_CACHED_MEM_FREE (ucmd %p, h %d, ubuff %lx)",
 409                 ucmd, ucmd->h, ucmd->ubuff);
 410
 411         ucmd->user_cmd_payload_len =
 412                 offsetof(struct scst_user_get_cmd, on_cached_mem_free) +
 413                 sizeof(ucmd->user_cmd.on_cached_mem_free);
 414         ucmd->user_cmd.cmd_h = ucmd->h;
 415         ucmd->user_cmd.subcode = SCST_USER_ON_CACHED_MEM_FREE;
 416         ucmd->user_cmd.on_cached_mem_free.pbuf = ucmd->ubuff;
 417
 418         ucmd->state = UCMD_STATE_ON_CACHE_FREEING;
 419
 420         dev_user_add_to_ready(ucmd);
 421
 422         TRACE_EXIT();
 423         return;
 424 }
 425
 426 static void dev_user_unmap_buf(struct scst_user_cmd *ucmd)
 427 {
 428         int i;
 429
 430         TRACE_ENTRY();
 431
 432         TRACE_MEM("Unmapping data pages (ucmd %p, ubuff %lx, num %d)", ucmd,
 433                 ucmd->ubuff, ucmd->num_data_pages);
 434
 435         for (i = 0; i < ucmd->num_data_pages; i++) {
 436                 struct page *page = ucmd->data_pages[i];
 437
 438                 if (ucmd->buf_dirty)
 439                         SetPageDirty(page);
 440
 441                 page_cache_release(page);
 442         }
 443
 444         kfree(ucmd->data_pages);
 445         ucmd->data_pages = NULL;
 446
 447         TRACE_EXIT();
 448         return;
 449 }
 450
 451 static void __dev_user_free_sg_entries(struct scst_user_cmd *ucmd)
 452 {
 453         TRACE_ENTRY();
 454
 455         sBUG_ON(ucmd->data_pages == NULL);
 456
 457         TRACE_MEM("Freeing data pages (ucmd=%p, ubuff=%lx, buff_cached=%d)",
 458                 ucmd, ucmd->ubuff, ucmd->buff_cached);
 459
 460         dev_user_unmap_buf(ucmd);
 461
 462         if (ucmd->buff_cached)
 463                 dev_user_on_cached_mem_free(ucmd);
 464         else
 465                 ucmd_put(ucmd);
 466
 467         TRACE_EXIT();
 468         return;
 469 }
 470
 471 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
 472         void *priv)
 473 {
 474         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 475
 476         TRACE_MEM("Freeing data pages (sg=%p, sg_count=%d, priv %p)", sg,
 477                 sg_count, ucmd);
 478
 479         __dev_user_free_sg_entries(ucmd);
 480
 481         return;
 482 }
 483
 484 static inline int is_buff_cached(struct scst_user_cmd *ucmd)
 485 {
 486         int mem_reuse_type = ucmd->dev->memory_reuse_type;
 487
 488         if ((mem_reuse_type == SCST_USER_MEM_REUSE_ALL) ||
 489             ((ucmd->cmd->data_direction == SCST_DATA_READ) &&
 490              (mem_reuse_type == SCST_USER_MEM_REUSE_READ)) ||
 491             ((ucmd->cmd->data_direction == SCST_DATA_WRITE) &&
 492              (mem_reuse_type == SCST_USER_MEM_REUSE_WRITE)))
 493                 return 1;
 494         else
 495                 return 0;
 496 }
 497
 498 static inline int is_need_offs_page(unsigned long buf, int len)
 499 {
 500         return ((buf & ~PAGE_MASK) != 0) &&
 501                 ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
 502 }
 503
 504 /*
 505  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
 506  * Unmaps the buffer, if needed in case of error
 507  */
 508 static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 509 {
 510         int res = 0;
 511         struct scst_cmd *cmd = ucmd->cmd;
 512         struct scst_user_dev *dev = ucmd->dev;
 513         struct sgv_pool *pool;
 514         gfp_t gfp_mask;
 515         int flags = 0;
 516         int bufflen, orig_bufflen;
 517         int last_len = 0;
 518         int out_sg_pages = 0;
 519
 520         TRACE_ENTRY();
 521
 522         gfp_mask = __GFP_NOWARN;
 523         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
 524
 525         if (cmd->data_direction != SCST_DATA_BIDI) {
 526                 orig_bufflen = cmd->bufflen;
 527                 pool = (struct sgv_pool *)cmd->tgt_dev->dh_priv;
 528         } else {
 529                 /* Make in_sg->offset 0 */
 530                 int len = cmd->bufflen + ucmd->first_page_offset;
 531                 out_sg_pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
 532                 orig_bufflen = (out_sg_pages << PAGE_SHIFT) + cmd->in_bufflen;
 533                 pool = dev->pool;
 534         }
 535         bufflen = orig_bufflen;
 536
 537         EXTRACHECKS_BUG_ON(bufflen == 0);
 538
 539         if (cached_buff) {
 540                 flags |= SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL;
 541                 if (ucmd->ubuff == 0)
 542                         flags |= SCST_POOL_NO_ALLOC_ON_CACHE_MISS;
 543         } else {
 544                 TRACE_MEM("%s", "Not cached buff");
 545                 flags |= SCST_POOL_ALLOC_NO_CACHED;
 546                 if (ucmd->ubuff == 0) {
 547                         res = 1;
 548                         goto out;
 549                 }
 550                 bufflen += ucmd->first_page_offset;
 551                 if (is_need_offs_page(ucmd->ubuff, orig_bufflen))
 552                         last_len = bufflen & ~PAGE_MASK;
 553                 else
 554                         last_len = orig_bufflen & ~PAGE_MASK;
 555                 if (last_len == 0)
 556                         last_len = PAGE_SIZE;
 557         }
 558         ucmd->buff_cached = cached_buff;
 559
 560         cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt,
 561                         &ucmd->sgv, &dev->udev_mem_lim, ucmd);
 562         if (cmd->sg != NULL) {
 563                 struct scst_user_cmd *buf_ucmd =
 564                         (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
 565
 566                 TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
 567                         "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
 568                         cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
 569
 570                 ucmd->ubuff = buf_ucmd->ubuff;
 571                 ucmd->buf_ucmd = buf_ucmd;
 572
 573                 EXTRACHECKS_BUG_ON((ucmd->data_pages != NULL) &&
 574                                    (ucmd != buf_ucmd));
 575
 576                 if (last_len != 0) {
 577                         cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
 578                         cmd->sg[cmd->sg_cnt-1].length += last_len;
 579                 }
 580
 581                 TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
 582                         "last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
 583                         cmd->sg[cmd->sg_cnt-1].length);
 584
 585                 if (cmd->data_direction == SCST_DATA_BIDI) {
 586                         cmd->in_sg = &cmd->sg[out_sg_pages];
 587                         cmd->in_sg_cnt = cmd->sg_cnt - out_sg_pages;
 588                         cmd->sg_cnt = out_sg_pages;
 589                         TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d",
 590                                 cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt);
 591                 }
 592
 593                 if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
 594                         static int ll;
 595                         if (ll < 10) {
 596                                 PRINT_INFO("Unable to complete command due to "
 597                                         "SG IO count limitation (requested %d, "
 598                                         "available %d, tgt lim %d)",
 599                                         cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt,
 600                                         cmd->tgt->sg_tablesize);
 601                                 ll++;
 602                         }
 603                         cmd->sg = NULL;
 604                         /* sgv will be freed in dev_user_free_sgv() */
 605                         res = -1;
 606                 }
 607         } else {
 608                 TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
 609                         "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,
 610                         ucmd->buff_cached, cmd->sg_cnt, ucmd->ubuff, ucmd->sgv);
 611                 if (unlikely(cmd->sg_cnt == 0)) {
 612                         TRACE_MEM("Refused allocation (ucmd %p)", ucmd);
 613                         sBUG_ON(ucmd->sgv != NULL);
 614                         res = -1;
 615                 } else {
 616                         switch (ucmd->state) {
 617                         case UCMD_STATE_BUF_ALLOCING:
 618                                 res = 1;
 619                                 break;
 620                         case UCMD_STATE_EXECING:
 621                                 res = -1;
 622                                 break;
 623                         default:
 624                                 sBUG();
 625                                 break;
 626                         }
 627                 }
 628         }
 629
 630 out:
 631         TRACE_EXIT_RES(res);
 632         return res;
 633 }
 634
 635 static int dev_user_alloc_space(struct scst_user_cmd *ucmd)
 636 {
 637         int rc, res = SCST_CMD_STATE_DEFAULT;
 638         struct scst_cmd *cmd = ucmd->cmd;
 639
 640         TRACE_ENTRY();
 641
 642         ucmd->state = UCMD_STATE_BUF_ALLOCING;
 643         cmd->dh_data_buf_alloced = 1;
 644
 645         rc = dev_user_alloc_sg(ucmd, is_buff_cached(ucmd));
 646         if (rc == 0)
 647                 goto out;
 648         else if (rc < 0) {
 649                 scst_set_busy(cmd);
 650                 res = scst_get_cmd_abnormal_done_state(cmd);
 651                 goto out;
 652         }
 653
 654         if ((cmd->data_direction != SCST_DATA_WRITE) &&
 655             !scst_is_cmd_local(cmd)) {
 656                 TRACE_DBG("Delayed alloc, ucmd %p", ucmd);
 657                 goto out;
 658         }
 659
 660         ucmd->user_cmd_payload_len =
 661                 offsetof(struct scst_user_get_cmd, alloc_cmd) +
 662                 sizeof(ucmd->user_cmd.alloc_cmd);
 663         ucmd->user_cmd.cmd_h = ucmd->h;
 664         ucmd->user_cmd.subcode = SCST_USER_ALLOC_MEM;
 665         ucmd->user_cmd.alloc_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 666         memcpy(ucmd->user_cmd.alloc_cmd.cdb, cmd->cdb, cmd->cdb_len);
 667         ucmd->user_cmd.alloc_cmd.cdb_len = cmd->cdb_len;
 668         ucmd->user_cmd.alloc_cmd.ext_cdb_len = cmd->ext_cdb_len;
 669         ucmd->user_cmd.alloc_cmd.alloc_len = ucmd->buff_cached ?
 670                 (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 671         ucmd->user_cmd.alloc_cmd.queue_type = cmd->queue_type;
 672         ucmd->user_cmd.alloc_cmd.data_direction = cmd->data_direction;
 673         ucmd->user_cmd.alloc_cmd.sn = cmd->tgt_sn;
 674
 675         dev_user_add_to_ready(ucmd);
 676
 677         res = SCST_CMD_STATE_STOP;
 678
 679 out:
 680         TRACE_EXIT_RES(res);
 681         return res;
 682 }
 683
 684 static struct scst_user_cmd *dev_user_alloc_ucmd(struct scst_user_dev *dev,
 685         gfp_t gfp_mask)
 686 {
 687         struct scst_user_cmd *ucmd = NULL;
 688
 689         TRACE_ENTRY();
 690
 691 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
 692         ucmd = kmem_cache_alloc(user_cmd_cachep, gfp_mask);
 693         if (ucmd != NULL)
 694                 memset(ucmd, 0, sizeof(*ucmd));
 695 #else
 696         ucmd = kmem_cache_zalloc(user_cmd_cachep, gfp_mask);
 697 #endif
 698         if (unlikely(ucmd == NULL)) {
 699                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate "
 700                         "user cmd (gfp_mask %x)", gfp_mask);
 701                 goto out;
 702         }
 703         ucmd->dev = dev;
 704         atomic_set(&ucmd->ucmd_ref, 1);
 705
 706         cmd_insert_hash(ucmd);
 707
 708         TRACE_MEM("ucmd %p allocated", ucmd);
 709
 710 out:
 711         TRACE_EXIT_HRES((unsigned long)ucmd);
 712         return ucmd;
 713 }
 714
 715 static int dev_user_get_block(struct scst_cmd *cmd)
 716 {
 717         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 718         /*
 719          * No need for locks here, since *_detach() can not be
 720          * called, when there are existing commands.
 721          */
 722         TRACE_EXIT_RES(dev->block);
 723         return dev->block;
 724 }
 725
 726 static int dev_user_parse(struct scst_cmd *cmd)
 727 {
 728         int rc, res = SCST_CMD_STATE_DEFAULT;
 729         struct scst_user_cmd *ucmd;
 730         int atomic = scst_cmd_atomic(cmd);
 731         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
 732         gfp_t gfp_mask = atomic ? GFP_ATOMIC : GFP_KERNEL;
 733
 734         TRACE_ENTRY();
 735
 736         if (cmd->dh_priv == NULL) {
 737                 ucmd = dev_user_alloc_ucmd(dev, gfp_mask);
 738                 if (unlikely(ucmd == NULL)) {
 739                         if (atomic) {
 740                                 res = SCST_CMD_STATE_NEED_THREAD_CTX;
 741                                 goto out;
 742                         } else {
 743                                 scst_set_busy(cmd);
 744                                 goto out_error;
 745                         }
 746                 }
 747                 ucmd->cmd = cmd;
 748                 cmd->dh_priv = ucmd;
 749         } else {
 750                 ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 751                 TRACE_DBG("Used ucmd %p, state %x", ucmd, ucmd->state);
 752         }
 753
 754         TRACE_DBG("ucmd %p, cmd %p, state %x", ucmd, cmd, ucmd->state);
 755
 756         if (ucmd->state != UCMD_STATE_NEW)
 757                 goto alloc;
 758
 759         switch (dev->parse_type) {
 760         case SCST_USER_PARSE_STANDARD:
 761                 TRACE_DBG("PARSE STANDARD: ucmd %p", ucmd);
 762                 rc = dev->generic_parse(cmd, dev_user_get_block);
 763                 if ((rc != 0) || (cmd->op_flags & SCST_INFO_INVALID))
 764                         goto out_invalid;
 765                 break;
 766
 767         case SCST_USER_PARSE_EXCEPTION:
 768                 TRACE_DBG("PARSE EXCEPTION: ucmd %p", ucmd);
 769                 rc = dev->generic_parse(cmd, dev_user_get_block);
 770                 if ((rc == 0) && (!(cmd->op_flags & SCST_INFO_INVALID)))
 771                         break;
 772                 else if (rc == SCST_CMD_STATE_NEED_THREAD_CTX) {
 773                         TRACE_MEM("Restarting PARSE to thread context "
 774                                 "(ucmd %p)", ucmd);
 775                         res = SCST_CMD_STATE_NEED_THREAD_CTX;
 776                         goto out;
 777                 }
 778                 /* else go through */
 779
 780         case SCST_USER_PARSE_CALL:
 781                 TRACE_DBG("Preparing PARSE for user space (ucmd=%p, h=%d, "
 782                         "bufflen %d)", ucmd, ucmd->h, cmd->bufflen);
 783                 ucmd->user_cmd_payload_len =
 784                         offsetof(struct scst_user_get_cmd, parse_cmd) +
 785                         sizeof(ucmd->user_cmd.parse_cmd);
 786                 ucmd->user_cmd.cmd_h = ucmd->h;
 787                 ucmd->user_cmd.subcode = SCST_USER_PARSE;
 788                 ucmd->user_cmd.parse_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 789                 memcpy(ucmd->user_cmd.parse_cmd.cdb, cmd->cdb, cmd->cdb_len);
 790                 ucmd->user_cmd.parse_cmd.cdb_len = cmd->cdb_len;
 791                 ucmd->user_cmd.parse_cmd.ext_cdb_len = cmd->ext_cdb_len;
 792                 ucmd->user_cmd.parse_cmd.timeout = cmd->timeout / HZ;
 793                 ucmd->user_cmd.parse_cmd.bufflen = cmd->bufflen;
 794                 ucmd->user_cmd.parse_cmd.in_bufflen = cmd->in_bufflen;
 795                 ucmd->user_cmd.parse_cmd.queue_type = cmd->queue_type;
 796                 ucmd->user_cmd.parse_cmd.data_direction = cmd->data_direction;
 797                 ucmd->user_cmd.parse_cmd.expected_values_set =
 798                                         cmd->expected_values_set;
 799                 ucmd->user_cmd.parse_cmd.expected_data_direction =
 800                                         cmd->expected_data_direction;
 801                 ucmd->user_cmd.parse_cmd.expected_transfer_len =
 802                                         cmd->expected_transfer_len;
 803                 ucmd->user_cmd.parse_cmd.sn = cmd->tgt_sn;
 804                 ucmd->state = UCMD_STATE_PARSING;
 805                 dev_user_add_to_ready(ucmd);
 806                 res = SCST_CMD_STATE_STOP;
 807                 goto out;
 808
 809         default:
 810                 sBUG();
 811                 goto out;
 812         }
 813
 814 alloc:
 815         if (cmd->bufflen == 0) {
 816                 /*
 817                  * According to SPC bufflen 0 for data transfer commands isn't
 818                  * an error, so we need to fix the transfer direction.
 819                  */
 820                 cmd->data_direction = SCST_DATA_NONE;
 821         }
 822
 823         if (cmd->data_direction != SCST_DATA_NONE)
 824                 res = dev_user_alloc_space(ucmd);
 825
 826 out:
 827         TRACE_EXIT_RES(res);
 828         return res;
 829
 830 out_invalid:
 831         PRINT_ERROR("PARSE failed (ucmd %p, rc %d, invalid %d)", ucmd, rc,
 832                 cmd->op_flags & SCST_INFO_INVALID);
 833         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 834
 835 out_error:
 836         res = scst_get_cmd_abnormal_done_state(cmd);
 837         goto out;
 838 }
 839
 840 static void dev_user_flush_dcache(struct scst_user_cmd *ucmd)
 841 {
 842         struct scst_user_cmd *buf_ucmd = ucmd->buf_ucmd;
 843         unsigned long start = buf_ucmd->ubuff;
 844         int i, bufflen = ucmd->cmd->bufflen;
 845
 846         TRACE_ENTRY();
 847
 848         if (start == 0)
 849                 goto out;
 850
 851         /*
 852          * Possibly, flushing of all the pages from ucmd->cmd->sg can be
 853          * faster, since it should be cache hot, while ucmd->buf_ucmd and
 854          * buf_ucmd->data_pages are cache cold. But, from other side,
 855          * sizeof(buf_ucmd->data_pages[0]) is considerably smaller, than
 856          * sizeof(ucmd->cmd->sg[0]), so on big buffers going over
 857          * data_pages array can lead to less cache misses. So, real numbers are
 858          * needed. ToDo.
 859          */
 860
 861         for (i = 0; (bufflen > 0) && (i < buf_ucmd->num_data_pages); i++) {
 862                 struct page *page;
 863                 page = buf_ucmd->data_pages[i];
 864 #ifdef ARCH_HAS_FLUSH_ANON_PAGE
 865                 struct vm_area_struct *vma = find_vma(current->mm, start);
 866                 if (vma != NULL)
 867                         flush_anon_page(vma, page, start);
 868 #endif
 869                 flush_dcache_page(page);
 870                 start += PAGE_SIZE;
 871                 bufflen -= PAGE_SIZE;
 872         }
 873
 874 out:
 875         TRACE_EXIT();
 876         return;
 877 }
 878
 879 static int dev_user_exec(struct scst_cmd *cmd)
 880 {
 881         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 882         int res = SCST_EXEC_COMPLETED;
 883
 884         TRACE_ENTRY();
 885
 886 #if 0 /* We set exec_atomic in 0 to let SCST core know that we need a thread
 887        * context to complete the necessary actions, but all we are going to
 888        * do in this function is, in fact, atomic, so let's skip this check.
 889        */
 890         if (scst_cmd_atomic(cmd)) {
 891                 TRACE_DBG("%s", "User exec() can not be called in atomic "
 892                         "context, rescheduling to the thread");
 893                 res = SCST_EXEC_NEED_THREAD;
 894                 goto out;
 895         }
 896 #endif
 897
 898         TRACE_DBG("Preparing EXEC for user space (ucmd=%p, h=%d, "
 899                 "bufflen %d, data_len %d, ubuff %lx)", ucmd, ucmd->h,
 900                 cmd->bufflen, cmd->data_len, ucmd->ubuff);
 901
 902         if (cmd->data_direction == SCST_DATA_WRITE)
 903                 dev_user_flush_dcache(ucmd);
 904
 905         BUILD_BUG_ON(sizeof(ucmd->user_cmd.exec_cmd.cdb) != sizeof(cmd->cdb));
 906
 907         ucmd->user_cmd_payload_len =
 908                 offsetof(struct scst_user_get_cmd, exec_cmd) +
 909                 sizeof(ucmd->user_cmd.exec_cmd);
 910         ucmd->user_cmd.cmd_h = ucmd->h;
 911         ucmd->user_cmd.subcode = SCST_USER_EXEC;
 912         ucmd->user_cmd.exec_cmd.sess_h = (unsigned long)cmd->tgt_dev;
 913         memcpy(ucmd->user_cmd.exec_cmd.cdb, cmd->cdb, cmd->cdb_len);
 914         ucmd->user_cmd.exec_cmd.cdb_len = cmd->cdb_len;
 915         ucmd->user_cmd.exec_cmd.ext_cdb_len = cmd->ext_cdb_len;
 916         ucmd->user_cmd.exec_cmd.bufflen = cmd->bufflen;
 917         ucmd->user_cmd.exec_cmd.data_len = cmd->data_len;
 918         ucmd->user_cmd.exec_cmd.pbuf = ucmd->ubuff;
 919         if ((ucmd->ubuff == 0) && (cmd->data_direction != SCST_DATA_NONE)) {
 920                 ucmd->user_cmd.exec_cmd.alloc_len = ucmd->buff_cached ?
 921                         (cmd->sg_cnt << PAGE_SHIFT) : cmd->bufflen;
 922         }
 923         ucmd->user_cmd.exec_cmd.queue_type = cmd->queue_type;
 924         ucmd->user_cmd.exec_cmd.data_direction = cmd->data_direction;
 925         ucmd->user_cmd.exec_cmd.partial = 0;
 926         ucmd->user_cmd.exec_cmd.timeout = cmd->timeout / HZ;
 927         ucmd->user_cmd.exec_cmd.p_in_buf = ucmd->ubuff +
 928                                                 (cmd->sg_cnt << PAGE_SHIFT);
 929         ucmd->user_cmd.exec_cmd.in_bufflen = cmd->in_bufflen;
 930         ucmd->user_cmd.exec_cmd.sn = cmd->tgt_sn;
 931
 932         ucmd->state = UCMD_STATE_EXECING;
 933
 934         dev_user_add_to_ready(ucmd);
 935
 936         TRACE_EXIT_RES(res);
 937         return res;
 938 }
 939
 940 static void dev_user_free_sgv(struct scst_user_cmd *ucmd)
 941 {
 942         if (ucmd->sgv != NULL) {
 943                 sgv_pool_free(ucmd->sgv, &ucmd->dev->udev_mem_lim);
 944                 ucmd->sgv = NULL;
 945         } else if (ucmd->data_pages != NULL) {
 946                 /* We mapped pages, but for some reason didn't allocate them */
 947                 ucmd_get(ucmd);
 948                 __dev_user_free_sg_entries(ucmd);
 949         }
 950         return;
 951 }
 952
 953 static void dev_user_on_free_cmd(struct scst_cmd *cmd)
 954 {
 955         struct scst_user_cmd *ucmd = (struct scst_user_cmd *)cmd->dh_priv;
 956
 957         TRACE_ENTRY();
 958
 959         if (unlikely(ucmd == NULL))
 960                 goto out;
 961
 962         TRACE_MEM("ucmd %p, cmd %p, buff_cached %d, ubuff %lx", ucmd, ucmd->cmd,
 963                 ucmd->buff_cached, ucmd->ubuff);
 964
 965         ucmd->cmd = NULL;
 966         if (cmd->data_direction == SCST_DATA_WRITE && ucmd->buf_ucmd != NULL)
 967                 ucmd->buf_ucmd->buf_dirty = 1;
 968
 969         if (ucmd->dev->on_free_cmd_type == SCST_USER_ON_FREE_CMD_IGNORE) {
 970                 ucmd->state = UCMD_STATE_ON_FREE_SKIPPED;
 971                 /* The state assignment must be before freeing sgv! */
 972                 goto out_reply;
 973         }
 974
 975         if (unlikely(!ucmd->seen_by_user)) {
 976                 TRACE_MGMT_DBG("Not seen by user ucmd %p", ucmd);
 977                 goto out_reply;
 978         }
 979
 980         ucmd->user_cmd_payload_len =
 981                 offsetof(struct scst_user_get_cmd, on_free_cmd) +
 982                 sizeof(ucmd->user_cmd.on_free_cmd);
 983         ucmd->user_cmd.cmd_h = ucmd->h;
 984         ucmd->user_cmd.subcode = SCST_USER_ON_FREE_CMD;
 985         ucmd->user_cmd.on_free_cmd.pbuf = ucmd->ubuff;
 986         ucmd->user_cmd.on_free_cmd.resp_data_len = cmd->resp_data_len;
 987         ucmd->user_cmd.on_free_cmd.buffer_cached = ucmd->buff_cached;
 988         ucmd->user_cmd.on_free_cmd.aborted = ucmd->aborted;
 989         ucmd->user_cmd.on_free_cmd.status = cmd->status;
 990         ucmd->user_cmd.on_free_cmd.delivery_status = cmd->delivery_status;
 991
 992         ucmd->state = UCMD_STATE_ON_FREEING;
 993
 994         dev_user_add_to_ready(ucmd);
 995
 996 out:
 997         TRACE_EXIT();
 998         return;
 999
1000 out_reply:
1001         dev_user_process_reply_on_free(ucmd);
1002         goto out;
1003 }
1004
1005 static void dev_user_set_block(struct scst_cmd *cmd, int block)
1006 {
1007         struct scst_user_dev *dev = (struct scst_user_dev *)cmd->dev->dh_priv;
1008         /*
1009          * No need for locks here, since *_detach() can not be
1010          * called, when there are existing commands.
1011          */
1012         TRACE_DBG("dev %p, new block %d", dev, block);
1013         if (block != 0)
1014                 dev->block = block;
1015         else
1016                 dev->block = dev->def_block;
1017         return;
1018 }
1019
1020 static int dev_user_disk_done(struct scst_cmd *cmd)
1021 {
1022         int res = SCST_CMD_STATE_DEFAULT;
1023
1024         TRACE_ENTRY();
1025
1026         res = scst_block_generic_dev_done(cmd, dev_user_set_block);
1027
1028         TRACE_EXIT_RES(res);
1029         return res;
1030 }
1031
1032 static int dev_user_tape_done(struct scst_cmd *cmd)
1033 {
1034         int res = SCST_CMD_STATE_DEFAULT;
1035
1036         TRACE_ENTRY();
1037
1038         res = scst_tape_generic_dev_done(cmd, dev_user_set_block);
1039
1040         TRACE_EXIT_RES(res);
1041         return res;
1042 }
1043
1044 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd)
1045 {
1046         struct scst_user_dev *dev = ucmd->dev;
1047         unsigned long flags;
1048         int do_wake;
1049
1050         TRACE_ENTRY();
1051
1052         do_wake = (in_interrupt() ||
1053                    (ucmd->state == UCMD_STATE_ON_CACHE_FREEING));
1054         if (ucmd->cmd)
1055                 do_wake |= ucmd->cmd->preprocessing_only;
1056
1057         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
1058
1059         ucmd->this_state_unjammed = 0;
1060
1061         if ((ucmd->state == UCMD_STATE_PARSING) ||
1062             (ucmd->state == UCMD_STATE_BUF_ALLOCING)) {
1063                 /*
1064                  * If we don't put such commands in the queue head, then under
1065                  * high load we might delay threads, waiting for memory
1066                  * allocations, for too long and start loosing NOPs, which
1067                  * would lead to consider us by remote initiators as
1068                  * unresponsive and stuck => broken connections, etc. If none
1069                  * of our commands completed in NOP timeout to allow the head
1070                  * commands to go, then we are really overloaded and/or stuck.
1071                  */
1072                 TRACE_DBG("Adding ucmd %p (state %d) to head of ready "
1073                         "cmd list", ucmd, ucmd->state);
1074                 list_add(&ucmd->ready_cmd_list_entry,
1075                         &dev->ready_cmd_list);
1076                 do_wake = 1;
1077         } else if (unlikely(ucmd->state == UCMD_STATE_TM_EXECING) ||
1078                    unlikely(ucmd->state == UCMD_STATE_ATTACH_SESS) ||
1079                    unlikely(ucmd->state == UCMD_STATE_DETACH_SESS)) {
1080                 TRACE_MGMT_DBG("Adding mgmt ucmd %p (state %d) to head of "
1081                         "ready cmd list", ucmd, ucmd->state);
1082                 list_add(&ucmd->ready_cmd_list_entry,
1083                         &dev->ready_cmd_list);
1084                 do_wake = 1;
1085         } else if ((ucmd->cmd != NULL) &&
1086            unlikely((ucmd->cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))) {
1087                 TRACE_DBG("Adding HQ ucmd %p to head of ready cmd list", ucmd);
1088                 list_add(&ucmd->ready_cmd_list_entry, &dev->ready_cmd_list);
1089         } else {
1090                 TRACE_DBG("Adding ucmd %p to ready cmd list", ucmd);
1091                 list_add_tail(&ucmd->ready_cmd_list_entry,
1092                               &dev->ready_cmd_list);
1093         }
1094
1095         if (do_wake) {
1096                 TRACE_DBG("Waking up dev %p", dev);
1097                 wake_up(&dev->cmd_lists.cmd_list_waitQ);
1098         }
1099
1100         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
1101
1102         TRACE_EXIT();
1103         return;
1104 }
1105
1106 static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
1107         int num_pg)
1108 {
1109         int res = 0, rc;
1110         int i;
1111         struct task_struct *tsk = current;
1112
1113         TRACE_ENTRY();
1114
1115         if (unlikely(ubuff == 0))
1116                 goto out_nomem;
1117
1118         sBUG_ON(ucmd->data_pages != NULL);
1119
1120         ucmd->num_data_pages = num_pg;
1121
1122         ucmd->data_pages =
1123                 kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
1124                           GFP_KERNEL);
1125         if (ucmd->data_pages == NULL) {
1126                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
1127                         "(num_data_pages=%d)", ucmd->num_data_pages);
1128                 res = -ENOMEM;
1129                 goto out_nomem;
1130         }
1131
1132         TRACE_MEM("Mapping buffer (ucmd %p, ubuff %lx, ucmd->num_data_pages %d,"
1133                 " first_page_offset %d, len %d)", ucmd, ubuff,
1134                 ucmd->num_data_pages, (int)(ubuff & ~PAGE_MASK),
1135                 ucmd->cmd->bufflen);
1136
1137         down_read(&tsk->mm->mmap_sem);
1138         rc = get_user_pages(tsk, tsk->mm, ubuff, ucmd->num_data_pages,
1139                 1/*writable*/, 0/*don't force*/, ucmd->data_pages, NULL);
1140         up_read(&tsk->mm->mmap_sem);
1141
1142         /* get_user_pages() flushes dcache */
1143
1144         if (rc < ucmd->num_data_pages)
1145                 goto out_unmap;
1146
1147         ucmd->ubuff = ubuff;
1148         ucmd->first_page_offset = (ubuff & ~PAGE_MASK);
1149
1150 out:
1151         TRACE_EXIT_RES(res);
1152         return res;
1153
1154 out_nomem:
1155         scst_set_busy(ucmd->cmd);
1156         /* go through */
1157
1158 out_err:
1159         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1160         goto out;
1161
1162 out_unmap:
1163         PRINT_ERROR("Failed to get %d user pages (rc %d)",
1164                 ucmd->num_data_pages, rc);
1165         if (rc > 0) {
1166                 for (i = 0; i < rc; i++)
1167                         page_cache_release(ucmd->data_pages[i]);
1168         }
1169         kfree(ucmd->data_pages);
1170         ucmd->data_pages = NULL;
1171         res = -EFAULT;
1172         scst_set_cmd_error(ucmd->cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1173         goto out_err;
1174 }
1175
1176 static int dev_user_process_reply_alloc(struct scst_user_cmd *ucmd,
1177         struct scst_user_reply_cmd *reply)
1178 {
1179         int res = 0;
1180         struct scst_cmd *cmd = ucmd->cmd;
1181
1182         TRACE_ENTRY();
1183
1184         TRACE_DBG("ucmd %p, pbuf %llx", ucmd, reply->alloc_reply.pbuf);
1185
1186         if (likely(reply->alloc_reply.pbuf != 0)) {
1187                 int pages;
1188                 if (ucmd->buff_cached) {
1189                         if (unlikely((reply->alloc_reply.pbuf & ~PAGE_MASK) != 0)) {
1190                                 PRINT_ERROR("Supplied pbuf %llx isn't "
1191                                         "page aligned",
1192                                         reply->alloc_reply.pbuf);
1193                                 goto out_hwerr;
1194                         }
1195                         pages = cmd->sg_cnt;
1196                 } else
1197                         pages = calc_num_pg(reply->alloc_reply.pbuf,
1198                                             cmd->bufflen);
1199                 res = dev_user_map_buf(ucmd, reply->alloc_reply.pbuf, pages);
1200         } else {
1201                 scst_set_busy(ucmd->cmd);
1202                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
1203         }
1204
1205 out_process:
1206         scst_process_active_cmd(cmd, false);
1207
1208         TRACE_EXIT_RES(res);
1209         return res;
1210
1211 out_hwerr:
1212         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1213         scst_set_cmd_abnormal_done_state(ucmd->cmd);
1214         res = -EINVAL;
1215         goto out_process;
1216 }
1217
1218 static int dev_user_process_reply_parse(struct scst_user_cmd *ucmd,
1219         struct scst_user_reply_cmd *reply)
1220 {
1221         int res = 0;
1222         struct scst_user_scsi_cmd_reply_parse *preply =
1223                 &reply->parse_reply;
1224         struct scst_cmd *cmd = ucmd->cmd;
1225
1226         TRACE_ENTRY();
1227
1228         if (unlikely(preply->queue_type > SCST_CMD_QUEUE_ACA))
1229                 goto out_inval;
1230
1231         if (unlikely((preply->data_direction != SCST_DATA_WRITE) &&
1232                      (preply->data_direction != SCST_DATA_READ) &&
1233                      (preply->data_direction != SCST_DATA_NONE)))
1234                 goto out_inval;
1235
1236         if (unlikely((preply->data_direction != SCST_DATA_NONE) &&
1237                      (preply->bufflen == 0)))
1238                 goto out_inval;
1239
1240         if (unlikely((preply->bufflen < 0) || (preply->data_len < 0)))
1241                 goto out_inval;
1242
1243         TRACE_DBG("ucmd %p, queue_type %x, data_direction, %x, bufflen %d, "
1244                 "data_len %d, pbuf %llx", ucmd, preply->queue_type,
1245                 preply->data_direction, preply->bufflen, preply->data_len,
1246                 reply->alloc_reply.pbuf);
1247
1248         cmd->queue_type = preply->queue_type;
1249         cmd->data_direction = preply->data_direction;
1250         cmd->bufflen = preply->bufflen;
1251         cmd->data_len = preply->data_len;
1252
1253 out_process:
1254         scst_process_active_cmd(cmd, false);
1255
1256         TRACE_EXIT_RES(res);
1257         return res;
1258
1259 out_inval:
1260         PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %x, cmd %p)",
1261                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1262         PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply));
1263         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1264         scst_set_cmd_abnormal_done_state(cmd);
1265         res = -EINVAL;
1266         goto out_process;
1267 }
1268
1269 static int dev_user_process_reply_on_free(struct scst_user_cmd *ucmd)
1270 {
1271         int res = 0;
1272
1273         TRACE_ENTRY();
1274
1275         TRACE_DBG("ON FREE ucmd %p", ucmd);
1276
1277         dev_user_free_sgv(ucmd);
1278         ucmd_put(ucmd);
1279
1280         TRACE_EXIT_RES(res);
1281         return res;
1282 }
1283
1284 static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd)
1285 {
1286         int res = 0;
1287
1288         TRACE_ENTRY();
1289
1290         TRACE_DBG("ON CACHE FREE ucmd %p", ucmd);
1291
1292         ucmd_put(ucmd);
1293
1294         TRACE_EXIT_RES(res);
1295         return res;
1296 }
1297
1298 static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd,
1299         struct scst_user_reply_cmd *reply)
1300 {
1301         int res = 0;
1302         struct scst_user_scsi_cmd_reply_exec *ereply =
1303                 &reply->exec_reply;
1304         struct scst_cmd *cmd = ucmd->cmd;
1305
1306         TRACE_ENTRY();
1307
1308         if (ereply->reply_type == SCST_EXEC_REPLY_COMPLETED) {
1309                 if (ucmd->background_exec) {
1310                         TRACE_DBG("Background ucmd %p finished", ucmd);
1311                         ucmd_put(ucmd);
1312                         goto out;
1313                 }
1314                 if (unlikely(ereply->resp_data_len > cmd->bufflen))
1315                         goto out_inval;
1316                 if (unlikely((cmd->data_direction != SCST_DATA_READ) &&
1317                              (ereply->resp_data_len != 0)))
1318                         goto out_inval;
1319         } else if (ereply->reply_type == SCST_EXEC_REPLY_BACKGROUND) {
1320                 if (unlikely(ucmd->background_exec))
1321                         goto out_inval;
1322                 if (unlikely((cmd->data_direction == SCST_DATA_READ) ||
1323                              (cmd->resp_data_len != 0)))
1324                         goto out_inval;
1325                 /*
1326                  * background_exec assignment must be after ucmd get.
1327                  * Otherwise, due to reorder, in dev_user_process_reply()
1328                  * it is possible that ucmd is destroyed before it "got" here.
1329                  */
1330                 ucmd_get_ordered(ucmd);
1331                 ucmd->background_exec = 1;
1332                 TRACE_DBG("Background ucmd %p", ucmd);
1333                 goto out_compl;
1334         } else
1335                 goto out_inval;
1336
1337         TRACE_DBG("ucmd %p, status %d, resp_data_len %d", ucmd,
1338                 ereply->status, ereply->resp_data_len);
1339
1340          if (ereply->resp_data_len != 0) {
1341                 if (ucmd->ubuff == 0) {
1342                         int pages, rc;
1343                         if (unlikely(ereply->pbuf == 0))
1344                                 goto out_busy;
1345                         if (ucmd->buff_cached) {
1346                                 if (unlikely((ereply->pbuf & ~PAGE_MASK) != 0)) {
1347                                         PRINT_ERROR("Supplied pbuf %llx isn't "
1348                                                 "page aligned", ereply->pbuf);
1349                                         goto out_hwerr;
1350                                 }
1351                                 pages = cmd->sg_cnt;
1352                         } else
1353                                 pages = calc_num_pg(ereply->pbuf, cmd->bufflen);
1354                         rc = dev_user_map_buf(ucmd, ereply->pbuf, pages);
1355                         if ((rc != 0) || (ucmd->ubuff == 0))
1356                                 goto out_compl;
1357
1358                         rc = dev_user_alloc_sg(ucmd, ucmd->buff_cached);
1359                         if (unlikely(rc != 0))
1360                                 goto out_busy;
1361                 } else
1362                         dev_user_flush_dcache(ucmd);
1363                 cmd->may_need_dma_sync = 1;
1364                 scst_set_resp_data_len(cmd, ereply->resp_data_len);
1365         } else if (cmd->resp_data_len != ereply->resp_data_len) {
1366                 if (ucmd->ubuff == 0)
1367                         cmd->resp_data_len = ereply->resp_data_len;
1368                 else
1369                         scst_set_resp_data_len(cmd, ereply->resp_data_len);
1370         }
1371
1372         cmd->status = ereply->status;
1373         if (ereply->sense_len != 0) {
1374                 res = scst_alloc_sense(cmd, 0);
1375                 if (res != 0)
1376                         goto out_compl;
1377                 res = copy_from_user(cmd->sense,
1378                         (void __user *)(unsigned long)ereply->psense_buffer,
1379                         min((int)cmd->sense_bufflen, (int)ereply->sense_len));
1380                 if (res < 0) {
1381                         PRINT_ERROR("%s", "Unable to get sense data");
1382                         goto out_hwerr_res_set;
1383                 }
1384         }
1385
1386 out_compl:
1387         cmd->completed = 1;
1388         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT);
1389         /* !! At this point cmd can be already freed !! */
1390
1391 out:
1392         TRACE_EXIT_RES(res);
1393         return res;
1394
1395 out_inval:
1396         PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %x, cmd %p)",
1397                 (long long unsigned int)cmd->lun, cmd->cdb[0], cmd);
1398         PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply));
1399
1400 out_hwerr:
1401         res = -EINVAL;
1402
1403 out_hwerr_res_set:
1404         if (ucmd->background_exec) {
1405                 ucmd_put(ucmd);
1406                 goto out;
1407         } else {
1408                 scst_set_cmd_error(cmd,
1409                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1410                 goto out_compl;
1411         }
1412
1413 out_busy:
1414         scst_set_busy(cmd);
1415         goto out_compl;
1416 }
1417
1418 static int dev_user_process_reply(struct scst_user_dev *dev,
1419         struct scst_user_reply_cmd *reply)
1420 {
1421         int res = 0;
1422         struct scst_user_cmd *ucmd;
1423         int state;
1424
1425         TRACE_ENTRY();
1426
1427         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1428
1429         ucmd = __ucmd_find_hash(dev, reply->cmd_h);
1430         if (unlikely(ucmd == NULL)) {
1431                 TRACE_MGMT_DBG("cmd_h %d not found", reply->cmd_h);
1432                 res = -ESRCH;
1433                 goto out_unlock;
1434         }
1435
1436         if (unlikely(ucmd_get_check(ucmd))) {
1437                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", reply->cmd_h);
1438                 res = -ESRCH;
1439                 goto out_unlock;
1440         }
1441
1442         /* To sync. with dev_user_process_reply_exec(). See comment there. */
1443         smp_mb();
1444         if (ucmd->background_exec) {
1445                 state = UCMD_STATE_EXECING;
1446                 goto unlock_process;
1447         }
1448
1449         if (unlikely(ucmd->this_state_unjammed)) {
1450                 TRACE_MGMT_DBG("Reply on unjammed ucmd %p, ignoring",
1451                         ucmd);
1452                 goto out_unlock_put;
1453         }
1454
1455         if (unlikely(!ucmd->sent_to_user)) {
1456                 TRACE_MGMT_DBG("Ucmd %p isn't in the sent to user "
1457                         "state %x", ucmd, ucmd->state);
1458                 res = -EINVAL;
1459                 goto out_unlock_put;
1460         }
1461
1462         if (unlikely(reply->subcode != ucmd->user_cmd.subcode))
1463                 goto out_wrong_state;
1464
1465         if (unlikely(_IOC_NR(reply->subcode) != ucmd->state))
1466                 goto out_wrong_state;
1467
1468         state = ucmd->state;
1469         ucmd->sent_to_user = 0;
1470
1471 unlock_process:
1472         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1473
1474         switch (state) {
1475         case UCMD_STATE_PARSING:
1476                 res = dev_user_process_reply_parse(ucmd, reply);
1477                 break;
1478
1479         case UCMD_STATE_BUF_ALLOCING:
1480                 res = dev_user_process_reply_alloc(ucmd, reply);
1481                 break;
1482
1483         case UCMD_STATE_EXECING:
1484                 res = dev_user_process_reply_exec(ucmd, reply);
1485                 break;
1486
1487         case UCMD_STATE_ON_FREEING:
1488                 res = dev_user_process_reply_on_free(ucmd);
1489                 break;
1490
1491         case UCMD_STATE_ON_CACHE_FREEING:
1492                 res = dev_user_process_reply_on_cache_free(ucmd);
1493                 break;
1494
1495         case UCMD_STATE_TM_EXECING:
1496                 res = dev_user_process_reply_tm_exec(ucmd, reply->result);
1497                 break;
1498
1499         case UCMD_STATE_ATTACH_SESS:
1500         case UCMD_STATE_DETACH_SESS:
1501                 res = dev_user_process_reply_sess(ucmd, reply->result);
1502                 break;
1503
1504         default:
1505                 sBUG();
1506                 break;
1507         }
1508
1509 out_put:
1510         ucmd_put(ucmd);
1511
1512 out:
1513         TRACE_EXIT_RES(res);
1514         return res;
1515
1516 out_wrong_state:
1517         PRINT_ERROR("Command's %p subcode %x doesn't match internal "
1518                 "command's state %x or reply->subcode (%x) != ucmd->subcode "
1519                 "(%x)", ucmd, _IOC_NR(reply->subcode), ucmd->state,
1520                 reply->subcode, ucmd->user_cmd.subcode);
1521         res = -EINVAL;
1522         dev_user_unjam_cmd(ucmd, 0, NULL);
1523
1524 out_unlock_put:
1525         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1526         goto out_put;
1527
1528 out_unlock:
1529         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1530         goto out;
1531 }
1532
1533 static int dev_user_reply_cmd(struct file *file, void __user *arg)
1534 {
1535         int res = 0;
1536         struct scst_user_dev *dev;
1537         struct scst_user_reply_cmd reply;
1538
1539         TRACE_ENTRY();
1540
1541         mutex_lock(&dev_priv_mutex);
1542         dev = (struct scst_user_dev *)file->private_data;
1543         res = dev_user_check_reg(dev);
1544         if (unlikely(res != 0)) {
1545                 mutex_unlock(&dev_priv_mutex);
1546                 goto out;
1547         }
1548         down_read(&dev->dev_rwsem);
1549         mutex_unlock(&dev_priv_mutex);
1550
1551         res = copy_from_user(&reply, arg, sizeof(reply));
1552         if (unlikely(res < 0))
1553                 goto out_up;
1554
1555         TRACE_DBG("Reply for dev %s", dev->name);
1556
1557         TRACE_BUFFER("Reply", &reply, sizeof(reply));
1558
1559         res = dev_user_process_reply(dev, &reply);
1560         if (unlikely(res < 0))
1561                 goto out_up;
1562
1563 out_up:
1564         up_read(&dev->dev_rwsem);
1565
1566 out:
1567         TRACE_EXIT_RES(res);
1568         return res;
1569 }
1570
1571 static int dev_user_get_ext_cdb(struct file *file, void __user *arg)
1572 {
1573         int res = 0;
1574         struct scst_user_dev *dev;
1575         struct scst_user_cmd *ucmd;
1576         struct scst_cmd *cmd;
1577         struct scst_user_get_ext_cdb get;
1578
1579         TRACE_ENTRY();
1580
1581         mutex_lock(&dev_priv_mutex);
1582         dev = (struct scst_user_dev *)file->private_data;
1583         res = dev_user_check_reg(dev);
1584         if (unlikely(res != 0)) {
1585                 mutex_unlock(&dev_priv_mutex);
1586                 goto out;
1587         }
1588         down_read(&dev->dev_rwsem);
1589         mutex_unlock(&dev_priv_mutex);
1590
1591         res = copy_from_user(&get, arg, sizeof(get));
1592         if (unlikely(res < 0))
1593                 goto out_up;
1594
1595         TRACE_MGMT_DBG("Get ext cdb for dev %s", dev->name);
1596
1597         TRACE_BUFFER("Get ext cdb", &get, sizeof(get));
1598
1599         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1600
1601         ucmd = __ucmd_find_hash(dev, get.cmd_h);
1602         if (unlikely(ucmd == NULL)) {
1603                 TRACE_MGMT_DBG("cmd_h %d not found", get.cmd_h);
1604                 res = -ESRCH;
1605                 goto out_unlock;
1606         }
1607
1608         if (unlikely(ucmd_get_check(ucmd))) {
1609                 TRACE_MGMT_DBG("Found being destroyed cmd_h %d", get.cmd_h);
1610                 res = -ESRCH;
1611                 goto out_unlock;
1612         }
1613
1614         if ((ucmd->cmd != NULL) && (ucmd->state <= UCMD_STATE_EXECING) &&
1615             (ucmd->sent_to_user || ucmd->background_exec)) {
1616                 cmd = ucmd->cmd;
1617                 scst_cmd_get(cmd);
1618         } else {
1619                 TRACE_MGMT_DBG("Invalid ucmd state %d for cmd_h %d",
1620                         ucmd->state, get.cmd_h);
1621                 res = -EINVAL;
1622                 goto out_put;
1623         }
1624
1625         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1626
1627         if (cmd == NULL)
1628                 goto out_put;
1629
1630         if (cmd->ext_cdb == NULL)
1631                 goto out_cmd_put;
1632
1633         TRACE_BUFFER("EXT CDB", cmd->ext_cdb, cmd->ext_cdb_len);
1634         res = copy_to_user((void __user *)(unsigned long)get.ext_cdb_buffer,
1635                 cmd->ext_cdb, cmd->ext_cdb_len);
1636
1637 out_cmd_put:
1638         scst_cmd_put(cmd);
1639
1640 out_put:
1641         ucmd_put(ucmd);
1642
1643 out_up:
1644         up_read(&dev->dev_rwsem);
1645
1646 out:
1647         TRACE_EXIT_RES(res);
1648         return res;
1649
1650 out_unlock:
1651         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1652         goto out_up;
1653 }
1654
1655 static int dev_user_process_scst_commands(struct scst_user_dev *dev)
1656         __releases(&dev->cmd_lists.cmd_list_lock)
1657         __acquires(&dev->cmd_lists.cmd_list_lock)
1658 {
1659         int res = 0;
1660
1661         TRACE_ENTRY();
1662
1663         while (!list_empty(&dev->cmd_lists.active_cmd_list)) {
1664                 struct scst_cmd *cmd = list_entry(
1665                         dev->cmd_lists.active_cmd_list.next, typeof(*cmd),
1666                         cmd_list_entry);
1667                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
1668                 list_del(&cmd->cmd_list_entry);
1669                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1670                 scst_process_active_cmd(cmd, false);
1671                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1672                 res++;
1673         }
1674
1675         TRACE_EXIT_RES(res);
1676         return res;
1677 }
1678
1679 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1680 static struct scst_user_cmd *__dev_user_get_next_cmd(struct list_head *cmd_list)
1681         __releases(&dev->cmd_lists.cmd_list_lock)
1682         __acquires(&dev->cmd_lists.cmd_list_lock)
1683 {
1684         struct scst_user_cmd *u;
1685
1686 again:
1687         u = NULL;
1688         if (!list_empty(cmd_list)) {
1689                 u = list_entry(cmd_list->next, typeof(*u),
1690                                ready_cmd_list_entry);
1691
1692                 TRACE_DBG("Found ready ucmd %p", u);
1693                 list_del(&u->ready_cmd_list_entry);
1694
1695                 EXTRACHECKS_BUG_ON(u->this_state_unjammed);
1696
1697                 if (u->cmd != NULL) {
1698                         if (u->state == UCMD_STATE_EXECING) {
1699                                 struct scst_user_dev *dev = u->dev;
1700                                 int rc;
1701
1702                                 EXTRACHECKS_BUG_ON(u->jammed);
1703
1704                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1705
1706                                 rc = scst_check_local_events(u->cmd);
1707                                 if (unlikely(rc != 0)) {
1708                                         u->cmd->scst_cmd_done(u->cmd,
1709                                                 SCST_CMD_STATE_DEFAULT,
1710                                                 SCST_CONTEXT_DIRECT);
1711                                         /*
1712                                          * !! At this point cmd & u can be !!
1713                                          * !! already freed                !!
1714                                          */
1715                                         spin_lock_irq(
1716                                                 &dev->cmd_lists.cmd_list_lock);
1717                                         goto again;
1718                                 }
1719
1720                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1721                         } else if (unlikely(test_bit(SCST_CMD_ABORTED,
1722                                         &u->cmd->cmd_flags))) {
1723                                 switch (u->state) {
1724                                 case UCMD_STATE_PARSING:
1725                                 case UCMD_STATE_BUF_ALLOCING:
1726                                         TRACE_MGMT_DBG("Aborting ucmd %p", u);
1727                                         dev_user_unjam_cmd(u, 0, NULL);
1728                                         goto again;
1729                                 case UCMD_STATE_EXECING:
1730                                         EXTRACHECKS_BUG_ON(1);
1731                                 }
1732                         }
1733                 }
1734                 u->sent_to_user = 1;
1735                 u->seen_by_user = 1;
1736         }
1737         return u;
1738 }
1739
1740 static inline int test_cmd_lists(struct scst_user_dev *dev)
1741 {
1742         int res = !list_empty(&dev->cmd_lists.active_cmd_list) ||
1743                   !list_empty(&dev->ready_cmd_list) ||
1744                   !dev->blocking || dev->cleanup_done ||
1745                   signal_pending(current);
1746         return res;
1747 }
1748
1749 /* Called under cmd_lists.cmd_list_lock and IRQ off */
1750 static int dev_user_get_next_cmd(struct scst_user_dev *dev,
1751         struct scst_user_cmd **ucmd)
1752 {
1753         int res = 0;
1754         wait_queue_t wait;
1755
1756         TRACE_ENTRY();
1757
1758         init_waitqueue_entry(&wait, current);
1759
1760         while (1) {
1761                 if (!test_cmd_lists(dev)) {
1762                         add_wait_queue_exclusive(&dev->cmd_lists.cmd_list_waitQ,
1763                                 &wait);
1764                         for (;;) {
1765                                 set_current_state(TASK_INTERRUPTIBLE);
1766                                 if (test_cmd_lists(dev))
1767                                         break;
1768                                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1769                                 schedule();
1770                                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1771                         }
1772                         set_current_state(TASK_RUNNING);
1773                         remove_wait_queue(&dev->cmd_lists.cmd_list_waitQ,
1774                                 &wait);
1775                 }
1776
1777                 dev_user_process_scst_commands(dev);
1778
1779                 *ucmd = __dev_user_get_next_cmd(&dev->ready_cmd_list);
1780                 if (*ucmd != NULL)
1781                         break;
1782
1783                 if (!dev->blocking || dev->cleanup_done) {
1784                         res = -EAGAIN;
1785                         TRACE_DBG("No ready commands, returning %d", res);
1786                         break;
1787                 }
1788
1789                 if (signal_pending(current)) {
1790                         res = -EINTR;
1791                         TRACE_DBG("Signal pending, returning %d", res);
1792                         break;
1793                 }
1794         }
1795
1796         TRACE_EXIT_RES(res);
1797         return res;
1798 }
1799
1800 static int dev_user_reply_get_cmd(struct file *file, void __user *arg)
1801 {
1802         int res = 0;
1803         struct scst_user_dev *dev;
1804         struct scst_user_get_cmd *cmd;
1805         struct scst_user_reply_cmd *reply;
1806         struct scst_user_cmd *ucmd;
1807         uint64_t ureply;
1808
1809         TRACE_ENTRY();
1810
1811         mutex_lock(&dev_priv_mutex);
1812         dev = (struct scst_user_dev *)file->private_data;
1813         res = dev_user_check_reg(dev);
1814         if (unlikely(res != 0)) {
1815                 mutex_unlock(&dev_priv_mutex);
1816                 goto out;
1817         }
1818         down_read(&dev->dev_rwsem);
1819         mutex_unlock(&dev_priv_mutex);
1820
1821         /* get_user() can't be used with 64-bit values on x86_32 */
1822         res = copy_from_user(&ureply, (uint64_t __user *)
1823                 &((struct scst_user_get_cmd __user *)arg)->preply,
1824                 sizeof(ureply));
1825         if (unlikely(res < 0))
1826                 goto out_up;
1827
1828         TRACE_DBG("ureply %lld (dev %s)", (long long unsigned int)ureply,
1829                 dev->name);
1830
1831         cmd = kmem_cache_alloc(user_get_cmd_cachep, GFP_KERNEL);
1832         if (unlikely(cmd == NULL)) {
1833                 res = -ENOMEM;
1834                 goto out_up;
1835         }
1836
1837         if (ureply != 0) {
1838                 unsigned long u = (unsigned long)ureply;
1839                 reply = (struct scst_user_reply_cmd *)cmd;
1840                 res = copy_from_user(reply, (void __user *)u, sizeof(*reply));
1841                 if (unlikely(res < 0))
1842                         goto out_free;
1843
1844                 TRACE_BUFFER("Reply", reply, sizeof(*reply));
1845
1846                 res = dev_user_process_reply(dev, reply);
1847                 if (unlikely(res < 0))
1848                         goto out_free;
1849         }
1850
1851         kmem_cache_free(user_get_cmd_cachep, cmd);
1852
1853         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1854 again:
1855         res = dev_user_get_next_cmd(dev, &ucmd);
1856         if (res == 0) {
1857                 int len;
1858                 /*
1859                  * A misbehaving user space handler can make ucmd to get dead
1860                  * immediately after we released the lock, which can lead to
1861                  * copy of dead data to the user space, which can lead to a
1862                  * leak of sensitive information.
1863                  */
1864                 if (unlikely(ucmd_get_check(ucmd))) {
1865                         /* Oops, this ucmd is already being destroyed. Retry. */
1866                         goto again;
1867                 }
1868                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1869
1870                 EXTRACHECKS_BUG_ON(ucmd->user_cmd_payload_len == 0);
1871
1872                 len = ucmd->user_cmd_payload_len;
1873                 TRACE_DBG("ucmd %p (user_cmd %p), payload_len %d (len %d)",
1874                         ucmd, &ucmd->user_cmd, ucmd->user_cmd_payload_len, len);
1875                 TRACE_BUFFER("UCMD", &ucmd->user_cmd, len);
1876                 res = copy_to_user(arg, &ucmd->user_cmd, len);
1877                 if (unlikely(res != 0)) {
1878                         /* Requeue ucmd back */
1879                         TRACE_DBG("Copy to user failed (%d), requeuing ucmd %p "
1880                                 "back to head of ready cmd list", res, ucmd);
1881                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
1882                         list_add(&ucmd->ready_cmd_list_entry,
1883                                 &dev->ready_cmd_list);
1884                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1885                 }
1886 #ifdef CONFIG_SCST_EXTRACHECKS
1887                 else
1888                         ucmd->user_cmd_payload_len = 0;
1889 #endif
1890                 ucmd_put(ucmd);
1891         } else
1892                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
1893
1894 out_up:
1895         up_read(&dev->dev_rwsem);
1896
1897 out:
1898         TRACE_EXIT_RES(res);
1899         return res;
1900
1901 out_free:
1902         kmem_cache_free(user_get_cmd_cachep, cmd);
1903         goto out_up;
1904 }
1905
1906 static long dev_user_ioctl(struct file *file, unsigned int cmd,
1907         unsigned long arg)
1908 {
1909         long res;
1910
1911         TRACE_ENTRY();
1912
1913         switch (cmd) {
1914         case SCST_USER_REPLY_AND_GET_CMD:
1915                 TRACE_DBG("%s", "REPLY_AND_GET_CMD");
1916                 res = dev_user_reply_get_cmd(file, (void __user *)arg);
1917                 break;
1918
1919         case SCST_USER_REPLY_CMD:
1920                 TRACE_DBG("%s", "REPLY_CMD");
1921                 res = dev_user_reply_cmd(file, (void __user *)arg);
1922                 break;
1923
1924         case SCST_USER_GET_EXTENDED_CDB:
1925                 TRACE_DBG("%s", "GET_EXTENDED_CDB");
1926                 res = dev_user_get_ext_cdb(file, (void __user *)arg);
1927                 break;
1928
1929         case SCST_USER_REGISTER_DEVICE:
1930         {
1931                 struct scst_user_dev_desc *dev_desc;
1932                 TRACE_DBG("%s", "REGISTER_DEVICE");
1933                 dev_desc = kmalloc(sizeof(*dev_desc), GFP_KERNEL);
1934                 if (dev_desc == NULL) {
1935                         res = -ENOMEM;
1936                         goto out;
1937                 }
1938                 res = copy_from_user(dev_desc, (void __user *)arg,
1939                                      sizeof(*dev_desc));
1940                 if (res < 0) {
1941                         kfree(dev_desc);
1942                         goto out;
1943                 }
1944                 TRACE_BUFFER("dev_desc", dev_desc, sizeof(*dev_desc));
1945                 dev_desc->name[sizeof(dev_desc->name)-1] = '\0';
1946                 res = dev_user_register_dev(file, dev_desc);
1947                 kfree(dev_desc);
1948                 break;
1949         }
1950
1951         case SCST_USER_UNREGISTER_DEVICE:
1952                 TRACE_DBG("%s", "UNREGISTER_DEVICE");
1953                 res = dev_user_unregister_dev(file);
1954                 break;
1955
1956         case SCST_USER_FLUSH_CACHE:
1957                 TRACE_DBG("%s", "FLUSH_CACHE");
1958                 res = dev_user_flush_cache(file);
1959                 break;
1960
1961         case SCST_USER_SET_OPTIONS:
1962         {
1963                 struct scst_user_opt opt;
1964                 TRACE_DBG("%s", "SET_OPTIONS");
1965                 res = copy_from_user(&opt, (void __user *)arg, sizeof(opt));
1966                 if (res < 0)
1967                         goto out;
1968                 TRACE_BUFFER("opt", &opt, sizeof(opt));
1969                 res = dev_user_set_opt(file, &opt);
1970                 break;
1971         }
1972
1973         case SCST_USER_GET_OPTIONS:
1974                 TRACE_DBG("%s", "GET_OPTIONS");
1975                 res = dev_user_get_opt(file, (void __user *)arg);
1976                 break;
1977
1978         case SCST_USER_DEVICE_CAPACITY_CHANGED:
1979                 TRACE_DBG("%s", "CAPACITY_CHANGED");
1980                 res = dev_user_capacity_changed(file);
1981                 break;
1982
1983         default:
1984                 PRINT_ERROR("Invalid ioctl cmd %x", cmd);
1985                 res = -EINVAL;
1986                 goto out;
1987         }
1988
1989 out:
1990         TRACE_EXIT_RES(res);
1991         return res;
1992 }
1993
1994 static unsigned int dev_user_poll(struct file *file, poll_table *wait)
1995 {
1996         int res = 0;
1997         struct scst_user_dev *dev;
1998
1999         TRACE_ENTRY();
2000
2001         mutex_lock(&dev_priv_mutex);
2002         dev = (struct scst_user_dev *)file->private_data;
2003         res = dev_user_check_reg(dev);
2004         if (unlikely(res != 0)) {
2005                 mutex_unlock(&dev_priv_mutex);
2006                 goto out;
2007         }
2008         down_read(&dev->dev_rwsem);
2009         mutex_unlock(&dev_priv_mutex);
2010
2011         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2012
2013         if (!list_empty(&dev->ready_cmd_list) ||
2014             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2015                 res |= POLLIN | POLLRDNORM;
2016                 goto out_unlock;
2017         }
2018
2019         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2020
2021         TRACE_DBG("Before poll_wait() (dev %s)", dev->name);
2022         poll_wait(file, &dev->cmd_lists.cmd_list_waitQ, wait);
2023         TRACE_DBG("After poll_wait() (dev %s)", dev->name);
2024
2025         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2026
2027         if (!list_empty(&dev->ready_cmd_list) ||
2028             !list_empty(&dev->cmd_lists.active_cmd_list)) {
2029                 res |= POLLIN | POLLRDNORM;
2030                 goto out_unlock;
2031         }
2032
2033 out_unlock:
2034         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2035
2036         up_read(&dev->dev_rwsem);
2037
2038 out:
2039         TRACE_EXIT_HRES(res);
2040         return res;
2041 }
2042
2043 /*
2044  * Called under cmd_lists.cmd_list_lock, but can drop it inside, then reacquire.
2045  */
2046 static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy,
2047         unsigned long *flags)
2048         __releases(&dev->cmd_lists.cmd_list_lock)
2049         __acquires(&dev->cmd_lists.cmd_list_lock)
2050 {
2051         int state = ucmd->state;
2052         struct scst_user_dev *dev = ucmd->dev;
2053
2054         TRACE_ENTRY();
2055
2056         if (ucmd->this_state_unjammed)
2057                 goto out;
2058
2059         TRACE_MGMT_DBG("Unjamming ucmd %p (busy %d, state %x)", ucmd, busy,
2060                 state);
2061
2062         ucmd->jammed = 1;
2063         ucmd->this_state_unjammed = 1;
2064         ucmd->sent_to_user = 0;
2065
2066         switch (state) {
2067         case UCMD_STATE_PARSING:
2068         case UCMD_STATE_BUF_ALLOCING:
2069                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2070                         ucmd->aborted = 1;
2071                 else {
2072                         if (busy)
2073                                 scst_set_busy(ucmd->cmd);
2074                         else
2075                                 scst_set_cmd_error(ucmd->cmd,
2076                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2077                 }
2078                 scst_set_cmd_abnormal_done_state(ucmd->cmd);
2079
2080                 TRACE_MGMT_DBG("Adding ucmd %p to active list", ucmd);
2081                 list_add(&ucmd->cmd->cmd_list_entry,
2082                         &ucmd->cmd->cmd_lists->active_cmd_list);
2083                 wake_up(&ucmd->dev->cmd_lists.cmd_list_waitQ);
2084                 break;
2085
2086         case UCMD_STATE_EXECING:
2087                 if (flags != NULL)
2088                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2089                                                *flags);
2090                 else
2091                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2092
2093                 TRACE_MGMT_DBG("EXEC: unjamming ucmd %p", ucmd);
2094
2095                 if (test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags))
2096                         ucmd->aborted = 1;
2097                 else {
2098                         if (busy)
2099                                 scst_set_busy(ucmd->cmd);
2100                         else
2101                                 scst_set_cmd_error(ucmd->cmd,
2102                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2103                 }
2104
2105                 ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT,
2106                         SCST_CONTEXT_DIRECT);
2107                 /* !! At this point cmd and ucmd can be already freed !! */
2108
2109                 if (flags != NULL)
2110                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2111                                           *flags);
2112                 else
2113                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2114                 break;
2115
2116         case UCMD_STATE_ON_FREEING:
2117         case UCMD_STATE_ON_CACHE_FREEING:
2118         case UCMD_STATE_TM_EXECING:
2119         case UCMD_STATE_ATTACH_SESS:
2120         case UCMD_STATE_DETACH_SESS:
2121         {
2122                 if (flags != NULL)
2123                         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock,
2124                                                *flags);
2125                 else
2126                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2127
2128                 switch (state) {
2129                 case UCMD_STATE_ON_FREEING:
2130                         dev_user_process_reply_on_free(ucmd);
2131                         break;
2132
2133                 case UCMD_STATE_ON_CACHE_FREEING:
2134                         dev_user_process_reply_on_cache_free(ucmd);
2135                         break;
2136
2137                 case UCMD_STATE_TM_EXECING:
2138                         dev_user_process_reply_tm_exec(ucmd,
2139                                                        SCST_MGMT_STATUS_FAILED);
2140                         break;
2141
2142                 case UCMD_STATE_ATTACH_SESS:
2143                 case UCMD_STATE_DETACH_SESS:
2144                         dev_user_process_reply_sess(ucmd, -EFAULT);
2145                         break;
2146                 }
2147
2148                 if (flags != NULL)
2149                         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock,
2150                                           *flags);
2151                 else
2152                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2153                 break;
2154         }
2155
2156         default:
2157                 PRINT_CRIT_ERROR("Wrong ucmd state %x", state);
2158                 sBUG();
2159                 break;
2160         }
2161
2162 out:
2163         TRACE_EXIT();
2164         return;
2165 }
2166
2167 static void dev_user_unjam_dev(struct scst_user_dev *dev)
2168         __releases(&dev->cmd_lists.cmd_list_lock)
2169         __acquires(&dev->cmd_lists.cmd_list_lock)
2170 {
2171         int i;
2172         struct scst_user_cmd *ucmd;
2173
2174         TRACE_ENTRY();
2175
2176         TRACE_MGMT_DBG("Unjamming dev %p", dev);
2177
2178         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2179
2180 repeat:
2181         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
2182                 struct list_head *head = &dev->ucmd_hash[i];
2183
2184                 list_for_each_entry(ucmd, head, hash_list_entry) {
2185                         if (!ucmd->sent_to_user)
2186                                 continue;
2187
2188                         if (ucmd_get_check(ucmd))
2189                                 continue;
2190
2191                         TRACE_MGMT_DBG("ucmd %p, state %x, scst_cmd %p", ucmd,
2192                                 ucmd->state, ucmd->cmd);
2193
2194                         dev_user_unjam_cmd(ucmd, 0, NULL);
2195
2196                         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2197                         ucmd_put(ucmd);
2198                         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2199
2200                         goto repeat;
2201                 }
2202         }
2203
2204         if (dev_user_process_scst_commands(dev) != 0)
2205                 goto repeat;
2206
2207         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2208
2209         TRACE_EXIT();
2210         return;
2211 }
2212
2213 static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
2214         int status)
2215 {
2216         int res = 0;
2217
2218         TRACE_ENTRY();
2219
2220         TRACE_MGMT_DBG("TM reply (ucmd %p, fn %d, status %d)", ucmd,
2221                 ucmd->user_cmd.tm_cmd.fn, status);
2222
2223         if (status == SCST_MGMT_STATUS_TASK_NOT_EXIST) {
2224                 /*
2225                  * It is possible that user space seen TM cmd before cmd
2226                  * to abort or will never see it at all, because it was
2227                  * aborted on the way there. So, it is safe to return
2228                  * success instead, because, if there is the TM cmd at this
2229                  * point, then the cmd to abort apparrently does exist.
2230                  */
2231                 status = SCST_MGMT_STATUS_SUCCESS;
2232         }
2233
2234         scst_async_mcmd_completed(ucmd->mcmd, status);
2235
2236         ucmd_put(ucmd);
2237
2238         TRACE_EXIT_RES(res);
2239         return res;
2240 }
2241
2242 static void dev_user_abort_ready_commands(struct scst_user_dev *dev)
2243 {
2244         struct scst_user_cmd *ucmd;
2245         unsigned long flags;
2246
2247         TRACE_ENTRY();
2248
2249         spin_lock_irqsave(&dev->cmd_lists.cmd_list_lock, flags);
2250 again:
2251         list_for_each_entry(ucmd, &dev->ready_cmd_list, ready_cmd_list_entry) {
2252                 if ((ucmd->cmd != NULL) && !ucmd->seen_by_user &&
2253                     test_bit(SCST_CMD_ABORTED, &ucmd->cmd->cmd_flags)) {
2254                         switch (ucmd->state) {
2255                         case UCMD_STATE_PARSING:
2256                         case UCMD_STATE_BUF_ALLOCING:
2257                         case UCMD_STATE_EXECING:
2258                                 TRACE_MGMT_DBG("Aborting ready ucmd %p", ucmd);
2259                                 list_del(&ucmd->ready_cmd_list_entry);
2260                                 dev_user_unjam_cmd(ucmd, 0, &flags);
2261                                 goto again;
2262                         }
2263                 }
2264         }
2265
2266         spin_unlock_irqrestore(&dev->cmd_lists.cmd_list_lock, flags);
2267
2268         TRACE_EXIT();
2269         return;
2270 }
2271
2272 static int dev_user_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2273         struct scst_tgt_dev *tgt_dev)
2274 {
2275         struct scst_user_cmd *ucmd;
2276         struct scst_user_dev *dev =
2277                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2278         struct scst_user_cmd *ucmd_to_abort = NULL;
2279
2280         TRACE_ENTRY();
2281
2282         /*
2283          * In the used approach we don't do anything with hung devices, which
2284          * stopped responding and/or have stuck commands. We forcedly abort such
2285          * commands only if they not yet sent to the user space or if the device
2286          * is getting unloaded, e.g. if its handler program gets killed. This is
2287          * because it's pretty hard to distinguish between stuck and temporary
2288          * overloaded states of the device. There are several reasons for that:
2289          *
2290          * 1. Some commands need a lot of time to complete (several hours),
2291          *    so for an impatient user such command(s) will always look as
2292          *    stuck.
2293          *
2294          * 2. If we forcedly abort, i.e. abort before it's actually completed
2295          *    in the user space, just one command, we will have to put the whole
2296          *    device offline until we are sure that no more previously aborted
2297          *    commands will get executed. Otherwise, we might have a possibility
2298          *    for data corruption, when aborted and reported as completed
2299          *    command actually gets executed *after* new commands sent
2300          *    after the force abort was done. Many journaling file systems and
2301          *    databases use "provide required commands order via queue draining"
2302          *    approach and not putting the whole device offline after the forced
2303          *    abort will break it. This makes our decision, if a command stuck
2304          *    or not, cost a lot.
2305          *
2306          * So, we leave policy definition if a device stuck or not to
2307          * the user space and simply let all commands live until they are
2308          * completed or their devices get closed/killed. This approach is very
2309          * much OK, but can affect management commands, which need activity
2310          * suspending via scst_suspend_activity() function such as devices or
2311          * targets registration/removal. But during normal life such commands
2312          * should be rare. Plus, when possible, scst_suspend_activity() will
2313          * return after timeout EBUSY status to allow caller to not stuck
2314          * forever as well.
2315          *
2316          * But, anyway, ToDo, we should reimplement that in the SCST core, so
2317          * stuck commands would affect only related devices.
2318          */
2319
2320         dev_user_abort_ready_commands(dev);
2321
2322         /* We can't afford missing TM command due to memory shortage */
2323         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2324
2325         ucmd->user_cmd_payload_len =
2326                 offsetof(struct scst_user_get_cmd, tm_cmd) +
2327                 sizeof(ucmd->user_cmd.tm_cmd);
2328         ucmd->user_cmd.cmd_h = ucmd->h;
2329         ucmd->user_cmd.subcode = SCST_USER_TASK_MGMT;
2330         ucmd->user_cmd.tm_cmd.sess_h = (unsigned long)tgt_dev;
2331         ucmd->user_cmd.tm_cmd.fn = mcmd->fn;
2332         ucmd->user_cmd.tm_cmd.cmd_sn = mcmd->cmd_sn;
2333         ucmd->user_cmd.tm_cmd.cmd_sn_set = mcmd->cmd_sn_set;
2334
2335         if (mcmd->cmd_to_abort != NULL) {
2336                 ucmd_to_abort =
2337                         (struct scst_user_cmd *)mcmd->cmd_to_abort->dh_priv;
2338                 if (ucmd_to_abort != NULL)
2339                         ucmd->user_cmd.tm_cmd.cmd_h_to_abort = ucmd_to_abort->h;
2340         }
2341
2342         TRACE_MGMT_DBG("Preparing TM ucmd %p (h %d, fn %d, cmd_to_abort %p, "
2343                 "ucmd_to_abort %p, cmd_h_to_abort %d, mcmd %p)", ucmd, ucmd->h,
2344                 mcmd->fn, mcmd->cmd_to_abort, ucmd_to_abort,
2345                 ucmd->user_cmd.tm_cmd.cmd_h_to_abort, mcmd);
2346
2347         ucmd->mcmd = mcmd;
2348         ucmd->state = UCMD_STATE_TM_EXECING;
2349
2350         scst_prepare_async_mcmd(mcmd);
2351
2352         dev_user_add_to_ready(ucmd);
2353
2354         TRACE_EXIT();
2355         return SCST_DEV_TM_NOT_COMPLETED;
2356 }
2357
2358 static int dev_user_attach(struct scst_device *sdev)
2359 {
2360         int res = 0;
2361         struct scst_user_dev *dev = NULL, *d;
2362
2363         TRACE_ENTRY();
2364
2365         spin_lock(&dev_list_lock);
2366         list_for_each_entry(d, &dev_list, dev_list_entry) {
2367                 if (strcmp(d->name, sdev->virt_name) == 0) {
2368                         dev = d;
2369                         break;
2370                 }
2371         }
2372         spin_unlock(&dev_list_lock);
2373         if (dev == NULL) {
2374                 PRINT_ERROR("Device %s not found", sdev->virt_name);
2375                 res = -EINVAL;
2376                 goto out;
2377         }
2378
2379         sdev->p_cmd_lists = &dev->cmd_lists;
2380         sdev->dh_priv = dev;
2381         sdev->tst = dev->tst;
2382         sdev->queue_alg = dev->queue_alg;
2383         sdev->swp = dev->swp;
2384         sdev->tas = dev->tas;
2385         sdev->d_sense = dev->d_sense;
2386         sdev->has_own_order_mgmt = dev->has_own_order_mgmt;
2387
2388         dev->sdev = sdev;
2389
2390         PRINT_INFO("Attached user space SCSI target virtual device \"%s\"",
2391                 dev->name);
2392
2393 out:
2394         TRACE_EXIT();
2395         return res;
2396 }
2397
2398 static void dev_user_detach(struct scst_device *sdev)
2399 {
2400         struct scst_user_dev *dev = (struct scst_user_dev *)sdev->dh_priv;
2401
2402         TRACE_ENTRY();
2403
2404         TRACE_DBG("virt_id %d", sdev->virt_id);
2405
2406         PRINT_INFO("Detached user space SCSI target virtual device \"%s\"",
2407                 dev->name);
2408
2409         /* dev will be freed by the caller */
2410         sdev->dh_priv = NULL;
2411         dev->sdev = NULL;
2412
2413         TRACE_EXIT();
2414         return;
2415 }
2416
2417 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status)
2418 {
2419         int res = 0;
2420         unsigned long flags;
2421
2422         TRACE_ENTRY();
2423
2424         TRACE_MGMT_DBG("ucmd %p, cmpl %p, status %d", ucmd, ucmd->cmpl, status);
2425
2426         spin_lock_irqsave(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2427
2428         if (ucmd->state == UCMD_STATE_ATTACH_SESS) {
2429                 TRACE_MGMT_DBG("%s", "ATTACH_SESS finished");
2430                 ucmd->result = status;
2431         } else if (ucmd->state == UCMD_STATE_DETACH_SESS) {
2432                 TRACE_MGMT_DBG("%s", "DETACH_SESS finished");
2433         } else
2434                 sBUG();
2435
2436         if (ucmd->cmpl != NULL)
2437                 complete_all(ucmd->cmpl);
2438
2439         spin_unlock_irqrestore(&ucmd->dev->cmd_lists.cmd_list_lock, flags);
2440
2441         ucmd_put(ucmd);
2442
2443         TRACE_EXIT_RES(res);
2444         return res;
2445 }
2446
2447 static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
2448 {
2449         struct scst_user_dev *dev =
2450                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2451         int res = 0, rc;
2452         struct scst_user_cmd *ucmd;
2453         DECLARE_COMPLETION_ONSTACK(cmpl);
2454
2455         TRACE_ENTRY();
2456
2457         /*
2458          * We can't replace tgt_dev->pool, because it can be used to allocate
2459          * memory for SCST local commands, like REPORT LUNS, where there is no
2460          * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
2461          */
2462         if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
2463                 tgt_dev->dh_priv = dev->pool_clust;
2464         else
2465                 tgt_dev->dh_priv = dev->pool;
2466
2467         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
2468         if (ucmd == NULL)
2469                 goto out_nomem;
2470
2471         ucmd->cmpl = &cmpl;
2472
2473         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2474                 sizeof(ucmd->user_cmd.sess);
2475         ucmd->user_cmd.cmd_h = ucmd->h;
2476         ucmd->user_cmd.subcode = SCST_USER_ATTACH_SESS;
2477         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2478         ucmd->user_cmd.sess.lun = (uint64_t)tgt_dev->lun;
2479         ucmd->user_cmd.sess.threads_num = tgt_dev->sess->tgt->tgtt->threads_num;
2480         ucmd->user_cmd.sess.rd_only = tgt_dev->acg_dev->rd_only_flag;
2481         strncpy(ucmd->user_cmd.sess.initiator_name,
2482                 tgt_dev->sess->initiator_name,
2483                 sizeof(ucmd->user_cmd.sess.initiator_name)-1);
2484         ucmd->user_cmd.sess.initiator_name[
2485                 sizeof(ucmd->user_cmd.sess.initiator_name)-1] = '\0';
2486         if (tgt_dev->sess->tgt->default_group_name != NULL) {
2487                 strncpy(ucmd->user_cmd.sess.target_name,
2488                         &tgt_dev->sess->tgt->default_group_name[sizeof(SCST_DEFAULT_ACG_NAME)],
2489                         sizeof(ucmd->user_cmd.sess.target_name)-1);
2490                 ucmd->user_cmd.sess.target_name[
2491                         sizeof(ucmd->user_cmd.sess.target_name)-1] = '\0';
2492         }
2493
2494         TRACE_MGMT_DBG("Preparing ATTACH_SESS %p (h %d, sess_h %llx, LUN %llx, "
2495                 "threads_num %d, rd_only_flag %d, initiator %s, target %s)",
2496                 ucmd, ucmd->h, ucmd->user_cmd.sess.sess_h,
2497                 ucmd->user_cmd.sess.lun, ucmd->user_cmd.sess.threads_num,
2498                 ucmd->user_cmd.sess.rd_only, ucmd->user_cmd.sess.initiator_name,
2499                 ucmd->user_cmd.sess.target_name);
2500
2501         ucmd->state = UCMD_STATE_ATTACH_SESS;
2502
2503         ucmd_get(ucmd);
2504
2505         dev_user_add_to_ready(ucmd);
2506
2507         rc = wait_for_completion_timeout(ucmd->cmpl, DEV_USER_ATTACH_TIMEOUT);
2508         if (rc > 0)
2509                 res = ucmd->result;
2510         else {
2511                 PRINT_ERROR("%s", "ATTACH_SESS command timeout");
2512                 res = -EFAULT;
2513         }
2514
2515         sBUG_ON(irqs_disabled());
2516
2517         spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
2518         ucmd->cmpl = NULL;
2519         spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
2520
2521         ucmd_put(ucmd);
2522
2523 out:
2524         TRACE_EXIT_RES(res);
2525         return res;
2526
2527 out_nomem:
2528         res = -ENOMEM;
2529         goto out;
2530 }
2531
2532 static void dev_user_detach_tgt(struct scst_tgt_dev *tgt_dev)
2533 {
2534         struct scst_user_dev *dev =
2535                 (struct scst_user_dev *)tgt_dev->dev->dh_priv;
2536         struct scst_user_cmd *ucmd;
2537
2538         TRACE_ENTRY();
2539
2540         /*
2541          * We can't miss TM command due to memory shortage, because it might
2542          * lead to a memory leak in the user space handler.
2543          */
2544         ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL|__GFP_NOFAIL);
2545         if (ucmd == NULL)
2546                 goto out;
2547
2548         TRACE_MGMT_DBG("Preparing DETACH_SESS %p (h %d, sess_h %llx)", ucmd,
2549                 ucmd->h, ucmd->user_cmd.sess.sess_h);
2550
2551         ucmd->user_cmd_payload_len = offsetof(struct scst_user_get_cmd, sess) +
2552                 sizeof(ucmd->user_cmd.sess);
2553         ucmd->user_cmd.cmd_h = ucmd->h;
2554         ucmd->user_cmd.subcode = SCST_USER_DETACH_SESS;
2555         ucmd->user_cmd.sess.sess_h = (unsigned long)tgt_dev;
2556
2557         ucmd->state = UCMD_STATE_DETACH_SESS;
2558
2559         dev_user_add_to_ready(ucmd);
2560
2561 out:
2562         TRACE_EXIT();
2563         return;
2564 }
2565
2566 /* No locks are needed, but the activity must be suspended */
2567 static void dev_user_setup_functions(struct scst_user_dev *dev)
2568 {
2569         TRACE_ENTRY();
2570
2571         dev->devtype.parse = dev_user_parse;
2572         dev->devtype.dev_done = NULL;
2573
2574         if (dev->parse_type != SCST_USER_PARSE_CALL) {
2575                 switch (dev->devtype.type) {
2576                 case TYPE_DISK:
2577                         dev->generic_parse = scst_sbc_generic_parse;
2578                         dev->devtype.dev_done = dev_user_disk_done;
2579                         break;
2580
2581                 case TYPE_TAPE:
2582                         dev->generic_parse = scst_tape_generic_parse;
2583                         dev->devtype.dev_done = dev_user_tape_done;
2584                         break;
2585
2586                 case TYPE_MOD:
2587                         dev->generic_parse = scst_modisk_generic_parse;
2588                         dev->devtype.dev_done = dev_user_disk_done;
2589                         break;
2590
2591                 case TYPE_ROM:
2592                         dev->generic_parse = scst_cdrom_generic_parse;
2593                         dev->devtype.dev_done = dev_user_disk_done;
2594                         break;
2595
2596                 case TYPE_MEDIUM_CHANGER:
2597                         dev->generic_parse = scst_changer_generic_parse;
2598                         break;
2599
2600                 case TYPE_PROCESSOR:
2601                         dev->generic_parse = scst_processor_generic_parse;
2602                         break;
2603
2604                 case TYPE_RAID:
2605                         dev->generic_parse = scst_raid_generic_parse;
2606                         break;
2607
2608                 default:
2609                         PRINT_INFO("Unknown SCSI type %x, using PARSE_CALL "
2610                                 "for it", dev->devtype.type);
2611                         dev->parse_type = SCST_USER_PARSE_CALL;
2612                         break;
2613                 }
2614         } else {
2615                 dev->generic_parse = NULL;
2616                 dev->devtype.dev_done = NULL;
2617         }
2618
2619         TRACE_EXIT();
2620         return;
2621 }
2622
2623 static int dev_user_check_version(const struct scst_user_dev_desc *dev_desc)
2624 {
2625         char ver[sizeof(DEV_USER_VERSION)+1];
2626         int res;
2627
2628         res = copy_from_user(ver,
2629                         (void __user *)(unsigned long)dev_desc->version_str,
2630                         sizeof(ver));
2631         if (res < 0) {
2632                 PRINT_ERROR("%s", "Unable to get version string");
2633                 goto out;
2634         }
2635         ver[sizeof(ver)-1] = '\0';
2636
2637         if (strcmp(ver, DEV_USER_VERSION) != 0) {
2638                 /* ->name already 0-terminated in dev_user_ioctl() */
2639                 PRINT_ERROR("Incorrect version of user device %s (%s)",
2640                         dev_desc->name, ver);
2641                 res = -EINVAL;
2642                 goto out;
2643         }
2644
2645 out:
2646         return res;
2647 }
2648
2649 static int dev_user_register_dev(struct file *file,
2650         const struct scst_user_dev_desc *dev_desc)
2651 {
2652         int res = -ENOMEM, i;
2653         struct scst_user_dev *dev, *d;
2654         int block;
2655
2656         TRACE_ENTRY();
2657
2658         res = dev_user_check_version(dev_desc);
2659         if (res != 0)
2660                 goto out;
2661
2662         switch (dev_desc->type) {
2663         case TYPE_DISK:
2664         case TYPE_ROM:
2665         case TYPE_MOD:
2666                 if (dev_desc->block_size == 0) {
2667                         PRINT_ERROR("Wrong block size %d",
2668                                     dev_desc->block_size);
2669                         res = -EINVAL;
2670                         goto out;
2671                 }
2672                 block = scst_calc_block_shift(dev_desc->block_size);
2673                 if (block == -1) {
2674                         res = -EINVAL;
2675                         goto out;
2676                 }
2677                 break;
2678         default:
2679                 block = dev_desc->block_size;
2680                 break;
2681         }
2682
2683         if (!try_module_get(THIS_MODULE)) {
2684                 PRINT_ERROR("%s", "Fail to get module");
2685                 goto out;
2686         }
2687
2688         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2689         if (dev == NULL)
2690                 goto out_put;
2691
2692         init_rwsem(&dev->dev_rwsem);
2693         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
2694         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
2695         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
2696         INIT_LIST_HEAD(&dev->ready_cmd_list);
2697         if (file->f_flags & O_NONBLOCK) {
2698                 TRACE_DBG("%s", "Non-blocking operations");
2699                 dev->blocking = 0;
2700         } else
2701                 dev->blocking = 1;
2702         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++)
2703                 INIT_LIST_HEAD(&dev->ucmd_hash[i]);
2704
2705         strncpy(dev->name, dev_desc->name, sizeof(dev->name)-1);
2706         dev->name[sizeof(dev->name)-1] = '\0';
2707
2708         scst_init_mem_lim(&dev->udev_mem_lim);
2709
2710         dev->pool = sgv_pool_create(dev->name, sgv_no_clustering);
2711         if (dev->pool == NULL)
2712                 goto out_free_dev;
2713         sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
2714                 dev_user_free_sg_entries);
2715
2716         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust",
2717                 dev->name);
2718         dev->pool_clust = sgv_pool_create(dev->devtype.name,
2719                                 sgv_tail_clustering);
2720         if (dev->pool_clust == NULL)
2721                 goto out_free0;
2722         sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
2723                 dev_user_free_sg_entries);
2724
2725         scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
2726                 dev->name);
2727         dev->devtype.type = dev_desc->type;
2728         dev->devtype.threads_num = -1;
2729         dev->devtype.parse_atomic = 1;
2730         dev->devtype.exec_atomic = 0; /* no point to make it 1 */
2731         dev->devtype.dev_done_atomic = 1;
2732         dev->devtype.no_proc = 1;
2733         dev->devtype.attach = dev_user_attach;
2734         dev->devtype.detach = dev_user_detach;
2735         dev->devtype.attach_tgt = dev_user_attach_tgt;
2736         dev->devtype.detach_tgt = dev_user_detach_tgt;
2737         dev->devtype.exec = dev_user_exec;
2738         dev->devtype.on_free_cmd = dev_user_on_free_cmd;
2739         dev->devtype.task_mgmt_fn = dev_user_task_mgmt_fn;
2740
2741         init_completion(&dev->cleanup_cmpl);
2742         dev->block = block;
2743         dev->def_block = block;
2744
2745         res = __dev_user_set_opt(dev, &dev_desc->opt);
2746
2747         TRACE_MEM("dev %p, name %s", dev, dev->name);
2748
2749         spin_lock(&dev_list_lock);
2750
2751         list_for_each_entry(d, &dev_list, dev_list_entry) {
2752                 if (strcmp(d->name, dev->name) == 0) {
2753                         PRINT_ERROR("Device %s already exist",
2754                                 dev->name);
2755                         res = -EEXIST;
2756                         spin_unlock(&dev_list_lock);
2757                         goto out_free;
2758                 }
2759         }
2760
2761         list_add_tail(&dev->dev_list_entry, &dev_list);
2762
2763         spin_unlock(&dev_list_lock);
2764
2765         if (res != 0)
2766                 goto out_del_free;
2767
2768         res = scst_register_virtual_dev_driver(&dev->devtype);
2769         if (res < 0)
2770                 goto out_del_free;
2771
2772         dev->virt_id = scst_register_virtual_device(&dev->devtype, dev->name);
2773         if (dev->virt_id < 0) {
2774                 res = dev->virt_id;
2775                 goto out_unreg_handler;
2776         }
2777
2778         mutex_lock(&dev_priv_mutex);
2779         if (file->private_data != NULL) {
2780                 mutex_unlock(&dev_priv_mutex);
2781                 PRINT_ERROR("%s", "Device already registered");
2782                 res = -EINVAL;
2783                 goto out_unreg_drv;
2784         }
2785         file->private_data = dev;
2786         mutex_unlock(&dev_priv_mutex);
2787
2788 out:
2789         TRACE_EXIT_RES(res);
2790         return res;
2791
2792 out_unreg_drv:
2793         scst_unregister_virtual_device(dev->virt_id);
2794
2795 out_unreg_handler:
2796         scst_unregister_virtual_dev_driver(&dev->devtype);
2797
2798 out_del_free:
2799         spin_lock(&dev_list_lock);
2800         list_del(&dev->dev_list_entry);
2801         spin_unlock(&dev_list_lock);
2802
2803 out_free:
2804         sgv_pool_destroy(dev->pool_clust);
2805
2806 out_free0:
2807         sgv_pool_destroy(dev->pool);
2808
2809 out_free_dev:
2810         kfree(dev);
2811
2812 out_put:
2813         module_put(THIS_MODULE);
2814         goto out;
2815 }
2816
2817 static int dev_user_unregister_dev(struct file *file)
2818 {
2819         int res;
2820         struct scst_user_dev *dev;
2821
2822         TRACE_ENTRY();
2823
2824         mutex_lock(&dev_priv_mutex);
2825         dev = (struct scst_user_dev *)file->private_data;
2826         res = dev_user_check_reg(dev);
2827         if (res != 0) {
2828                 mutex_unlock(&dev_priv_mutex);
2829                 goto out;
2830         }
2831         down_read(&dev->dev_rwsem);
2832         mutex_unlock(&dev_priv_mutex);
2833
2834         res = scst_suspend_activity(true);
2835         if (res != 0)
2836                 goto out_up;
2837
2838         up_read(&dev->dev_rwsem);
2839
2840         dev_user_release(NULL, file);
2841
2842         scst_resume_activity();
2843
2844 out:
2845         TRACE_EXIT_RES(res);
2846         return res;
2847
2848 out_up:
2849         up_read(&dev->dev_rwsem);
2850         goto out;
2851 }
2852
2853 static int dev_user_flush_cache(struct file *file)
2854 {
2855         int res;
2856         struct scst_user_dev *dev;
2857
2858         TRACE_ENTRY();
2859
2860         mutex_lock(&dev_priv_mutex);
2861         dev = (struct scst_user_dev *)file->private_data;
2862         res = dev_user_check_reg(dev);
2863         if (res != 0) {
2864                 mutex_unlock(&dev_priv_mutex);
2865                 goto out;
2866         }
2867         down_read(&dev->dev_rwsem);
2868         mutex_unlock(&dev_priv_mutex);
2869
2870         res = scst_suspend_activity(true);
2871         if (res != 0)
2872                 goto out_up;
2873
2874         sgv_pool_flush(dev->pool);
2875         sgv_pool_flush(dev->pool_clust);
2876
2877         scst_resume_activity();
2878
2879 out_up:
2880         up_read(&dev->dev_rwsem);
2881
2882 out:
2883         TRACE_EXIT_RES(res);
2884         return res;
2885 }
2886
2887 static int dev_user_capacity_changed(struct file *file)
2888 {
2889         int res;
2890         struct scst_user_dev *dev;
2891
2892         TRACE_ENTRY();
2893
2894         mutex_lock(&dev_priv_mutex);
2895         dev = (struct scst_user_dev *)file->private_data;
2896         res = dev_user_check_reg(dev);
2897         if (res != 0) {
2898                 mutex_unlock(&dev_priv_mutex);
2899                 goto out;
2900         }
2901         down_read(&dev->dev_rwsem);
2902         mutex_unlock(&dev_priv_mutex);
2903
2904         scst_capacity_data_changed(dev->sdev);
2905
2906         up_read(&dev->dev_rwsem);
2907
2908 out:
2909         TRACE_EXIT_RES(res);
2910         return res;
2911 }
2912
2913
2914 static int __dev_user_set_opt(struct scst_user_dev *dev,
2915         const struct scst_user_opt *opt)
2916 {
2917         int res = 0;
2918
2919         TRACE_ENTRY();
2920
2921         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
2922                 "memory_reuse_type %x, partial_transfers_type %x, "
2923                 "partial_len %d", dev->name, opt->parse_type,
2924                 opt->on_free_cmd_type, opt->memory_reuse_type,
2925                 opt->partial_transfers_type, opt->partial_len);
2926
2927         if (opt->parse_type > SCST_USER_MAX_PARSE_OPT ||
2928             opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT ||
2929             opt->memory_reuse_type > SCST_USER_MAX_MEM_REUSE_OPT ||
2930             opt->partial_transfers_type > SCST_USER_MAX_PARTIAL_TRANSFERS_OPT) {
2931                 PRINT_ERROR("%s", "Invalid option");
2932                 res = -EINVAL;
2933                 goto out;
2934         }
2935
2936         if (((opt->tst != SCST_CONTR_MODE_ONE_TASK_SET) &&
2937              (opt->tst != SCST_CONTR_MODE_SEP_TASK_SETS)) ||
2938             ((opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) &&
2939              (opt->queue_alg != SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER)) ||
2940             (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) ||
2941             (opt->d_sense > 1)) {
2942                 PRINT_ERROR("Invalid SCSI option (tst %x, queue_alg %x, swp %x,"
2943                         " tas %x, d_sense %d, has_own_order_mgmt %x)", opt->tst,
2944                         opt->queue_alg, opt->swp, opt->tas, opt->d_sense,
2945                         opt->has_own_order_mgmt);
2946                 res = -EINVAL;
2947                 goto out;
2948         }
2949
2950         dev->parse_type = opt->parse_type;
2951         dev->on_free_cmd_type = opt->on_free_cmd_type;
2952         dev->memory_reuse_type = opt->memory_reuse_type;
2953         dev->partial_transfers_type = opt->partial_transfers_type;
2954         dev->partial_len = opt->partial_len;
2955
2956         dev->tst = opt->tst;
2957         dev->queue_alg = opt->queue_alg;
2958         dev->swp = opt->swp;
2959         dev->tas = opt->tas;
2960         dev->tst = opt->tst;
2961         dev->d_sense = opt->d_sense;
2962         dev->has_own_order_mgmt = opt->has_own_order_mgmt;
2963         if (dev->sdev != NULL) {
2964                 dev->sdev->tst = opt->tst;
2965                 dev->sdev->queue_alg = opt->queue_alg;
2966                 dev->sdev->swp = opt->swp;
2967                 dev->sdev->tas = opt->tas;
2968                 dev->sdev->d_sense = opt->d_sense;
2969                 dev->sdev->has_own_order_mgmt = opt->has_own_order_mgmt;
2970         }
2971
2972         dev_user_setup_functions(dev);
2973
2974 out:
2975         TRACE_EXIT_RES(res);
2976         return res;
2977 }
2978
2979 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
2980 {
2981         int res;
2982         struct scst_user_dev *dev;
2983
2984         TRACE_ENTRY();
2985
2986         mutex_lock(&dev_priv_mutex);
2987         dev = (struct scst_user_dev *)file->private_data;
2988         res = dev_user_check_reg(dev);
2989         if (res != 0) {
2990                 mutex_unlock(&dev_priv_mutex);
2991                 goto out;
2992         }
2993         down_read(&dev->dev_rwsem);
2994         mutex_unlock(&dev_priv_mutex);
2995
2996         res = scst_suspend_activity(true);
2997         if (res != 0)
2998                 goto out;
2999
3000         res = __dev_user_set_opt(dev, opt);
3001
3002         scst_resume_activity();
3003
3004         up_read(&dev->dev_rwsem);
3005
3006 out:
3007         TRACE_EXIT_RES(res);
3008         return res;
3009 }
3010
3011 static int dev_user_get_opt(struct file *file, void __user *arg)
3012 {
3013         int res;
3014         struct scst_user_dev *dev;
3015         struct scst_user_opt opt;
3016
3017         TRACE_ENTRY();
3018
3019         mutex_lock(&dev_priv_mutex);
3020         dev = (struct scst_user_dev *)file->private_data;
3021         res = dev_user_check_reg(dev);
3022         if (res != 0) {
3023                 mutex_unlock(&dev_priv_mutex);
3024                 goto out;
3025         }
3026         down_read(&dev->dev_rwsem);
3027         mutex_unlock(&dev_priv_mutex);
3028
3029         opt.parse_type = dev->parse_type;
3030         opt.on_free_cmd_type = dev->on_free_cmd_type;
3031         opt.memory_reuse_type = dev->memory_reuse_type;
3032         opt.partial_transfers_type = dev->partial_transfers_type;
3033         opt.partial_len = dev->partial_len;
3034         opt.tst = dev->tst;
3035         opt.queue_alg = dev->queue_alg;
3036         opt.tas = dev->tas;
3037         opt.swp = dev->swp;
3038         opt.d_sense = dev->d_sense;
3039         opt.has_own_order_mgmt = dev->has_own_order_mgmt;
3040
3041         TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, "
3042                 "memory_reuse_type %x, partial_transfers_type %x, "
3043                 "partial_len %d", dev->name, opt.parse_type,
3044                 opt.on_free_cmd_type, opt.memory_reuse_type,
3045                 opt.partial_transfers_type, opt.partial_len);
3046
3047         res = copy_to_user(arg, &opt, sizeof(opt));
3048
3049         up_read(&dev->dev_rwsem);
3050 out:
3051         TRACE_EXIT_RES(res);
3052         return res;
3053 }
3054
3055 static int dev_usr_parse(struct scst_cmd *cmd)
3056 {
3057         sBUG();
3058         return SCST_CMD_STATE_DEFAULT;
3059 }
3060
3061 /* Needed only for /proc support */
3062 #define USR_TYPE {                      \
3063         .name =         DEV_USER_NAME,  \
3064         .type =         -1,             \
3065         .parse =        dev_usr_parse,  \
3066 }
3067
3068 static struct scst_dev_type dev_user_devtype = USR_TYPE;
3069
3070 static int dev_user_release(struct inode *inode, struct file *file)
3071 {
3072         int res = 0;
3073         struct scst_user_dev *dev;
3074
3075         TRACE_ENTRY();
3076
3077         mutex_lock(&dev_priv_mutex);
3078         dev = (struct scst_user_dev *)file->private_data;
3079         if (dev == NULL) {
3080                 mutex_unlock(&dev_priv_mutex);
3081                 goto out;
3082         }
3083         file->private_data = NULL;
3084
3085         TRACE(TRACE_MGMT, "Releasing dev %s", dev->name);
3086
3087         spin_lock(&dev_list_lock);
3088         list_del(&dev->dev_list_entry);
3089         spin_unlock(&dev_list_lock);
3090
3091         dev->blocking = 0;
3092         wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
3093
3094         down_write(&dev->dev_rwsem);
3095         mutex_unlock(&dev_priv_mutex);
3096
3097         spin_lock(&cleanup_lock);
3098         list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
3099         spin_unlock(&cleanup_lock);
3100
3101         wake_up(&cleanup_list_waitQ);
3102
3103         scst_unregister_virtual_device(dev->virt_id);
3104         scst_unregister_virtual_dev_driver(&dev->devtype);
3105
3106         sgv_pool_destroy(dev->pool_clust);
3107         sgv_pool_destroy(dev->pool);
3108
3109         TRACE_DBG("Unregistering finished (dev %p)", dev);
3110
3111         dev->cleanup_done = 1;
3112
3113         wake_up(&cleanup_list_waitQ);
3114         wake_up(&dev->cmd_lists.cmd_list_waitQ);
3115
3116         wait_for_completion(&dev->cleanup_cmpl);
3117
3118         up_write(&dev->dev_rwsem); /* to make the debug check happy */
3119
3120         TRACE_DBG("Releasing completed (dev %p)", dev);
3121
3122         kfree(dev);
3123
3124         module_put(THIS_MODULE);
3125
3126 out:
3127         TRACE_EXIT_RES(res);
3128         return res;
3129 }
3130
3131 static int dev_user_process_cleanup(struct scst_user_dev *dev)
3132 {
3133         struct scst_user_cmd *ucmd;
3134         int rc, res = 1;
3135
3136         TRACE_ENTRY();
3137
3138         sBUG_ON(dev->blocking);
3139         wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
3140
3141         while (1) {
3142                 TRACE_DBG("Cleanuping dev %p", dev);
3143
3144                 dev_user_unjam_dev(dev);
3145
3146                 spin_lock_irq(&dev->cmd_lists.cmd_list_lock);
3147
3148                 rc = dev_user_get_next_cmd(dev, &ucmd);
3149                 if (rc == 0)
3150                         dev_user_unjam_cmd(ucmd, 1, NULL);
3151
3152                 spin_unlock_irq(&dev->cmd_lists.cmd_list_lock);
3153
3154                 if (rc == -EAGAIN) {
3155                         if (dev->cleanup_done)
3156                                 break;
3157                         else {
3158                                 TRACE_DBG("No more commands (dev %p)", dev);
3159                                 goto out;
3160                         }
3161                 }
3162         }
3163
3164 #ifdef CONFIG_SCST_EXTRACHECKS
3165 {
3166         int i;
3167         for (i = 0; i < (int)ARRAY_SIZE(dev->ucmd_hash); i++) {
3168                 struct list_head *head = &dev->ucmd_hash[i];
3169                 struct scst_user_cmd *ucmd2;
3170 again:
3171                 list_for_each_entry(ucmd2, head, hash_list_entry) {
3172                         PRINT_ERROR("Lost ucmd %p (state %x, ref %d)", ucmd2,
3173                                 ucmd2->state, atomic_read(&ucmd2->ucmd_ref));
3174                         ucmd_put(ucmd2);
3175                         goto again;
3176                 }
3177         }
3178 }
3179 #endif
3180
3181         TRACE_DBG("Cleanuping done (dev %p)", dev);
3182         complete_all(&dev->cleanup_cmpl);
3183         res = 0;
3184
3185 out:
3186         TRACE_EXIT_RES(res);
3187         return res;
3188 }
3189
3190 static inline int test_cleanup_list(void)
3191 {
3192         int res = !list_empty(&cleanup_list) ||
3193                   unlikely(kthread_should_stop());
3194         return res;
3195 }
3196
3197 static int dev_user_cleanup_thread(void *arg)
3198 {
3199         TRACE_ENTRY();
3200
3201         PRINT_INFO("Cleanup thread started, PID %d", current->pid);
3202
3203         current->flags |= PF_NOFREEZE;
3204
3205         spin_lock(&cleanup_lock);
3206         while (!kthread_should_stop()) {
3207                 wait_queue_t wait;
3208                 init_waitqueue_entry(&wait, current);
3209
3210                 if (!test_cleanup_list()) {
3211                         add_wait_queue_exclusive(&cleanup_list_waitQ, &wait);
3212                         for (;;) {
3213                                 set_current_state(TASK_INTERRUPTIBLE);
3214                                 if (test_cleanup_list())
3215                                         break;
3216                                 spin_unlock(&cleanup_lock);
3217                                 schedule();
3218                                 spin_lock(&cleanup_lock);
3219                         }
3220                         set_current_state(TASK_RUNNING);
3221                         remove_wait_queue(&cleanup_list_waitQ, &wait);
3222                 }
3223
3224                 /*
3225                  * We have to poll devices, because commands can go from SCST
3226                  * core on cmd_list_waitQ and we have no practical way to
3227                  * detect them.
3228                  */
3229
3230                 while (1) {
3231                         struct scst_user_dev *dev;
3232                         LIST_HEAD(cl_devs);
3233
3234                         while (!list_empty(&cleanup_list)) {
3235                                 int rc;
3236
3237                                 dev = list_entry(cleanup_list.next,
3238                                         typeof(*dev), cleanup_list_entry);
3239                                 list_del(&dev->cleanup_list_entry);
3240
3241                                 spin_unlock(&cleanup_lock);
3242                                 rc = dev_user_process_cleanup(dev);
3243                                 spin_lock(&cleanup_lock);
3244
3245                                 if (rc != 0)
3246                                         list_add_tail(&dev->cleanup_list_entry,
3247                                                 &cl_devs);
3248                         }
3249
3250                         if (list_empty(&cl_devs))
3251                                 break;
3252
3253                         spin_unlock(&cleanup_lock);
3254                         msleep(100);
3255                         spin_lock(&cleanup_lock);
3256
3257                         while (!list_empty(&cl_devs)) {
3258                                 dev = list_entry(cl_devs.next, typeof(*dev),
3259                                         cleanup_list_entry);
3260                                 list_move_tail(&dev->cleanup_list_entry,
3261                                         &cleanup_list);
3262                         }
3263                 }
3264         }
3265         spin_unlock(&cleanup_lock);
3266
3267         /*
3268          * If kthread_should_stop() is true, we are guaranteed to be
3269          * on the module unload, so cleanup_list must be empty.
3270          */
3271         sBUG_ON(!list_empty(&cleanup_list));
3272
3273         PRINT_INFO("Cleanup thread PID %d finished", current->pid);
3274
3275         TRACE_EXIT();
3276         return 0;
3277 }
3278
3279 static int __init init_scst_user(void)
3280 {
3281         int res = 0;
3282         struct max_get_reply {
3283                 union {
3284                         struct scst_user_get_cmd g;
3285                         struct scst_user_reply_cmd r;
3286                 };
3287         };
3288 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3289         struct class_device *class_member;
3290 #else
3291         struct device *dev;
3292 #endif
3293
3294         TRACE_ENTRY();
3295
3296 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
3297         PRINT_ERROR("%s", "HIGHMEM kernel configurations are not supported. "
3298                 "Consider changing VMSPLIT option or use a 64-bit "
3299                 "configuration instead. See README file for details.");
3300         res = -EINVAL;
3301         goto out;
3302 #endif
3303
3304         user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
3305         if (user_cmd_cachep == NULL) {
3306                 res = -ENOMEM;
3307                 goto out;
3308         }
3309
3310         user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
3311         if (user_get_cmd_cachep == NULL) {
3312                 res = -ENOMEM;
3313                 goto out_cache;
3314         }
3315
3316         dev_user_devtype.module = THIS_MODULE;
3317
3318         res = scst_register_virtual_dev_driver(&dev_user_devtype);
3319         if (res < 0)
3320                 goto out_cache1;
3321
3322         res = scst_dev_handler_build_std_proc(&dev_user_devtype);
3323         if (res != 0)
3324                 goto out_unreg;
3325
3326         dev_user_sysfs_class = class_create(THIS_MODULE, DEV_USER_NAME);
3327         if (IS_ERR(dev_user_sysfs_class)) {
3328                 PRINT_ERROR("%s", "Unable create sysfs class for SCST user "
3329                         "space handler");
3330                 res = PTR_ERR(dev_user_sysfs_class);
3331                 goto out_proc;
3332         }
3333
3334         res = register_chrdev(DEV_USER_MAJOR, DEV_USER_NAME, &dev_user_fops);
3335         if (res) {
3336                 PRINT_ERROR("Unable to get major %d for SCSI tapes",
3337                             DEV_USER_MAJOR);
3338                 goto out_class;
3339         }
3340
3341 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3342         class_member = class_device_create(dev_user_sysfs_class, NULL,
3343                                 MKDEV(DEV_USER_MAJOR, 0), NULL, DEV_USER_NAME);
3344         if (IS_ERR(class_member)) {
3345                 res = PTR_ERR(class_member);
3346                 goto out_chrdev;
3347         }
3348 #else
3349         dev = device_create(dev_user_sysfs_class, NULL,
3350                             MKDEV(DEV_USER_MAJOR, 0),
3351 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
3352                                 NULL,
3353 #endif
3354                                 DEV_USER_NAME);
3355         if (IS_ERR(dev)) {
3356                 res = PTR_ERR(dev);
3357                 goto out_chrdev;
3358         }
3359 #endif
3360
3361         cleanup_thread = kthread_run(dev_user_cleanup_thread, NULL,
3362                 "scst_usr_cleanupd");
3363         if (IS_ERR(cleanup_thread)) {
3364                 res = PTR_ERR(cleanup_thread);
3365                 PRINT_ERROR("kthread_create() failed: %d", res);
3366                 goto out_dev;
3367         }
3368
3369 out:
3370         TRACE_EXIT_RES(res);
3371         return res;
3372
3373 out_dev:
3374 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3375         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3376 #else
3377         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3378 #endif
3379
3380 out_chrdev:
3381         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3382
3383 out_class:
3384         class_destroy(dev_user_sysfs_class);
3385
3386 out_proc:
3387         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3388
3389 out_unreg:
3390         scst_unregister_dev_driver(&dev_user_devtype);
3391
3392 out_cache1:
3393         kmem_cache_destroy(user_get_cmd_cachep);
3394
3395 out_cache:
3396         kmem_cache_destroy(user_cmd_cachep);
3397         goto out;
3398 }
3399
3400 static void __exit exit_scst_user(void)
3401 {
3402         int rc;
3403
3404         TRACE_ENTRY();
3405
3406         rc = kthread_stop(cleanup_thread);
3407         if (rc < 0)
3408                 TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3409
3410         unregister_chrdev(DEV_USER_MAJOR, DEV_USER_NAME);
3411 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21)
3412         class_device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3413 #else
3414         device_destroy(dev_user_sysfs_class, MKDEV(DEV_USER_MAJOR, 0));
3415 #endif
3416         class_destroy(dev_user_sysfs_class);
3417
3418         scst_dev_handler_destroy_std_proc(&dev_user_devtype);
3419         scst_unregister_virtual_dev_driver(&dev_user_devtype);
3420
3421         kmem_cache_destroy(user_get_cmd_cachep);
3422         kmem_cache_destroy(user_cmd_cachep);
3423
3424         TRACE_EXIT();
3425         return;
3426 }
3427
3428 module_init(init_scst_user);
3429 module_exit(exit_scst_user);
3430
3431 MODULE_AUTHOR("Vladislav Bolkhovitin");
3432 MODULE_LICENSE("GPL");
3433 MODULE_DESCRIPTION("Virtual user space device handler for SCST");
3434 MODULE_VERSION(SCST_VERSION_STRING);
3435 MODULE_ALIAS_CHARDEV_MAJOR(DEV_USER_MAJOR);