scst/src/scst_main.c

   1 /*
   2  *  scst_main.c
   3  *
   4  *  Copyright (C) 2004 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
   6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
   7  *
   8  *  This program is free software; you can redistribute it and/or
   9  *  modify it under the terms of the GNU General Public License
  10  *  as published by the Free Software Foundation, version 2
  11  *  of the License.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  *  GNU General Public License for more details.
  17  */
  18
  19 #include <linux/module.h>
  20
  21 #include <linux/init.h>
  22 #include <linux/kernel.h>
  23 #include <linux/errno.h>
  24 #include <linux/list.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched.h>
  28 #include <linux/unistd.h>
  29 #include <linux/string.h>
  30 #include <linux/kthread.h>
  31
  32 #include "scst.h"
  33 #include "scst_priv.h"
  34 #include "scst_mem.h"
  35
  36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
  37 #warning "HIGHMEM kernel configurations are fully supported, but not\
  38  recommended for performance reasons. Consider changing VMSPLIT\
  39  option or use a 64-bit configuration instead. See README file for\
  40  details."
  41 #endif
  42
  43 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
  44 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
  45 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
  46 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
  47  your kernel and CONFIG_SCST_STRICT_SERIALIZING isn't defined.\
  48  Pass-through dev handlers will not work."
  49 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
  50 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
  51 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
  52  your kernel. Pass-through dev handlers will not work."
  53 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
  54 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
  55
  56 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
  57 #if !defined(SCST_IO_CONTEXT)
  58 #warning "Patch io_context-<kernel-version> was not applied\
  59  on your kernel. SCST will be working with not the best performance."
  60 #endif
  61 #else
  62 #warning "There is no patch io_context-<kernel-version>\
  63  for your kernel version. For performance reasons it is strongly recommended\
  64  to upgrade your kernel to version >= 2.6.26.x."
  65 #endif
  66
  67 /**
  68  ** SCST global variables. They are all uninitialized to have their layout in
  69  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
  70  ** variable separately from nonzero-initialized ones.
  71  **/
  72
  73 /*
  74  * All targets, devices and dev_types management is done under this mutex.
  75  *
  76  * It must NOT be used in any works (schedule_work(), etc.), because
  77  * otherwise a deadlock (double lock, actually) is possible, e.g., with
  78  * scst_user detach_tgt(), which is called under scst_mutex and calls
  79  * flush_scheduled_work().
  80  */
  81 struct mutex scst_mutex;
  82
  83  /* All 3 protected by scst_mutex */
  84 struct list_head scst_template_list;
  85 struct list_head scst_dev_list;
  86 struct list_head scst_dev_type_list;
  87
  88 spinlock_t scst_main_lock;
  89
  90 static struct kmem_cache *scst_mgmt_cachep;
  91 mempool_t *scst_mgmt_mempool;
  92 static struct kmem_cache *scst_mgmt_stub_cachep;
  93 mempool_t *scst_mgmt_stub_mempool;
  94 static struct kmem_cache *scst_ua_cachep;
  95 mempool_t *scst_ua_mempool;
  96 static struct kmem_cache *scst_sense_cachep;
  97 mempool_t *scst_sense_mempool;
  98 static struct kmem_cache *scst_aen_cachep;
  99 mempool_t *scst_aen_mempool;
 100 struct kmem_cache *scst_tgtd_cachep;
 101 struct kmem_cache *scst_sess_cachep;
 102 struct kmem_cache *scst_acgd_cachep;
 103
 104 struct list_head scst_acg_list;
 105 #ifdef CONFIG_SCST_PROC
 106 struct scst_acg *scst_default_acg;
 107 #endif
 108
 109 spinlock_t scst_init_lock;
 110 wait_queue_head_t scst_init_cmd_list_waitQ;
 111 struct list_head scst_init_cmd_list;
 112 unsigned int scst_init_poll_cnt;
 113
 114 struct kmem_cache *scst_cmd_cachep;
 115
 116 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
 117 unsigned long scst_trace_flag;
 118 #endif
 119
 120 unsigned long scst_flags;
 121 atomic_t scst_cmd_count;
 122
 123 struct scst_cmd_lists scst_main_cmd_lists;
 124
 125 struct scst_tasklet scst_tasklets[NR_CPUS];
 126
 127 spinlock_t scst_mcmd_lock;
 128 struct list_head scst_active_mgmt_cmd_list;
 129 struct list_head scst_delayed_mgmt_cmd_list;
 130 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
 131
 132 wait_queue_head_t scst_mgmt_waitQ;
 133 spinlock_t scst_mgmt_lock;
 134 struct list_head scst_sess_init_list;
 135 struct list_head scst_sess_shut_list;
 136
 137 wait_queue_head_t scst_dev_cmd_waitQ;
 138
 139 static struct mutex scst_suspend_mutex;
 140 /* protected by scst_suspend_mutex */
 141 static struct list_head scst_cmd_lists_list;
 142
 143 static int scst_threads;
 144 struct mutex scst_global_threads_mutex;
 145 u32 scst_nr_global_threads;
 146 static struct list_head scst_global_threads_list;
 147 static struct task_struct *scst_init_cmd_thread;
 148 static struct task_struct *scst_mgmt_thread;
 149 static struct task_struct *scst_mgmt_cmd_thread;
 150
 151 static int suspend_count;
 152
 153 static int scst_virt_dev_last_id; /* protected by scst_mutex */
 154
 155 static unsigned int scst_max_cmd_mem;
 156 unsigned int scst_max_dev_cmd_mem;
 157
 158 module_param_named(scst_threads, scst_threads, int, 0);
 159 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
 160
 161 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, S_IRUGO);
 162 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
 163         "all SCSI commands of all devices at any given time in MB");
 164
 165 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, S_IRUGO);
 166 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
 167         "by all SCSI commands of a device at any given time in MB");
 168
 169 struct scst_dev_type scst_null_devtype = {
 170         .name = "none",
 171 };
 172
 173 static void __scst_resume_activity(void);
 174
 175 int __scst_register_target_template(struct scst_tgt_template *vtt,
 176         const char *version)
 177 {
 178         int res = 0;
 179         struct scst_tgt_template *t;
 180         static DEFINE_MUTEX(m);
 181
 182         TRACE_ENTRY();
 183
 184         INIT_LIST_HEAD(&vtt->tgt_list);
 185
 186         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
 187                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
 188                 res = -EINVAL;
 189                 goto out_err;
 190         }
 191
 192         if (!vtt->detect) {
 193                 PRINT_ERROR("Target driver %s must have "
 194                         "detect() method.", vtt->name);
 195                 res = -EINVAL;
 196                 goto out_err;
 197         }
 198
 199         if (!vtt->release) {
 200                 PRINT_ERROR("Target driver %s must have "
 201                         "release() method.", vtt->name);
 202                 res = -EINVAL;
 203                 goto out_err;
 204         }
 205
 206         if (!vtt->xmit_response) {
 207                 PRINT_ERROR("Target driver %s must have "
 208                         "xmit_response() method.", vtt->name);
 209                 res = -EINVAL;
 210                 goto out_err;
 211         }
 212
 213         if (vtt->threads_num < 0) {
 214                 PRINT_ERROR("Wrong threads_num value %d for "
 215                         "target \"%s\"", vtt->threads_num,
 216                         vtt->name);
 217                 res = -EINVAL;
 218                 goto out_err;
 219         }
 220
 221 #ifndef CONFIG_SCST_PROC
 222         if (!vtt->enable_tgt || !vtt->is_tgt_enabled) {
 223                 PRINT_WARNING("Target driver %s doesn't have enable_tgt() "
 224                         "and/or is_tgt_enabled() method(s). This is unsafe "
 225                         "and can lead that initiators connected on the "
 226                         "initialization time can see an unexpected set of "
 227                         "devices or no devices at all!", vtt->name);
 228         }
 229 #else
 230         if (!vtt->no_proc_entry) {
 231                 res = scst_build_proc_target_dir_entries(vtt);
 232                 if (res < 0)
 233                         goto out_err;
 234         }
 235 #endif
 236
 237         res = scst_create_tgtt_sysfs(vtt);
 238         if (res)
 239                 goto out_sysfs_err;
 240
 241         if (vtt->rdy_to_xfer == NULL)
 242                 vtt->rdy_to_xfer_atomic = 1;
 243
 244         if (mutex_lock_interruptible(&m) != 0)
 245                 goto out_sysfs_err;
 246
 247         if (mutex_lock_interruptible(&scst_mutex) != 0)
 248                 goto out_m_err;
 249         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
 250                 if (strcmp(t->name, vtt->name) == 0) {
 251                         PRINT_ERROR("Target driver %s already registered",
 252                                 vtt->name);
 253                         mutex_unlock(&scst_mutex);
 254                         goto out_m_err;
 255                 }
 256         }
 257         mutex_unlock(&scst_mutex);
 258
 259         TRACE_DBG("%s", "Calling target driver's detect()");
 260         res = vtt->detect(vtt);
 261         TRACE_DBG("Target driver's detect() returned %d", res);
 262         if (res < 0) {
 263                 PRINT_ERROR("%s", "The detect() routine failed");
 264                 res = -EINVAL;
 265                 goto out_m_err;
 266         }
 267
 268         mutex_lock(&scst_mutex);
 269         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
 270         mutex_unlock(&scst_mutex);
 271
 272         res = 0;
 273
 274         PRINT_INFO("Target template %s registered successfully", vtt->name);
 275
 276         mutex_unlock(&m);
 277
 278 out:
 279         TRACE_EXIT_RES(res);
 280         return res;
 281
 282 out_m_err:
 283         mutex_unlock(&m);
 284
 285 out_sysfs_err:
 286 #ifdef CONFIG_SCST_PROC
 287         scst_cleanup_proc_target_dir_entries(vtt);
 288 #endif
 289         scst_tgtt_sysfs_put(vtt);
 290
 291 out_err:
 292         PRINT_ERROR("Failed to register target template %s", vtt->name);
 293         goto out;
 294 }
 295 EXPORT_SYMBOL(__scst_register_target_template);
 296
 297 void scst_unregister_target_template(struct scst_tgt_template *vtt)
 298 {
 299         struct scst_tgt *tgt;
 300         struct scst_tgt_template *t;
 301         int found = 0;
 302
 303         TRACE_ENTRY();
 304
 305         mutex_lock(&scst_mutex);
 306
 307         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
 308                 if (strcmp(t->name, vtt->name) == 0) {
 309                         found = 1;
 310                         break;
 311                 }
 312         }
 313         if (!found) {
 314                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
 315                 goto out_err_up;
 316         }
 317
 318 restart:
 319         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
 320                 mutex_unlock(&scst_mutex);
 321                 scst_unregister(tgt);
 322                 mutex_lock(&scst_mutex);
 323                 goto restart;
 324         }
 325         list_del(&vtt->scst_template_list_entry);
 326
 327         mutex_unlock(&scst_mutex);
 328
 329 #ifdef CONFIG_SCST_PROC
 330         scst_cleanup_proc_target_dir_entries(vtt);
 331 #endif
 332
 333         scst_tgtt_sysfs_put(vtt);
 334
 335         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
 336
 337 out:
 338         TRACE_EXIT();
 339         return;
 340
 341 out_err_up:
 342         mutex_unlock(&scst_mutex);
 343         goto out;
 344 }
 345 EXPORT_SYMBOL(scst_unregister_target_template);
 346
 347 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
 348         const char *target_name)
 349 {
 350         struct scst_tgt *tgt;
 351         int rc = 0;
 352
 353         TRACE_ENTRY();
 354
 355         tgt = scst_alloc_tgt(vtt);
 356         if (tgt == NULL) {
 357                 rc = -ENOMEM;
 358                 goto out_err;
 359         }
 360
 361         rc = scst_suspend_activity(true);
 362         if (rc != 0)
 363                 goto out_free_tgt_err;
 364
 365         if (mutex_lock_interruptible(&scst_mutex) != 0) {
 366                 rc = -EINTR;
 367                 goto out_resume_free;
 368         }
 369
 370         if (target_name != NULL) {
 371 #ifdef CONFIG_SCST_PROC
 372                 int len = strlen(target_name) +
 373                         strlen(SCST_DEFAULT_ACG_NAME) + 1 + 1;
 374
 375                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
 376                 if (tgt->default_group_name == NULL) {
 377                         TRACE(TRACE_OUT_OF_MEM, "Allocation of default "
 378                                 "group name failed (tgt %s)", target_name);
 379                         rc = -ENOMEM;
 380                         goto out_unlock_resume;
 381                 }
 382                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
 383                         target_name);
 384 #endif
 385
 386                 tgt->tgt_name = kmalloc(strlen(target_name) + 1, GFP_KERNEL);
 387                 if (tgt->tgt_name == NULL) {
 388                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name %s failed",
 389                                 target_name);
 390                         rc = -ENOMEM;
 391 #ifdef CONFIG_SCST_PROC
 392                         goto out_free_def_name;
 393 #else
 394                         goto out_unlock_resume;
 395 #endif
 396                 }
 397                 strcpy(tgt->tgt_name, target_name);
 398         } else {
 399                 static int tgt_num; /* protected by scst_mutex */
 400                 int len = strlen(vtt->name) +
 401                         strlen(SCST_DEFAULT_TGT_NAME_SUFFIX) + 11 + 1;
 402
 403                 tgt->tgt_name = kmalloc(len, GFP_KERNEL);
 404                 if (tgt->tgt_name == NULL) {
 405                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name failed "
 406                                 "(template name %s)", vtt->name);
 407                         rc = -ENOMEM;
 408 #ifdef CONFIG_SCST_PROC
 409                         goto out_free_def_name;
 410 #else
 411                         goto out_unlock_resume;
 412 #endif
 413                 }
 414                 sprintf(tgt->tgt_name, "%s%s%d", vtt->name,
 415                         SCST_DEFAULT_TGT_NAME_SUFFIX, tgt_num++);
 416         }
 417
 418         tgt->default_acg = scst_alloc_add_acg(tgt->tgt_name);
 419         if (tgt->default_acg == NULL)
 420                 goto out_free_tgt_name;
 421
 422 #ifdef CONFIG_SCST_PROC
 423         rc = scst_build_proc_target_entries(tgt);
 424         if (rc < 0)
 425                 goto out_free_acg;
 426 #endif
 427
 428         rc = scst_create_tgt_sysfs(tgt);
 429         if (rc < 0)
 430 #ifdef CONFIG_SCST_PROC
 431                 goto out_clean_proc;
 432 #else
 433                 goto out_free_acg;
 434 #endif
 435
 436         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
 437
 438         mutex_unlock(&scst_mutex);
 439         scst_resume_activity();
 440
 441         PRINT_INFO("Target %s (%p) for template %s registered successfully",
 442                 target_name, tgt, vtt->name);
 443
 444 out:
 445         TRACE_EXIT();
 446         return tgt;
 447
 448 #ifdef CONFIG_SCST_PROC
 449 out_clean_proc:
 450         scst_cleanup_proc_target_entries(tgt);
 451 #endif
 452
 453 out_free_acg:
 454         scst_destroy_acg(tgt->default_acg);
 455
 456 out_free_tgt_name:
 457         kfree(tgt->tgt_name);
 458
 459 #ifdef CONFIG_SCST_PROC
 460 out_free_def_name:
 461         kfree(tgt->default_group_name);
 462 #endif
 463
 464 out_unlock_resume:
 465         mutex_unlock(&scst_mutex);
 466
 467 out_resume_free:
 468         scst_resume_activity();
 469
 470 out_free_tgt_err:
 471         scst_tgt_sysfs_put(tgt);
 472         tgt = NULL;
 473
 474 out_err:
 475         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
 476                 target_name, vtt->name, rc);
 477         goto out;
 478 }
 479 EXPORT_SYMBOL(scst_register);
 480
 481 static inline int test_sess_list(struct scst_tgt *tgt)
 482 {
 483         int res;
 484         mutex_lock(&scst_mutex);
 485         res = list_empty(&tgt->sess_list);
 486         mutex_unlock(&scst_mutex);
 487         return res;
 488 }
 489
 490 void scst_unregister(struct scst_tgt *tgt)
 491 {
 492         struct scst_session *sess;
 493         struct scst_tgt_template *vtt = tgt->tgtt;
 494
 495         TRACE_ENTRY();
 496
 497         TRACE_DBG("%s", "Calling target driver's release()");
 498         tgt->tgtt->release(tgt);
 499         TRACE_DBG("%s", "Target driver's release() returned");
 500
 501         mutex_lock(&scst_mutex);
 502 again:
 503         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
 504                 if (sess->shut_phase == SCST_SESS_SPH_READY) {
 505                         /*
 506                          * Sometimes it's hard for target driver to track all
 507                          * its sessions (see scst_local, for example), so let's
 508                          * help it.
 509                          */
 510                         mutex_unlock(&scst_mutex);
 511                         scst_unregister_session(sess, 0, NULL);
 512                         mutex_lock(&scst_mutex);
 513                         goto again;
 514                 }
 515         }
 516         mutex_unlock(&scst_mutex);
 517
 518         TRACE_DBG("%s", "Waiting for sessions shutdown");
 519         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
 520         TRACE_DBG("%s", "wait_event() returned");
 521
 522         scst_suspend_activity(false);
 523         mutex_lock(&scst_mutex);
 524
 525         list_del(&tgt->tgt_list_entry);
 526
 527 #ifdef CONFIG_SCST_PROC
 528         scst_cleanup_proc_target_entries(tgt);
 529 #endif
 530
 531         mutex_unlock(&scst_mutex);
 532         scst_resume_activity();
 533
 534         /*
 535          * It should be before freeing of tgt_name, because acg_name
 536          * points to it.
 537          */
 538         scst_destroy_acg(tgt->default_acg);
 539
 540         kfree(tgt->tgt_name);
 541 #ifdef CONFIG_SCST_PROC
 542         kfree(tgt->default_group_name);
 543 #endif
 544
 545         del_timer_sync(&tgt->retry_timer);
 546
 547         scst_tgt_sysfs_put(tgt);
 548
 549         PRINT_INFO("Target %p for template %s unregistered successfully",
 550                 tgt, vtt->name);
 551
 552         TRACE_EXIT();
 553         return;
 554 }
 555 EXPORT_SYMBOL(scst_unregister);
 556
 557 static int scst_susp_wait(bool interruptible)
 558 {
 559         int res = 0;
 560
 561         TRACE_ENTRY();
 562
 563         if (interruptible) {
 564                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
 565                         (atomic_read(&scst_cmd_count) == 0),
 566                         SCST_SUSPENDING_TIMEOUT);
 567                 if (res <= 0) {
 568                         __scst_resume_activity();
 569                         if (res == 0)
 570                                 res = -EBUSY;
 571                 } else
 572                         res = 0;
 573         } else
 574                 wait_event(scst_dev_cmd_waitQ,
 575                            atomic_read(&scst_cmd_count) == 0);
 576
 577         TRACE_MGMT_DBG("wait_event() returned %d", res);
 578
 579         TRACE_EXIT_RES(res);
 580         return res;
 581 }
 582
 583 int scst_suspend_activity(bool interruptible)
 584 {
 585         int res = 0;
 586         bool rep = false;
 587
 588         TRACE_ENTRY();
 589
 590         if (interruptible) {
 591                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
 592                         res = -EINTR;
 593                         goto out;
 594                 }
 595         } else
 596                 mutex_lock(&scst_suspend_mutex);
 597
 598         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
 599         suspend_count++;
 600         if (suspend_count > 1)
 601                 goto out_up;
 602
 603         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
 604         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 605         /*
 606          * Assignment of SCST_FLAG_SUSPENDING and SCST_FLAG_SUSPENDED must be
 607          * ordered with scst_cmd_count. Otherwise lockless logic in
 608          * scst_translate_lun() and scst_mgmt_translate_lun() won't work.
 609          */
 610         smp_mb__after_set_bit();
 611
 612         /*
 613          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
 614          * information about scst_user behavior.
 615          *
 616          * ToDo: make the global suspending unneeded (switch to per-device
 617          * reference counting? That would mean to switch off from lockless
 618          * implementation of scst_translate_lun().. )
 619          */
 620
 621         if (atomic_read(&scst_cmd_count) != 0) {
 622                 PRINT_INFO("Waiting for %d active commands to complete... This "
 623                         "might take few minutes for disks or few hours for "
 624                         "tapes, if you use long executed commands, like "
 625                         "REWIND or FORMAT. In case, if you have a hung user "
 626                         "space device (i.e. made using scst_user module) not "
 627                         "responding to any commands, if might take virtually "
 628                         "forever until the corresponding user space "
 629                         "program recovers and starts responding or gets "
 630                         "killed.", atomic_read(&scst_cmd_count));
 631                 rep = true;
 632         }
 633
 634         res = scst_susp_wait(interruptible);
 635         if (res != 0)
 636                 goto out_clear;
 637
 638         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
 639         /* See comment about smp_mb() above */
 640         smp_mb__after_clear_bit();
 641
 642         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
 643                 atomic_read(&scst_cmd_count));
 644
 645         res = scst_susp_wait(interruptible);
 646         if (res != 0)
 647                 goto out_clear;
 648
 649         if (rep)
 650                 PRINT_INFO("%s", "All active commands completed");
 651
 652 out_up:
 653         mutex_unlock(&scst_suspend_mutex);
 654
 655 out:
 656         TRACE_EXIT_RES(res);
 657         return res;
 658
 659 out_clear:
 660         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
 661         /* See comment about smp_mb() above */
 662         smp_mb__after_clear_bit();
 663         goto out_up;
 664 }
 665 EXPORT_SYMBOL(scst_suspend_activity);
 666
 667 static void __scst_resume_activity(void)
 668 {
 669         struct scst_cmd_lists *l;
 670
 671         TRACE_ENTRY();
 672
 673         suspend_count--;
 674         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
 675         if (suspend_count > 0)
 676                 goto out;
 677
 678         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 679         /*
 680          * The barrier is needed to make sure all woken up threads see the
 681          * cleared flag. Not sure if it's really needed, but let's be safe.
 682          */
 683         smp_mb__after_clear_bit();
 684
 685         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
 686                 wake_up_all(&l->cmd_list_waitQ);
 687         }
 688         wake_up_all(&scst_init_cmd_list_waitQ);
 689
 690         spin_lock_irq(&scst_mcmd_lock);
 691         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
 692                 struct scst_mgmt_cmd *m;
 693                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
 694                                 mgmt_cmd_list_entry);
 695                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
 696                         "mgmt cmd list", m);
 697                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
 698         }
 699         spin_unlock_irq(&scst_mcmd_lock);
 700         wake_up_all(&scst_mgmt_cmd_list_waitQ);
 701
 702 out:
 703         TRACE_EXIT();
 704         return;
 705 }
 706
 707 void scst_resume_activity(void)
 708 {
 709         TRACE_ENTRY();
 710
 711         mutex_lock(&scst_suspend_mutex);
 712         __scst_resume_activity();
 713         mutex_unlock(&scst_suspend_mutex);
 714
 715         TRACE_EXIT();
 716         return;
 717 }
 718 EXPORT_SYMBOL(scst_resume_activity);
 719
 720 static int scst_register_device(struct scsi_device *scsidp)
 721 {
 722         int res = 0;
 723         struct scst_device *dev;
 724         struct scst_dev_type *dt;
 725
 726         TRACE_ENTRY();
 727
 728         res = scst_suspend_activity(true);
 729         if (res != 0)
 730                 goto out_err;
 731
 732         if (mutex_lock_interruptible(&scst_mutex) != 0) {
 733                 res = -EINTR;
 734                 goto out_resume;
 735         }
 736
 737         res = scst_alloc_device(GFP_KERNEL, &dev);
 738         if (res != 0)
 739                 goto out_up;
 740
 741         dev->type = scsidp->type;
 742
 743         dev->virt_name = kmalloc(50, GFP_KERNEL);
 744         if (dev->virt_name == NULL) {
 745                 PRINT_ERROR("%s", "Unable to alloc device name");
 746                 res = -ENOMEM;
 747                 goto out_free_dev;
 748         }
 749         snprintf(dev->virt_name, 50, "%d:%d:%d:%d", scsidp->host->host_no,
 750                 scsidp->channel, scsidp->id, scsidp->lun);
 751
 752 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
 753         dev->rq_disk = alloc_disk(1);
 754         if (dev->rq_disk == NULL) {
 755                 PRINT_ERROR("Unable to alloc disk object for device %s",
 756                         dev->virt_name);
 757                 res = -ENOMEM;
 758                 /* virt_name will be freed in scst_free_dev() */
 759                 goto out_free_dev;
 760         }
 761         dev->rq_disk->major = SCST_MAJOR;
 762 #endif
 763
 764         dev->scsi_dev = scsidp;
 765
 766         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 767
 768         res = scst_create_device_sysfs(dev);
 769         if (res != 0)
 770                 goto out_free;
 771
 772         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 773                 if (dt->type == scsidp->type) {
 774                         res = scst_assign_dev_handler(dev, dt);
 775                         if (res != 0)
 776                                 goto out_free;
 777                         break;
 778                 }
 779         }
 780
 781 out_up:
 782         mutex_unlock(&scst_mutex);
 783
 784 out_resume:
 785         scst_resume_activity();
 786
 787 out_err:
 788         if (res == 0) {
 789                 PRINT_INFO("Attached to scsi%d, channel %d, id %d, lun %d, "
 790                         "type %d", scsidp->host->host_no, scsidp->channel,
 791                         scsidp->id, scsidp->lun, scsidp->type);
 792         } else {
 793                 PRINT_ERROR("Failed to attach to scsi%d, channel %d, id %d, "
 794                         "lun %d, type %d", scsidp->host->host_no,
 795                         scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
 796         }
 797
 798         TRACE_EXIT_RES(res);
 799         return res;
 800
 801 out_free:
 802         list_del(&dev->dev_list_entry);
 803 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
 804         put_disk(dev->rq_disk);
 805 #endif
 806
 807 out_free_dev:
 808         scst_device_sysfs_put(dev);
 809         goto out_up;
 810 }
 811
 812 static void scst_unregister_device(struct scsi_device *scsidp)
 813 {
 814         struct scst_device *d, *dev = NULL;
 815         struct scst_acg_dev *acg_dev, *aa;
 816
 817         TRACE_ENTRY();
 818
 819         scst_suspend_activity(false);
 820         mutex_lock(&scst_mutex);
 821
 822         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 823                 if (d->scsi_dev == scsidp) {
 824                         dev = d;
 825                         TRACE_DBG("Target device %p found", dev);
 826                         break;
 827                 }
 828         }
 829         if (dev == NULL) {
 830                 PRINT_ERROR("%s", "Target device not found");
 831                 goto out_unblock;
 832         }
 833
 834         list_del(&dev->dev_list_entry);
 835
 836         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
 837                                  dev_acg_dev_list_entry) {
 838                 scst_acg_remove_dev(acg_dev->acg, dev, true);
 839         }
 840
 841         scst_assign_dev_handler(dev, &scst_null_devtype);
 842
 843 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
 844         put_disk(dev->rq_disk);
 845 #endif
 846
 847         scst_device_sysfs_put(dev);
 848
 849         PRINT_INFO("Detached from scsi%d, channel %d, id %d, lun %d, type %d",
 850                 scsidp->host->host_no, scsidp->channel, scsidp->id,
 851                 scsidp->lun, scsidp->type);
 852
 853 out_unblock:
 854         mutex_unlock(&scst_mutex);
 855         scst_resume_activity();
 856
 857         TRACE_EXIT();
 858         return;
 859 }
 860
 861 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
 862 {
 863         int res = 0;
 864
 865         if (dev_handler->parse == NULL) {
 866                 PRINT_ERROR("scst dev handler %s must have "
 867                         "parse() method.", dev_handler->name);
 868                 res = -EINVAL;
 869                 goto out;
 870         }
 871
 872         if (dev_handler->exec == NULL) {
 873 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
 874                 dev_handler->exec_atomic = 1;
 875 #else
 876                 dev_handler->exec_atomic = 0;
 877 #endif
 878         }
 879
 880         if (dev_handler->dev_done == NULL)
 881                 dev_handler->dev_done_atomic = 1;
 882
 883 out:
 884         TRACE_EXIT_RES(res);
 885         return res;
 886 }
 887
 888 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
 889         const char *dev_name)
 890 {
 891         int res, rc;
 892         struct scst_device *dev = NULL;
 893
 894         TRACE_ENTRY();
 895
 896         if (dev_handler == NULL) {
 897                 PRINT_ERROR("%s: valid device handler must be supplied",
 898                             __func__);
 899                 res = -EINVAL;
 900                 goto out;
 901         }
 902
 903         if (dev_name == NULL) {
 904                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
 905                 res = -EINVAL;
 906                 goto out;
 907         }
 908
 909         res = scst_dev_handler_check(dev_handler);
 910         if (res != 0)
 911                 goto out;
 912
 913         res = scst_suspend_activity(true);
 914         if (res != 0)
 915                 goto out;
 916
 917         if (mutex_lock_interruptible(&scst_mutex) != 0) {
 918                 res = -EINTR;
 919                 goto out_resume;
 920         }
 921
 922         res = scst_alloc_device(GFP_KERNEL, &dev);
 923         if (res != 0)
 924                 goto out_up;
 925
 926         dev->type = dev_handler->type;
 927         dev->scsi_dev = NULL;
 928         dev->virt_name = kstrdup(dev_name, GFP_KERNEL);
 929         if (dev->virt_name == NULL) {
 930                 PRINT_ERROR("Unable to allocate virt_name for dev %s",
 931                         dev_name);
 932                 res = -ENOMEM;
 933                 goto out_release;
 934         }
 935         dev->virt_id = scst_virt_dev_last_id++;
 936
 937         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 938
 939         res = dev->virt_id;
 940
 941         rc = scst_create_device_sysfs(dev);
 942         if (rc != 0) {
 943                 res = rc;
 944                 goto out_free_del;
 945         }
 946
 947         rc = scst_assign_dev_handler(dev, dev_handler);
 948         if (rc != 0) {
 949                 res = rc;
 950                 goto out_free_del;
 951         }
 952
 953 out_up:
 954         mutex_unlock(&scst_mutex);
 955
 956 out_resume:
 957         scst_resume_activity();
 958
 959 out:
 960         if (res > 0)
 961                 PRINT_INFO("Attached to virtual device %s (id %d)",
 962                         dev_name, dev->virt_id);
 963         else
 964                 PRINT_INFO("Failed to attach to virtual device %s", dev_name);
 965
 966         TRACE_EXIT_RES(res);
 967         return res;
 968
 969 out_free_del:
 970         list_del(&dev->dev_list_entry);
 971
 972 out_release:
 973         scst_device_sysfs_put(dev);
 974         goto out_up;
 975 }
 976 EXPORT_SYMBOL(scst_register_virtual_device);
 977
 978 void scst_unregister_virtual_device(int id)
 979 {
 980         struct scst_device *d, *dev = NULL;
 981         struct scst_acg_dev *acg_dev, *aa;
 982
 983         TRACE_ENTRY();
 984
 985         scst_suspend_activity(false);
 986         mutex_lock(&scst_mutex);
 987
 988         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 989                 if (d->virt_id == id) {
 990                         dev = d;
 991                         TRACE_DBG("Target device %p (id %d) found", dev, id);
 992                         break;
 993                 }
 994         }
 995         if (dev == NULL) {
 996                 PRINT_ERROR("Target virtual device (id %d) not found", id);
 997                 goto out_unblock;
 998         }
 999
1000         list_del(&dev->dev_list_entry);
1001
1002         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
1003                                  dev_acg_dev_list_entry) {
1004                 scst_acg_remove_dev(acg_dev->acg, dev, true);
1005         }
1006
1007         scst_assign_dev_handler(dev, &scst_null_devtype);
1008
1009         PRINT_INFO("Detached from virtual device %s (id %d)",
1010                 dev->virt_name, dev->virt_id);
1011
1012         scst_device_sysfs_put(dev);
1013
1014 out_unblock:
1015         mutex_unlock(&scst_mutex);
1016         scst_resume_activity();
1017
1018         TRACE_EXIT();
1019         return;
1020 }
1021 EXPORT_SYMBOL(scst_unregister_virtual_device);
1022
1023 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
1024         const char *version)
1025 {
1026         struct scst_dev_type *dt;
1027         struct scst_device *dev;
1028         int res;
1029         int exist;
1030
1031         TRACE_ENTRY();
1032
1033         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1034                 PRINT_ERROR("Incorrect version of dev handler %s",
1035                         dev_type->name);
1036                 res = -EINVAL;
1037                 goto out_error;
1038         }
1039
1040         res = scst_dev_handler_check(dev_type);
1041         if (res != 0)
1042                 goto out_error;
1043
1044 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
1045         if (dev_type->exec == NULL) {
1046 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
1047 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
1048                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1049                         "supported. Consider applying on your kernel patch "
1050                         "scst_exec_req_fifo-<kernel-version> or define "
1051                         "CONFIG_SCST_STRICT_SERIALIZING", dev_type->name);
1052 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
1053 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1054                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1055                         "supported. Consider applying on your kernel patch "
1056                         "scst_exec_req_fifo-<kernel-version>", dev_type->name);
1057 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1058                 res = -EINVAL;
1059                 goto out_error;
1060         }
1061 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
1062
1063         res = scst_suspend_activity(true);
1064         if (res != 0)
1065                 goto out_error;
1066
1067         if (mutex_lock_interruptible(&scst_mutex) != 0) {
1068                 res = -EINTR;
1069                 goto out_err_res;
1070         }
1071
1072         exist = 0;
1073         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1074                 if (strcmp(dt->name, dev_type->name) == 0) {
1075                         PRINT_ERROR("Device type handler \"%s\" already "
1076                                 "exist", dt->name);
1077                         exist = 1;
1078                         break;
1079                 }
1080         }
1081         if (exist)
1082                 goto out_up;
1083
1084 #ifdef CONFIG_SCST_PROC
1085         if (!dev_type->no_proc) {
1086                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1087                 if (res < 0)
1088                         goto out_up;
1089         }
1090 #endif
1091
1092         res = scst_create_devt_sysfs(dev_type);
1093         if (res < 0)
1094                 goto out_free;
1095
1096         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
1097
1098         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1099                 if (dev->scsi_dev == NULL || dev->handler != &scst_null_devtype)
1100                         continue;
1101                 if (dev->scsi_dev->type == dev_type->type)
1102                         scst_assign_dev_handler(dev, dev_type);
1103         }
1104
1105         mutex_unlock(&scst_mutex);
1106         scst_resume_activity();
1107
1108         if (res == 0) {
1109                 PRINT_INFO("Device handler \"%s\" for type %d registered "
1110                         "successfully", dev_type->name, dev_type->type);
1111         }
1112
1113 out:
1114         TRACE_EXIT_RES(res);
1115         return res;
1116
1117 out_free:
1118 #ifdef CONFIG_SCST_PROC
1119         if (!dev_type->no_proc)
1120                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1121 #endif
1122         scst_devt_sysfs_put(dev_type);
1123
1124 out_up:
1125         mutex_unlock(&scst_mutex);
1126
1127 out_err_res:
1128         scst_resume_activity();
1129
1130 out_error:
1131         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
1132                 dev_type->name, dev_type->type);
1133         goto out;
1134 }
1135 EXPORT_SYMBOL(__scst_register_dev_driver);
1136
1137 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
1138 {
1139         struct scst_device *dev;
1140         struct scst_dev_type *dt;
1141         int found = 0;
1142
1143         TRACE_ENTRY();
1144
1145         scst_suspend_activity(false);
1146         mutex_lock(&scst_mutex);
1147
1148         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1149                 if (strcmp(dt->name, dev_type->name) == 0) {
1150                         found = 1;
1151                         break;
1152                 }
1153         }
1154         if (!found) {
1155                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
1156                         dev_type->name);
1157                 goto out_up;
1158         }
1159
1160         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1161                 if (dev->handler == dev_type) {
1162                         scst_assign_dev_handler(dev, &scst_null_devtype);
1163                         TRACE_DBG("Dev handler removed from device %p", dev);
1164                 }
1165         }
1166
1167         list_del(&dev_type->dev_type_list_entry);
1168
1169         mutex_unlock(&scst_mutex);
1170         scst_resume_activity();
1171
1172 #ifdef CONFIG_SCST_PROC
1173         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1174 #endif
1175
1176         scst_devt_sysfs_put(dev_type);
1177
1178         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
1179                    dev_type->name, dev_type->type);
1180
1181 out:
1182         TRACE_EXIT();
1183         return;
1184
1185 out_up:
1186         mutex_unlock(&scst_mutex);
1187         scst_resume_activity();
1188         goto out;
1189 }
1190 EXPORT_SYMBOL(scst_unregister_dev_driver);
1191
1192 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1193         const char *version)
1194 {
1195         int res;
1196
1197         TRACE_ENTRY();
1198
1199         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1200                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1201                         dev_type->name);
1202                 res = -EINVAL;
1203                 goto out_err;
1204         }
1205
1206         res = scst_dev_handler_check(dev_type);
1207         if (res != 0)
1208                 goto out_err;
1209
1210 #ifdef CONFIG_SCST_PROC
1211         if (!dev_type->no_proc) {
1212                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1213                 if (res < 0)
1214                         goto out_err;
1215         }
1216 #endif
1217
1218         res = scst_create_devt_sysfs(dev_type);
1219         if (res < 0)
1220                 goto out_free;
1221
1222         if (dev_type->type != -1) {
1223                 PRINT_INFO("Virtual device handler %s for type %d "
1224                         "registered successfully", dev_type->name,
1225                         dev_type->type);
1226         } else {
1227                 PRINT_INFO("Virtual device handler \"%s\" registered "
1228                         "successfully", dev_type->name);
1229         }
1230
1231 out:
1232         TRACE_EXIT_RES(res);
1233         return res;
1234
1235 out_free:
1236 #ifdef CONFIG_SCST_PROC
1237         if (!dev_type->no_proc)
1238                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1239 #endif
1240
1241         scst_devt_sysfs_put(dev_type);
1242
1243 out_err:
1244         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1245                 dev_type->name);
1246         goto out;
1247 }
1248 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1249
1250 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1251 {
1252         TRACE_ENTRY();
1253
1254 #ifdef CONFIG_SCST_PROC
1255         if (!dev_type->no_proc)
1256                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1257 #endif
1258
1259         scst_devt_sysfs_put(dev_type);
1260
1261         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1262
1263         TRACE_EXIT();
1264         return;
1265 }
1266 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1267
1268 /* Called under scst_mutex */
1269 int scst_add_dev_threads(struct scst_device *dev, int num)
1270 {
1271         int i, res = 0;
1272         int n = 0;
1273         struct scst_cmd_thread_t *thr;
1274         char nm[12];
1275
1276         TRACE_ENTRY();
1277
1278         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1279                 n++;
1280         }
1281
1282         for (i = 0; i < num; i++) {
1283                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1284                 if (!thr) {
1285                         res = -ENOMEM;
1286                         PRINT_ERROR("Failed to allocate thr %d", res);
1287                         goto out_del;
1288                 }
1289                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1290                 nm[ARRAY_SIZE(nm)-1] = '\0';
1291                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1292                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1293                 if (IS_ERR(thr->cmd_thread)) {
1294                         res = PTR_ERR(thr->cmd_thread);
1295                         PRINT_ERROR("kthread_create() failed: %d", res);
1296                         kfree(thr);
1297                         goto out_del;
1298                 }
1299
1300                 list_add(&thr->thread_list_entry, &dev->threads_list);
1301
1302                 /*
1303                  * ToDo: better to use tgt_dev_io_context instead, but we
1304                  * are not ready for that yet.
1305                  */
1306                 __exit_io_context(thr->cmd_thread->io_context);
1307                 thr->cmd_thread->io_context = ioc_task_link(dev->dev_io_ctx);
1308                 TRACE_DBG("Setting dev io ctx %p on thr %d", dev->dev_io_ctx,
1309                         thr->cmd_thread->pid);
1310
1311                 wake_up_process(thr->cmd_thread);
1312         }
1313
1314 out:
1315         TRACE_EXIT_RES(res);
1316         return res;
1317
1318 out_del:
1319         scst_del_dev_threads(dev, i);
1320         goto out;
1321 }
1322
1323 /* Called under scst_mutex and suspended activity */
1324 static int scst_create_dev_threads(struct scst_device *dev)
1325 {
1326         int res = 0;
1327         int threads_num;
1328
1329         TRACE_ENTRY();
1330
1331         if (dev->handler->threads_num <= 0)
1332                 goto out;
1333
1334         threads_num = dev->handler->threads_num;
1335
1336         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1337         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1338         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1339
1340         res = scst_add_dev_threads(dev, threads_num);
1341         if (res != 0)
1342                 goto out;
1343
1344         mutex_lock(&scst_suspend_mutex);
1345         list_add_tail(&dev->cmd_lists.lists_list_entry,
1346                 &scst_cmd_lists_list);
1347         mutex_unlock(&scst_suspend_mutex);
1348
1349         dev->p_cmd_lists = &dev->cmd_lists;
1350
1351 out:
1352         TRACE_EXIT_RES(res);
1353         return res;
1354 }
1355
1356 /* Called under scst_mutex */
1357 void scst_del_dev_threads(struct scst_device *dev, int num)
1358 {
1359         struct scst_cmd_thread_t *ct, *tmp;
1360         int i = 0;
1361
1362         TRACE_ENTRY();
1363
1364         if (num == 0)
1365                 goto out;
1366
1367         list_for_each_entry_safe_reverse(ct, tmp, &dev->threads_list,
1368                                 thread_list_entry) {
1369                 int rc;
1370                 struct scst_tgt_dev *tgt_dev;
1371
1372                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1373                                 dev_tgt_dev_list_entry) {
1374                         struct scst_thr_data_hdr *td;
1375                         td = __scst_find_thr_data(tgt_dev, ct->cmd_thread);
1376                         if (td != NULL) {
1377                                 scst_thr_data_put(td);
1378                                 break;
1379                         }
1380                 }
1381
1382                 rc = kthread_stop(ct->cmd_thread);
1383                 if (rc < 0)
1384                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1385
1386                 list_del(&ct->thread_list_entry);
1387                 kfree(ct);
1388
1389                 if ((num > 0) && (++i >= num))
1390                         break;
1391         }
1392
1393 out:
1394         TRACE_EXIT();
1395         return;
1396 }
1397
1398 /* Called under scst_mutex and suspended activity */
1399 static void scst_stop_dev_threads(struct scst_device *dev)
1400 {
1401         TRACE_ENTRY();
1402
1403         if (list_empty(&dev->threads_list))
1404                 goto out;
1405
1406         scst_del_dev_threads(dev, -1);
1407
1408         if (dev->p_cmd_lists == &dev->cmd_lists) {
1409                 mutex_lock(&scst_suspend_mutex);
1410                 list_del(&dev->cmd_lists.lists_list_entry);
1411                 mutex_unlock(&scst_suspend_mutex);
1412         }
1413
1414 out:
1415         TRACE_EXIT();
1416         return;
1417 }
1418
1419 /* The activity supposed to be suspended and scst_mutex held */
1420 int scst_assign_dev_handler(struct scst_device *dev,
1421         struct scst_dev_type *handler)
1422 {
1423         int res = 0;
1424         struct scst_tgt_dev *tgt_dev;
1425         LIST_HEAD(attached_tgt_devs);
1426
1427         TRACE_ENTRY();
1428
1429         sBUG_ON(handler == NULL);
1430
1431         if (dev->handler == handler)
1432                 goto out;
1433
1434         if (dev->handler == NULL)
1435                 goto assign;
1436
1437         if (dev->handler->detach_tgt) {
1438                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1439                                 dev_tgt_dev_list_entry) {
1440                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1441                                 tgt_dev);
1442                         dev->handler->detach_tgt(tgt_dev);
1443                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1444                 }
1445         }
1446
1447         if (dev->handler->detach) {
1448                 TRACE_DBG("%s", "Calling dev handler's detach()");
1449                 dev->handler->detach(dev);
1450                 TRACE_DBG("%s", "Old handler's detach() returned");
1451         }
1452
1453         scst_stop_dev_threads(dev);
1454
1455         scst_devt_dev_sysfs_put(dev);
1456
1457 assign:
1458         dev->handler = handler;
1459
1460         if (handler == NULL)
1461                 goto out;
1462
1463         res = scst_create_devt_dev_sysfs(dev);
1464         if (res != 0)
1465                 goto out_null;
1466
1467         res = scst_create_dev_threads(dev);
1468         if (res != 0)
1469                 goto out_remove_sysfs;
1470
1471         if (handler->attach) {
1472                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1473                 res = handler->attach(dev);
1474                 TRACE_DBG("New dev handler's attach() returned %d", res);
1475                 if (res != 0) {
1476                         PRINT_ERROR("New device handler's %s attach() "
1477                                 "failed: %d", handler->name, res);
1478                         goto out_thr_null;
1479                 }
1480         }
1481
1482         if (handler->attach_tgt) {
1483                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1484                                 dev_tgt_dev_list_entry) {
1485                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1486                                 tgt_dev);
1487                         res = handler->attach_tgt(tgt_dev);
1488                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1489                         if (res != 0) {
1490                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1491                                     "failed: %d", handler->name, res);
1492                                 goto out_err_detach_tgt;
1493                         }
1494                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1495                                 &attached_tgt_devs);
1496                 }
1497         }
1498
1499 out:
1500         TRACE_EXIT_RES(res);
1501         return res;
1502
1503 out_err_detach_tgt:
1504         if (handler && handler->detach_tgt) {
1505                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1506                                  extra_tgt_dev_list_entry) {
1507                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1508                                 tgt_dev);
1509                         handler->detach_tgt(tgt_dev);
1510                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1511                 }
1512         }
1513         if (handler && handler->detach) {
1514                 TRACE_DBG("%s", "Calling handler's detach()");
1515                 handler->detach(dev);
1516                 TRACE_DBG("%s", "Handler's detach() returned");
1517         }
1518
1519 out_thr_null:
1520         scst_stop_dev_threads(dev);
1521
1522 out_remove_sysfs:
1523         scst_devt_dev_sysfs_put(dev);
1524
1525 out_null:
1526         dev->handler = &scst_null_devtype;
1527         goto out;
1528 }
1529
1530 int scst_global_threads_count(void)
1531 {
1532         int i;
1533
1534         /*
1535          * Just to lower the race window, when user can get just changed value
1536          */
1537         mutex_lock(&scst_global_threads_mutex);
1538         i = scst_nr_global_threads;
1539         mutex_unlock(&scst_global_threads_mutex);
1540         return i;
1541 }
1542
1543 static void scst_threads_info_init(void)
1544 {
1545         mutex_init(&scst_global_threads_mutex);
1546         INIT_LIST_HEAD(&scst_global_threads_list);
1547 }
1548
1549 /* scst_global_threads_mutex supposed to be held */
1550 void __scst_del_global_threads(int num)
1551 {
1552         struct scst_cmd_thread_t *ct, *tmp;
1553
1554         TRACE_ENTRY();
1555
1556         if (num == 0)
1557                 goto out;
1558
1559         list_for_each_entry_safe(ct, tmp, &scst_global_threads_list,
1560                                 thread_list_entry) {
1561                 int res;
1562
1563                 res = kthread_stop(ct->cmd_thread);
1564                 if (res < 0)
1565                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1566                 list_del(&ct->thread_list_entry);
1567                 kfree(ct);
1568                 scst_nr_global_threads--;
1569                 --num;
1570                 if (num == 0)
1571                         break;
1572         }
1573
1574 out:
1575         TRACE_EXIT();
1576         return;
1577 }
1578
1579 /* scst_global_threads_mutex supposed to be held */
1580 int __scst_add_global_threads(int num)
1581 {
1582         int res = 0, i;
1583         static int scst_thread_num;
1584
1585         TRACE_ENTRY();
1586
1587         for (i = 0; i < num; i++) {
1588                 struct scst_cmd_thread_t *thr;
1589
1590                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1591                 if (!thr) {
1592                         res = -ENOMEM;
1593                         PRINT_ERROR("fail to allocate thr %d", res);
1594                         goto out_error;
1595                 }
1596                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1597                         &scst_main_cmd_lists, "scsi_tgt%d",
1598                         scst_thread_num++);
1599                 if (IS_ERR(thr->cmd_thread)) {
1600                         res = PTR_ERR(thr->cmd_thread);
1601                         PRINT_ERROR("kthread_create() failed: %d", res);
1602                         kfree(thr);
1603                         goto out_error;
1604                 }
1605
1606                 list_add(&thr->thread_list_entry, &scst_global_threads_list);
1607                 scst_nr_global_threads++;
1608
1609                 wake_up_process(thr->cmd_thread);
1610         }
1611         res = 0;
1612
1613 out:
1614         TRACE_EXIT_RES(res);
1615         return res;
1616
1617 out_error:
1618         __scst_del_global_threads(i);
1619         goto out;
1620 }
1621
1622 int scst_add_global_threads(int num)
1623 {
1624         int res;
1625
1626         TRACE_ENTRY();
1627
1628         mutex_lock(&scst_global_threads_mutex);
1629         res = __scst_add_global_threads(num);
1630         mutex_unlock(&scst_global_threads_mutex);
1631
1632         TRACE_EXIT_RES(res);
1633         return res;
1634 }
1635 EXPORT_SYMBOL(scst_add_global_threads);
1636
1637 void scst_del_global_threads(int num)
1638 {
1639         TRACE_ENTRY();
1640
1641         mutex_lock(&scst_global_threads_mutex);
1642         __scst_del_global_threads(num);
1643         mutex_unlock(&scst_global_threads_mutex);
1644
1645         TRACE_EXIT();
1646         return;
1647 }
1648 EXPORT_SYMBOL(scst_del_global_threads);
1649
1650 static void scst_stop_all_threads(void)
1651 {
1652         TRACE_ENTRY();
1653
1654         mutex_lock(&scst_global_threads_mutex);
1655         __scst_del_global_threads(-1);
1656         if (scst_mgmt_cmd_thread)
1657                 kthread_stop(scst_mgmt_cmd_thread);
1658         if (scst_mgmt_thread)
1659                 kthread_stop(scst_mgmt_thread);
1660         if (scst_init_cmd_thread)
1661                 kthread_stop(scst_init_cmd_thread);
1662         mutex_unlock(&scst_global_threads_mutex);
1663
1664         TRACE_EXIT();
1665         return;
1666 }
1667
1668 static int scst_start_all_threads(int num)
1669 {
1670         int res;
1671
1672         TRACE_ENTRY();
1673
1674         mutex_lock(&scst_global_threads_mutex);
1675         res = __scst_add_global_threads(num);
1676         if (res < 0)
1677                 goto out;
1678
1679         scst_init_cmd_thread = kthread_run(scst_init_thread,
1680                 NULL, "scsi_tgt_init");
1681         if (IS_ERR(scst_init_cmd_thread)) {
1682                 res = PTR_ERR(scst_init_cmd_thread);
1683                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1684                 scst_init_cmd_thread = NULL;
1685                 goto out;
1686         }
1687
1688         scst_mgmt_cmd_thread = kthread_run(scst_tm_thread,
1689                 NULL, "scsi_tm");
1690         if (IS_ERR(scst_mgmt_cmd_thread)) {
1691                 res = PTR_ERR(scst_mgmt_cmd_thread);
1692                 PRINT_ERROR("kthread_create() for TM failed: %d", res);
1693                 scst_mgmt_cmd_thread = NULL;
1694                 goto out;
1695         }
1696
1697         scst_mgmt_thread = kthread_run(scst_global_mgmt_thread,
1698                 NULL, "scsi_tgt_mgmt");
1699         if (IS_ERR(scst_mgmt_thread)) {
1700                 res = PTR_ERR(scst_mgmt_thread);
1701                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1702                 scst_mgmt_thread = NULL;
1703                 goto out;
1704         }
1705
1706 out:
1707         mutex_unlock(&scst_global_threads_mutex);
1708         TRACE_EXIT_RES(res);
1709         return res;
1710 }
1711
1712 void scst_get(void)
1713 {
1714         __scst_get(0);
1715 }
1716 EXPORT_SYMBOL(scst_get);
1717
1718 void scst_put(void)
1719 {
1720         __scst_put();
1721 }
1722 EXPORT_SYMBOL(scst_put);
1723
1724 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1725 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1726 #else
1727 static int scst_add(struct device *cdev, struct class_interface *intf)
1728 #endif
1729 {
1730         struct scsi_device *scsidp;
1731         int res = 0;
1732
1733         TRACE_ENTRY();
1734
1735 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1736         scsidp = to_scsi_device(cdev->dev);
1737 #else
1738         scsidp = to_scsi_device(cdev->parent);
1739 #endif
1740
1741         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1742                 res = scst_register_device(scsidp);
1743
1744         TRACE_EXIT();
1745         return res;
1746 }
1747
1748 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1749 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1750 #else
1751 static void scst_remove(struct device *cdev, struct class_interface *intf)
1752 #endif
1753 {
1754         struct scsi_device *scsidp;
1755
1756         TRACE_ENTRY();
1757
1758 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1759         scsidp = to_scsi_device(cdev->dev);
1760 #else
1761         scsidp = to_scsi_device(cdev->parent);
1762 #endif
1763
1764         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1765                 scst_unregister_device(scsidp);
1766
1767         TRACE_EXIT();
1768         return;
1769 }
1770
1771 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1772 static struct class_interface scst_interface = {
1773         .add = scst_add,
1774         .remove = scst_remove,
1775 };
1776 #else
1777 static struct class_interface scst_interface = {
1778         .add_dev = scst_add,
1779         .remove_dev = scst_remove,
1780 };
1781 #endif
1782
1783 static void __init scst_print_config(void)
1784 {
1785         char buf[128];
1786         int i, j;
1787
1788         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1789         j = i;
1790
1791 #ifdef CONFIG_SCST_STRICT_SERIALIZING
1792         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1793 #endif
1794
1795 #ifdef CONFIG_SCST_EXTRACHECKS
1796         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1797                 (j == i) ? "" : ", ");
1798 #endif
1799
1800 #ifdef CONFIG_SCST_TRACING
1801         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1802                 (j == i) ? "" : ", ");
1803 #endif
1804
1805 #ifdef CONFIG_SCST_DEBUG
1806         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1807                 (j == i) ? "" : ", ");
1808 #endif
1809
1810 #ifdef CONFIG_SCST_DEBUG_TM
1811         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1812                 (j == i) ? "" : ", ");
1813 #endif
1814
1815 #ifdef CONFIG_SCST_DEBUG_RETRY
1816         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1817                 (j == i) ? "" : ", ");
1818 #endif
1819
1820 #ifdef CONFIG_SCST_DEBUG_OOM
1821         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1822                 (j == i) ? "" : ", ");
1823 #endif
1824
1825 #ifdef CONFIG_SCST_DEBUG_SN
1826         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1827                 (j == i) ? "" : ", ");
1828 #endif
1829
1830 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
1831         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1832                 (j == i) ? "" : ", ");
1833 #endif
1834
1835 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1836         i += snprintf(&buf[i], sizeof(buf) - i,
1837                 "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1838                 (j == i) ? "" : ", ");
1839 #endif
1840
1841 #ifdef CONFIG_SCST_STRICT_SECURITY
1842         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1843                 (j == i) ? "" : ", ");
1844 #endif
1845
1846         if (j != i)
1847                 PRINT_INFO("%s", buf);
1848 }
1849
1850 static int __init init_scst(void)
1851 {
1852         int res, i;
1853         int scst_num_cpus;
1854
1855         TRACE_ENTRY();
1856
1857 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1858         {
1859                 struct scsi_request *req;
1860                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1861                         sizeof(req->sr_sense_buffer));
1862         }
1863 #else
1864         {
1865                 struct scsi_sense_hdr *shdr;
1866                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1867         }
1868 #endif
1869         {
1870                 struct scst_tgt_dev *t;
1871                 struct scst_cmd *c;
1872                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1873                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1874         }
1875
1876 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
1877 #if !defined(SCST_IO_CONTEXT)
1878         PRINT_WARNING("%s", "Patch io_context was not applied on "
1879                 "your kernel. SCST will be working with not the best "
1880                 "performance.");
1881 #endif
1882 #else
1883         PRINT_WARNING("%s", "There is no patch io_context for your kernel "
1884                 "version. For performance reasons it is strongly recommended "
1885                 "to upgrade your kernel to version >= 2.6.27.x.");
1886 #endif
1887
1888         mutex_init(&scst_mutex);
1889         INIT_LIST_HEAD(&scst_template_list);
1890         INIT_LIST_HEAD(&scst_dev_list);
1891         INIT_LIST_HEAD(&scst_dev_type_list);
1892         spin_lock_init(&scst_main_lock);
1893         INIT_LIST_HEAD(&scst_acg_list);
1894         spin_lock_init(&scst_init_lock);
1895         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1896         INIT_LIST_HEAD(&scst_init_cmd_list);
1897 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1898         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1899 #endif
1900         atomic_set(&scst_cmd_count, 0);
1901         spin_lock_init(&scst_mcmd_lock);
1902         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1903         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1904         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1905         init_waitqueue_head(&scst_mgmt_waitQ);
1906         spin_lock_init(&scst_mgmt_lock);
1907         INIT_LIST_HEAD(&scst_sess_init_list);
1908         INIT_LIST_HEAD(&scst_sess_shut_list);
1909         init_waitqueue_head(&scst_dev_cmd_waitQ);
1910         mutex_init(&scst_suspend_mutex);
1911         INIT_LIST_HEAD(&scst_cmd_lists_list);
1912         scst_virt_dev_last_id = 1;
1913         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1914         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1915         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1916         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1917                 &scst_cmd_lists_list);
1918
1919         res = scst_lib_init();
1920         if (res != 0)
1921                 goto out;
1922
1923         scst_num_cpus = num_online_cpus();
1924
1925         /* ToDo: register_cpu_notifier() */
1926
1927         if (scst_threads == 0)
1928                 scst_threads = scst_num_cpus;
1929
1930         if (scst_threads < 1) {
1931                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1932                 scst_threads = scst_num_cpus;
1933         }
1934
1935         scst_threads_info_init();
1936
1937 #define INIT_CACHEP(p, s, o) do {                                       \
1938                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1939                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1940                           sizeof(struct s));                            \
1941                 if (p == NULL) {                                        \
1942                         res = -ENOMEM;                                  \
1943                         goto o;                                         \
1944                 }                                                       \
1945         } while (0)
1946
1947         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit);
1948         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1949                         out_destroy_mgmt_cache);
1950         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1951                         out_destroy_mgmt_stub_cache);
1952         {
1953                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1954                 INIT_CACHEP(scst_sense_cachep, scst_sense,
1955                             out_destroy_ua_cache);
1956         }
1957         INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
1958         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
1959         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1960         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1961         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1962
1963         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1964                 mempool_free_slab, scst_mgmt_cachep);
1965         if (scst_mgmt_mempool == NULL) {
1966                 res = -ENOMEM;
1967                 goto out_destroy_acg_cache;
1968         }
1969
1970         /*
1971          * All mgmt stubs, UAs and sense buffers are bursty and loosing them
1972          * may have fatal consequences, so let's have big pools for them.
1973          */
1974
1975         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1976                 mempool_free_slab, scst_mgmt_stub_cachep);
1977         if (scst_mgmt_stub_mempool == NULL) {
1978                 res = -ENOMEM;
1979                 goto out_destroy_mgmt_mempool;
1980         }
1981
1982         scst_ua_mempool = mempool_create(512, mempool_alloc_slab,
1983                 mempool_free_slab, scst_ua_cachep);
1984         if (scst_ua_mempool == NULL) {
1985                 res = -ENOMEM;
1986                 goto out_destroy_mgmt_stub_mempool;
1987         }
1988
1989         scst_sense_mempool = mempool_create(1024, mempool_alloc_slab,
1990                 mempool_free_slab, scst_sense_cachep);
1991         if (scst_sense_mempool == NULL) {
1992                 res = -ENOMEM;
1993                 goto out_destroy_ua_mempool;
1994         }
1995
1996         scst_aen_mempool = mempool_create(100, mempool_alloc_slab,
1997                 mempool_free_slab, scst_aen_cachep);
1998         if (scst_aen_mempool == NULL) {
1999                 res = -ENOMEM;
2000                 goto out_destroy_sense_mempool;
2001         }
2002
2003         res = scst_sysfs_init();
2004         if (res != 0)
2005                 goto out_destroy_aen_mempool;
2006
2007         if (scst_max_cmd_mem == 0) {
2008                 struct sysinfo si;
2009                 si_meminfo(&si);
2010 #if BITS_PER_LONG == 32
2011                 scst_max_cmd_mem = min(
2012                         (((uint64_t)(si.totalram - si.totalhigh) << PAGE_SHIFT)
2013                                 >> 20) >> 2, (uint64_t)1 << 30);
2014 #else
2015                 scst_max_cmd_mem = (((si.totalram - si.totalhigh) << PAGE_SHIFT)
2016                                         >> 20) >> 2;
2017 #endif
2018         }
2019
2020         if (scst_max_dev_cmd_mem != 0) {
2021                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
2022                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
2023                                 "scst_max_cmd_mem (%d)",
2024                                 scst_max_dev_cmd_mem,
2025                                 scst_max_cmd_mem);
2026                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
2027                 }
2028         } else
2029                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
2030
2031         res = scst_sgv_pools_init(
2032                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
2033         if (res != 0)
2034                 goto out_sysfs_cleanup;
2035
2036 #ifdef CONFIG_SCST_PROC
2037         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
2038         if (scst_default_acg == NULL) {
2039                 res = -ENOMEM;
2040                 goto out_destroy_sgv_pool;
2041         }
2042 #endif
2043
2044         res = scsi_register_interface(&scst_interface);
2045         if (res != 0)
2046 #ifdef CONFIG_SCST_PROC
2047                 goto out_free_acg;
2048 #else
2049                 goto out_destroy_sgv_pool;
2050 #endif
2051
2052         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
2053                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
2054                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
2055                 tasklet_init(&scst_tasklets[i].tasklet,
2056                              (void *)scst_cmd_tasklet,
2057                              (unsigned long)&scst_tasklets[i]);
2058         }
2059
2060         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
2061                 scst_threads);
2062
2063         res = scst_start_all_threads(scst_threads);
2064         if (res < 0)
2065                 goto out_thread_free;
2066
2067 #ifdef CONFIG_SCST_PROC
2068         res = scst_proc_init_module();
2069         if (res != 0)
2070                 goto out_thread_free;
2071 #endif
2072
2073         PRINT_INFO("SCST version %s loaded successfully (max mem for "
2074                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
2075                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
2076
2077         scst_print_config();
2078
2079 out:
2080         TRACE_EXIT_RES(res);
2081         return res;
2082
2083 out_thread_free:
2084         scst_stop_all_threads();
2085
2086         scsi_unregister_interface(&scst_interface);
2087
2088 #ifdef CONFIG_SCST_PROC
2089 out_free_acg:
2090         scst_destroy_acg(scst_default_acg);
2091 #endif
2092
2093 out_destroy_sgv_pool:
2094         scst_sgv_pools_deinit();
2095
2096 out_sysfs_cleanup:
2097         scst_sysfs_cleanup();
2098
2099 out_destroy_aen_mempool:
2100         mempool_destroy(scst_aen_mempool);
2101
2102 out_destroy_sense_mempool:
2103         mempool_destroy(scst_sense_mempool);
2104
2105 out_destroy_ua_mempool:
2106         mempool_destroy(scst_ua_mempool);
2107
2108 out_destroy_mgmt_stub_mempool:
2109         mempool_destroy(scst_mgmt_stub_mempool);
2110
2111 out_destroy_mgmt_mempool:
2112         mempool_destroy(scst_mgmt_mempool);
2113
2114 out_destroy_acg_cache:
2115         kmem_cache_destroy(scst_acgd_cachep);
2116
2117 out_destroy_tgt_cache:
2118         kmem_cache_destroy(scst_tgtd_cachep);
2119
2120 out_destroy_sess_cache:
2121         kmem_cache_destroy(scst_sess_cachep);
2122
2123 out_destroy_cmd_cache:
2124         kmem_cache_destroy(scst_cmd_cachep);
2125
2126 out_destroy_aen_cache:
2127         kmem_cache_destroy(scst_aen_cachep);
2128
2129 out_destroy_sense_cache:
2130         kmem_cache_destroy(scst_sense_cachep);
2131
2132 out_destroy_ua_cache:
2133         kmem_cache_destroy(scst_ua_cachep);
2134
2135 out_destroy_mgmt_stub_cache:
2136         kmem_cache_destroy(scst_mgmt_stub_cachep);
2137
2138 out_destroy_mgmt_cache:
2139         kmem_cache_destroy(scst_mgmt_cachep);
2140
2141 out_lib_exit:
2142         scst_lib_exit();
2143         goto out;
2144 }
2145
2146 static void __exit exit_scst(void)
2147 {
2148         TRACE_ENTRY();
2149
2150         /* ToDo: unregister_cpu_notifier() */
2151
2152 #ifdef CONFIG_SCST_PROC
2153         scst_proc_cleanup_module();
2154 #endif
2155         scst_sysfs_cleanup();
2156
2157         scst_stop_all_threads();
2158
2159         scsi_unregister_interface(&scst_interface);
2160 #ifdef CONFIG_SCST_PROC
2161         scst_destroy_acg(scst_default_acg);
2162 #endif
2163
2164         scst_sgv_pools_deinit();
2165
2166 #define DEINIT_CACHEP(p) do {           \
2167                 kmem_cache_destroy(p);  \
2168                 p = NULL;               \
2169         } while (0)
2170
2171         mempool_destroy(scst_mgmt_mempool);
2172         mempool_destroy(scst_mgmt_stub_mempool);
2173         mempool_destroy(scst_ua_mempool);
2174         mempool_destroy(scst_sense_mempool);
2175         mempool_destroy(scst_aen_mempool);
2176
2177         DEINIT_CACHEP(scst_mgmt_cachep);
2178         DEINIT_CACHEP(scst_mgmt_stub_cachep);
2179         DEINIT_CACHEP(scst_ua_cachep);
2180         DEINIT_CACHEP(scst_sense_cachep);
2181         DEINIT_CACHEP(scst_aen_cachep);
2182         DEINIT_CACHEP(scst_cmd_cachep);
2183         DEINIT_CACHEP(scst_sess_cachep);
2184         DEINIT_CACHEP(scst_tgtd_cachep);
2185         DEINIT_CACHEP(scst_acgd_cachep);
2186
2187         scst_lib_exit();
2188
2189         PRINT_INFO("%s", "SCST unloaded");
2190
2191         TRACE_EXIT();
2192         return;
2193 }
2194
2195 module_init(init_scst);
2196 module_exit(exit_scst);
2197
2198 MODULE_AUTHOR("Vladislav Bolkhovitin");
2199 MODULE_LICENSE("GPL");
2200 MODULE_DESCRIPTION("SCSI target core");
2201 MODULE_VERSION(SCST_VERSION_STRING);