scst/src/scst_main.c

   1 /*
   2  *  scst_main.c
   3  *
   4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *                 and Leonid Stoljar
   6  *
   7  *  This program is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU General Public License
   9  *  as published by the Free Software Foundation, version 2
  10  *  of the License.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  *  GNU General Public License for more details.
  16  */
  17
  18 #include <linux/module.h>
  19
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/errno.h>
  23 #include <linux/list.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched.h>
  27 #include <asm/unistd.h>
  28 #include <asm/string.h>
  29 #include <linux/kthread.h>
  30
  31 #include "scsi_tgt.h"
  32 #include "scst_priv.h"
  33 #include "scst_mem.h"
  34
  35 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
  36 #warning HIGHMEM kernel configurations are fully supported, but not \
  37         recommended for performance reasons. Consider change VMSPLIT \
  38         option or use 64-bit configuration instead. See README file for \
  39         details.
  40 #endif
  41
  42 #ifdef SCST_HIGHMEM
  43 #error SCST_HIGHMEM configuration isn't supported and broken, because there \
  44         is no real point to support it, at least it definitely doesn't worth \
  45         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
  46         or in 64-bit configuration instead. See README file for details.
  47 #endif
  48
  49 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
  50 #warning Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
  51         your kernel and STRICT_SERIALIZING isn't defined. Pass-through dev \
  52         handlers will not be supported.
  53 #endif
  54
  55 /*
  56  * All targets, devices and dev_types management is done under this mutex.
  57  *
  58  * It must NOT be used in any works (schedule_work(), etc.), because
  59  * otherwise a deadlock (double lock, actually) is possible, e.g., with
  60  * scst_user detach_tgt(), which is called under scst_mutex and calls
  61  * flush_scheduled_work().
  62  */
  63 DEFINE_MUTEX(scst_mutex);
  64
  65 LIST_HEAD(scst_template_list);
  66 LIST_HEAD(scst_dev_list);
  67 LIST_HEAD(scst_dev_type_list);
  68
  69 spinlock_t scst_main_lock = SPIN_LOCK_UNLOCKED;
  70
  71 struct kmem_cache *scst_mgmt_cachep;
  72 mempool_t *scst_mgmt_mempool;
  73 struct kmem_cache *scst_ua_cachep;
  74 mempool_t *scst_ua_mempool;
  75 struct kmem_cache *scst_tgtd_cachep;
  76 struct kmem_cache *scst_sess_cachep;
  77 struct kmem_cache *scst_acgd_cachep;
  78
  79 LIST_HEAD(scst_acg_list);
  80 struct scst_acg *scst_default_acg;
  81
  82 spinlock_t scst_init_lock = SPIN_LOCK_UNLOCKED;
  83 DECLARE_WAIT_QUEUE_HEAD(scst_init_cmd_list_waitQ);
  84 LIST_HEAD(scst_init_cmd_list);
  85 unsigned int scst_init_poll_cnt;
  86
  87 struct kmem_cache *scst_cmd_cachep;
  88
  89 #if defined(DEBUG) || defined(TRACING)
  90 unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
  91 #endif
  92
  93 unsigned long scst_flags;
  94 atomic_t scst_cmd_count = ATOMIC_INIT(0);
  95
  96 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
  97 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
  98 unsigned long scst_max_cmd_mem;
  99
 100 struct scst_cmd_lists scst_main_cmd_lists;
 101
 102 struct scst_tasklet scst_tasklets[NR_CPUS];
 103
 104
 105 spinlock_t scst_mcmd_lock = SPIN_LOCK_UNLOCKED;
 106 LIST_HEAD(scst_active_mgmt_cmd_list);
 107 LIST_HEAD(scst_delayed_mgmt_cmd_list);
 108 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
 109
 110 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
 111 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
 112 LIST_HEAD(scst_sess_init_list);
 113 LIST_HEAD(scst_sess_shut_list);
 114
 115 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
 116
 117 DEFINE_MUTEX(scst_suspend_mutex);
 118 LIST_HEAD(scst_cmd_lists_list); /* protected by scst_suspend_mutex */
 119
 120 static int scst_threads;
 121 struct scst_threads_info_t scst_threads_info;
 122
 123 static int suspend_count;
 124
 125 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
 126
 127 /*
 128  * This buffer and lock are intended to avoid memory allocation, which
 129  * could fail in improper places.
 130  */
 131 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
 132 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
 133
 134 module_param_named(scst_threads, scst_threads, int, 0);
 135 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
 136
 137 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
 138 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
 139         "the SCST commands at any given time in Mb");
 140
 141 int scst_register_target_template(struct scst_tgt_template *vtt)
 142 {
 143         int res = 0;
 144         struct scst_tgt_template *t;
 145         static DEFINE_MUTEX(m);
 146
 147         TRACE_ENTRY();
 148
 149         INIT_LIST_HEAD(&vtt->tgt_list);
 150
 151         if (!vtt->detect) {
 152                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 153                         "detect() method.", vtt->name);
 154                 res = -EINVAL;
 155                 goto out_err;
 156         }
 157
 158         if (!vtt->release) {
 159                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 160                         "release() method.", vtt->name);
 161                 res = -EINVAL;
 162                 goto out_err;
 163         }
 164
 165         if (!vtt->xmit_response) {
 166                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 167                         "xmit_response() method.", vtt->name);
 168                 res = -EINVAL;
 169                 goto out_err;
 170         }
 171
 172         if (vtt->threads_num < 0) {
 173                 PRINT_ERROR_PR("Wrong threads_num value %d for "
 174                         "target \"%s\"", vtt->threads_num,
 175                         vtt->name);
 176                 res = -EINVAL;
 177                 goto out_err;
 178         }
 179
 180         if (!vtt->no_proc_entry) {
 181                 res = scst_build_proc_target_dir_entries(vtt);
 182                 if (res < 0)
 183                         goto out_err;
 184         }
 185
 186         if (vtt->preprocessing_done == NULL)
 187                 vtt->preprocessing_done_atomic = 1;
 188
 189         if (mutex_lock_interruptible(&m) != 0)
 190                 goto out_err;
 191
 192         if (mutex_lock_interruptible(&scst_mutex) != 0)
 193                 goto out_m_up;
 194         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
 195                 if (strcmp(t->name, vtt->name) == 0) {
 196                         PRINT_ERROR_PR("Target driver %s already registered",
 197                                 vtt->name);
 198                         mutex_unlock(&scst_mutex);
 199                         goto out_cleanup;
 200                 }
 201         }
 202         mutex_unlock(&scst_mutex);
 203
 204         TRACE_DBG("%s", "Calling target driver's detect()");
 205         res = vtt->detect(vtt);
 206         TRACE_DBG("Target driver's detect() returned %d", res);
 207         if (res < 0) {
 208                 PRINT_ERROR_PR("%s", "The detect() routine failed");
 209                 res = -EINVAL;
 210                 goto out_cleanup;
 211         }
 212
 213         mutex_lock(&scst_mutex);
 214         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
 215         mutex_unlock(&scst_mutex);
 216
 217         res = 0;
 218
 219         PRINT_INFO_PR("Target template %s registered successfully", vtt->name);
 220
 221         mutex_unlock(&m);
 222
 223 out:
 224         TRACE_EXIT_RES(res);
 225         return res;
 226
 227 out_m_up:
 228         mutex_unlock(&m);
 229
 230 out_cleanup:
 231         scst_cleanup_proc_target_dir_entries(vtt);
 232
 233 out_err:
 234         PRINT_ERROR_PR("Failed to register target template %s", vtt->name);
 235         goto out;
 236 }
 237
 238 void scst_unregister_target_template(struct scst_tgt_template *vtt)
 239 {
 240         struct scst_tgt *tgt;
 241         struct scst_tgt_template *t;
 242         int found = 0;
 243
 244         TRACE_ENTRY();
 245
 246         mutex_lock(&scst_mutex);
 247
 248         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
 249                 if (strcmp(t->name, vtt->name) == 0) {
 250                         found = 1;
 251                         break;
 252                 }
 253         }
 254         if (!found) {
 255                 PRINT_ERROR_PR("Target driver %s isn't registered", vtt->name);
 256                 goto out_up;
 257         }
 258
 259 restart:
 260         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
 261                 mutex_unlock(&scst_mutex);
 262                 scst_unregister(tgt);
 263                 mutex_lock(&scst_mutex);
 264                 goto restart;
 265         }
 266         list_del(&vtt->scst_template_list_entry);
 267
 268         PRINT_INFO_PR("Target template %s unregistered successfully", vtt->name);
 269
 270 out_up:
 271         mutex_unlock(&scst_mutex);
 272
 273         scst_cleanup_proc_target_dir_entries(vtt);
 274
 275         TRACE_EXIT();
 276         return;
 277 }
 278
 279 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
 280         const char *target_name)
 281 {
 282         struct scst_tgt *tgt;
 283
 284         TRACE_ENTRY();
 285
 286         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
 287         if (tgt == NULL) {
 288                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
 289                 goto out_err;
 290         }
 291
 292         INIT_LIST_HEAD(&tgt->sess_list);
 293         init_waitqueue_head(&tgt->unreg_waitQ);
 294         tgt->tgtt = vtt;
 295         tgt->sg_tablesize = vtt->sg_tablesize;
 296         spin_lock_init(&tgt->tgt_lock);
 297         INIT_LIST_HEAD(&tgt->retry_cmd_list);
 298         atomic_set(&tgt->finished_cmds, 0);
 299         init_timer(&tgt->retry_timer);
 300         tgt->retry_timer.data = (unsigned long)tgt;
 301         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
 302
 303         scst_suspend_activity();
 304         mutex_lock(&scst_mutex);
 305
 306         if (target_name != NULL) {
 307                 int len = strlen(target_name) + 1 +
 308                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
 309
 310                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
 311                 if (tgt->default_group_name == NULL) {
 312                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
 313                                 "group name failed");
 314                         goto out_free_err;
 315                 }
 316                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
 317                         target_name);
 318         }
 319
 320         if (scst_build_proc_target_entries(tgt) < 0)
 321                 goto out_free_name;
 322         else
 323                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
 324
 325         mutex_unlock(&scst_mutex);
 326         scst_resume_activity();
 327
 328         PRINT_INFO_PR("Target %s for template %s registered successfully",
 329                 target_name, vtt->name);
 330
 331 out:
 332         TRACE_EXIT();
 333         return tgt;
 334
 335 out_free_name:
 336         if (tgt->default_group_name)
 337                 kfree(tgt->default_group_name);
 338
 339 out_free_err:
 340         mutex_unlock(&scst_mutex);
 341         scst_resume_activity();
 342
 343         kfree(tgt);
 344         tgt = NULL;
 345
 346 out_err:
 347         PRINT_ERROR_PR("Failed to register target for template %s", vtt->name);
 348         goto out;
 349 }
 350
 351 static inline int test_sess_list(struct scst_tgt *tgt)
 352 {
 353         int res;
 354         mutex_lock(&scst_mutex);
 355         res = list_empty(&tgt->sess_list);
 356         mutex_unlock(&scst_mutex);
 357         return res;
 358 }
 359
 360 void scst_unregister(struct scst_tgt *tgt)
 361 {
 362         struct scst_session *sess;
 363         struct scst_tgt_template *vtt = tgt->tgtt;
 364
 365         TRACE_ENTRY();
 366
 367         TRACE_DBG("%s", "Calling target driver's release()");
 368         tgt->tgtt->release(tgt);
 369         TRACE_DBG("%s", "Target driver's release() returned");
 370
 371         mutex_lock(&scst_mutex);
 372         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
 373                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
 374         }
 375         mutex_unlock(&scst_mutex);
 376
 377         TRACE_DBG("%s", "Waiting for sessions shutdown");
 378         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
 379         TRACE_DBG("%s", "wait_event() returned");
 380
 381         scst_suspend_activity();
 382         mutex_lock(&scst_mutex);
 383
 384         list_del(&tgt->tgt_list_entry);
 385
 386         scst_cleanup_proc_target_entries(tgt);
 387
 388         if (tgt->default_group_name)
 389                 kfree(tgt->default_group_name);
 390
 391         mutex_unlock(&scst_mutex);
 392         scst_resume_activity();
 393
 394         del_timer_sync(&tgt->retry_timer);
 395
 396         kfree(tgt);
 397
 398         PRINT_INFO_PR("Target for template %s unregistered successfully",
 399                 vtt->name);
 400
 401         TRACE_EXIT();
 402         return;
 403 }
 404
 405 void scst_suspend_activity(void)
 406 {
 407         TRACE_ENTRY();
 408
 409         mutex_lock(&scst_suspend_mutex);
 410
 411         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
 412         suspend_count++;
 413         if (suspend_count > 1)
 414                 goto out_up;
 415
 416         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
 417         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 418         smp_mb__after_set_bit();
 419
 420         TRACE_MGMT_DBG("Waiting for %d active commands to complete",
 421               atomic_read(&scst_cmd_count));
 422         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
 423         TRACE_MGMT_DBG("%s", "wait_event() returned");
 424
 425         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
 426         smp_mb__after_clear_bit();
 427
 428         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
 429               atomic_read(&scst_cmd_count));
 430         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
 431         TRACE_MGMT_DBG("%s", "wait_event() returned");
 432
 433 out_up:
 434         mutex_unlock(&scst_suspend_mutex);
 435
 436         TRACE_EXIT();
 437         return;
 438 }
 439
 440 void scst_resume_activity(void)
 441 {
 442         struct scst_cmd_lists *l;
 443
 444         TRACE_ENTRY();
 445
 446         mutex_lock(&scst_suspend_mutex);
 447
 448         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
 449         suspend_count--;
 450         if (suspend_count > 0)
 451                 goto out_up;
 452
 453         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 454         smp_mb__after_clear_bit();
 455
 456         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
 457                 wake_up_all(&l->cmd_list_waitQ);
 458         }
 459         wake_up_all(&scst_init_cmd_list_waitQ);
 460
 461         spin_lock_irq(&scst_mcmd_lock);
 462         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
 463                 struct scst_mgmt_cmd *m;
 464                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
 465                                 mgmt_cmd_list_entry);
 466                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
 467                         "mgmt cmd list", m);
 468                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
 469         }
 470         spin_unlock_irq(&scst_mcmd_lock);
 471         wake_up_all(&scst_mgmt_cmd_list_waitQ);
 472
 473 out_up:
 474         mutex_unlock(&scst_suspend_mutex);
 475
 476         TRACE_EXIT();
 477         return;
 478 }
 479
 480 static int scst_register_device(struct scsi_device *scsidp)
 481 {
 482         int res = 0;
 483         struct scst_device *dev;
 484         struct scst_dev_type *dt;
 485
 486         TRACE_ENTRY();
 487
 488         scst_suspend_activity();
 489         mutex_lock(&scst_mutex);
 490
 491         res = scst_alloc_device(GFP_KERNEL, &dev);
 492         if (res != 0)
 493                 goto out_up;
 494
 495         dev->type = scsidp->type;
 496
 497         dev->rq_disk = alloc_disk(1);
 498         if (dev->rq_disk == NULL) {
 499                 res = -ENOMEM;
 500                 goto out_free_dev;
 501         }
 502         dev->rq_disk->major = SCST_MAJOR;
 503
 504         dev->scsi_dev = scsidp;
 505
 506         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 507
 508         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 509                 if (dt->type == scsidp->type) {
 510                         res = scst_assign_dev_handler(dev, dt);
 511                         if (res != 0)
 512                                 goto out_free;
 513                         break;
 514                 }
 515         }
 516
 517 out_up:
 518         mutex_unlock(&scst_mutex);
 519         scst_resume_activity();
 520
 521         if (res == 0) {
 522                 PRINT_INFO_PR("Attached SCSI target mid-level at "
 523                     "scsi%d, channel %d, id %d, lun %d, type %d",
 524                     scsidp->host->host_no, scsidp->channel, scsidp->id,
 525                     scsidp->lun, scsidp->type);
 526         }
 527         else {
 528                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
 529                     "at scsi%d, channel %d, id %d, lun %d, type %d",
 530                     scsidp->host->host_no, scsidp->channel, scsidp->id,
 531                     scsidp->lun, scsidp->type);
 532         }
 533
 534         TRACE_EXIT_RES(res);
 535         return res;
 536
 537 out_free:
 538         list_del(&dev->dev_list_entry);
 539         put_disk(dev->rq_disk);
 540
 541 out_free_dev:
 542         scst_free_device(dev);
 543         goto out_up;
 544 }
 545
 546 static void scst_unregister_device(struct scsi_device *scsidp)
 547 {
 548         struct scst_device *d, *dev = NULL;
 549         struct scst_acg_dev *acg_dev, *aa;
 550
 551         TRACE_ENTRY();
 552
 553         scst_suspend_activity();
 554         mutex_lock(&scst_mutex);
 555
 556         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 557                 if (d->scsi_dev == scsidp) {
 558                         dev = d;
 559                         TRACE_DBG("Target device %p found", dev);
 560                         break;
 561                 }
 562         }
 563         if (dev == NULL) {
 564                 PRINT_ERROR_PR("%s", "Target device not found");
 565                 goto out_unblock;
 566         }
 567
 568         list_del(&dev->dev_list_entry);
 569
 570         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
 571                                  dev_acg_dev_list_entry)
 572         {
 573                 scst_acg_remove_dev(acg_dev->acg, dev);
 574         }
 575
 576         scst_assign_dev_handler(dev, NULL);
 577
 578         put_disk(dev->rq_disk);
 579         scst_free_device(dev);
 580
 581         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
 582                 "id %d, lun %d, type %d", scsidp->host->host_no,
 583                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
 584
 585 out_unblock:
 586         mutex_unlock(&scst_mutex);
 587         scst_resume_activity();
 588
 589         TRACE_EXIT();
 590         return;
 591 }
 592
 593 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
 594 {
 595         int res = 0;
 596
 597         if (dev_handler->parse == NULL) {
 598                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
 599                         "parse() method.", dev_handler->name);
 600                 res = -EINVAL;
 601                 goto out;
 602         }
 603
 604         if (dev_handler->exec == NULL)
 605                 dev_handler->exec_atomic = 1;
 606
 607         if (dev_handler->dev_done == NULL)
 608                 dev_handler->dev_done_atomic = 1;
 609
 610 out:
 611         TRACE_EXIT_RES(res);
 612         return res;
 613 }
 614
 615 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
 616         const char *dev_name)
 617 {
 618         int res, rc;
 619         struct scst_device *dev = NULL;
 620
 621         TRACE_ENTRY();
 622
 623         if (dev_handler == NULL) {
 624                 PRINT_ERROR_PR("%s: valid device handler must be supplied",
 625                         __FUNCTION__);
 626                 res = -EINVAL;
 627                 goto out;
 628         }
 629
 630         if (dev_name == NULL) {
 631                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
 632                 res = -EINVAL;
 633                 goto out;
 634         }
 635
 636         res = scst_dev_handler_check(dev_handler);
 637         if (res != 0)
 638                 goto out;
 639
 640         scst_suspend_activity();
 641         if (mutex_lock_interruptible(&scst_mutex) != 0) {
 642                 res = -EINTR;
 643                 goto out_resume;
 644         }
 645
 646         res = scst_alloc_device(GFP_KERNEL, &dev);
 647         if (res != 0)
 648                 goto out_up;
 649
 650         dev->type = dev_handler->type;
 651         dev->scsi_dev = NULL;
 652         dev->virt_name = dev_name;
 653         dev->virt_id = scst_virt_dev_last_id++;
 654
 655         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 656
 657         res = dev->virt_id;
 658
 659         rc = scst_assign_dev_handler(dev, dev_handler);
 660         if (rc != 0) {
 661                 res = rc;
 662                 goto out_free_del;
 663         }
 664
 665 out_up:
 666         mutex_unlock(&scst_mutex);
 667
 668 out_resume:
 669         scst_resume_activity();
 670
 671 out:
 672         if (res > 0) {
 673                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
 674                     "device %s (id %d)", dev_name, dev->virt_id);
 675         }
 676         else {
 677                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
 678                     "virtual device %s", dev_name);
 679         }
 680
 681         TRACE_EXIT_RES(res);
 682         return res;
 683
 684 out_free_del:
 685         list_del(&dev->dev_list_entry);
 686         scst_free_device(dev);
 687         goto out_up;
 688 }
 689
 690 void scst_unregister_virtual_device(int id)
 691 {
 692         struct scst_device *d, *dev = NULL;
 693         struct scst_acg_dev *acg_dev, *aa;
 694
 695         TRACE_ENTRY();
 696
 697         scst_suspend_activity();
 698         mutex_lock(&scst_mutex);
 699
 700         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 701                 if (d->virt_id == id) {
 702                         dev = d;
 703                         TRACE_DBG("Target device %p found", dev);
 704                         break;
 705                 }
 706         }
 707         if (dev == NULL) {
 708                 PRINT_ERROR_PR("%s", "Target device not found");
 709                 goto out_unblock;
 710         }
 711
 712         list_del(&dev->dev_list_entry);
 713
 714         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
 715                                  dev_acg_dev_list_entry)
 716         {
 717                 scst_acg_remove_dev(acg_dev->acg, dev);
 718         }
 719
 720         scst_assign_dev_handler(dev, NULL);
 721
 722         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
 723                 "(id %d)", dev->virt_name, dev->virt_id);
 724
 725         scst_free_device(dev);
 726
 727 out_unblock:
 728         mutex_unlock(&scst_mutex);
 729         scst_resume_activity();
 730
 731         TRACE_EXIT();
 732         return;
 733 }
 734
 735 int scst_register_dev_driver(struct scst_dev_type *dev_type)
 736 {
 737         struct scst_dev_type *dt;
 738         struct scst_device *dev;
 739         int res;
 740         int exist;
 741
 742         TRACE_ENTRY();
 743
 744         res = scst_dev_handler_check(dev_type);
 745         if (res != 0)
 746                 goto out_err;
 747
 748 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
 749         if (dev_type->exec == NULL) {
 750                 PRINT_ERROR_PR("Pass-through dev handlers (handler \"%s\") not "
 751                         "supported. Consider applying on your kernel patch "
 752                         "scst_exec_req_fifo-<kernel-version>.patch or define "
 753                         "STRICT_SERIALIZING", dev_type->name);
 754                 res = -EINVAL;
 755                 goto out_err;
 756         }
 757 #endif
 758
 759         scst_suspend_activity();
 760         if (mutex_lock_interruptible(&scst_mutex) != 0) {
 761                 res = -EINTR;
 762                 goto out_err;
 763         }
 764
 765         exist = 0;
 766         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 767                 if (strcmp(dt->name, dev_type->name) == 0) {
 768                         PRINT_ERROR_PR("Device type handler \"%s\" already "
 769                                 "exist", dt->name);
 770                         exist = 1;
 771                         break;
 772                 }
 773         }
 774         if (exist)
 775                 goto out_up;
 776
 777         res = scst_build_proc_dev_handler_dir_entries(dev_type);
 778         if (res < 0) {
 779                 goto out_up;
 780         }
 781
 782         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
 783
 784         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
 785                 if ((dev->scsi_dev == NULL) || (dev->handler != NULL))
 786                         continue;
 787                 if (dev->scsi_dev->type == dev_type->type)
 788                         scst_assign_dev_handler(dev, dev_type);
 789         }
 790
 791         mutex_unlock(&scst_mutex);
 792         scst_resume_activity();
 793
 794         if (res == 0) {
 795                 PRINT_INFO_PR("Device handler \"%s\" for type %d registered "
 796                         "successfully", dev_type->name, dev_type->type);
 797         }
 798
 799 out:
 800         TRACE_EXIT_RES(res);
 801         return res;
 802
 803 out_up:
 804         mutex_unlock(&scst_mutex);
 805
 806 out_err:
 807         scst_resume_activity();
 808         PRINT_ERROR_PR("Failed to register device handler \"%s\" for type %d",
 809                 dev_type->name, dev_type->type);
 810         goto out;
 811 }
 812
 813 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
 814 {
 815         struct scst_device *dev;
 816         struct scst_dev_type *dt;
 817         int found = 0;
 818
 819         TRACE_ENTRY();
 820
 821         scst_suspend_activity();
 822         mutex_lock(&scst_mutex);
 823
 824         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 825                 if (strcmp(dt->name, dev_type->name) == 0) {
 826                         found = 1;
 827                         break;
 828                 }
 829         }
 830         if (!found) {
 831                 PRINT_ERROR_PR("Dev handler \"%s\" isn't registered",
 832                         dev_type->name);
 833                 goto out_up;
 834         }
 835
 836         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
 837                 if (dev->handler == dev_type) {
 838                         scst_assign_dev_handler(dev, NULL);
 839                         TRACE_DBG("Dev handler removed from device %p", dev);
 840                 }
 841         }
 842
 843         list_del(&dev_type->dev_type_list_entry);
 844
 845         mutex_unlock(&scst_mutex);
 846         scst_resume_activity();
 847
 848         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
 849
 850         PRINT_INFO_PR("Device handler \"%s\" for type %d unloaded",
 851                    dev_type->name, dev_type->type);
 852
 853 out:
 854         TRACE_EXIT();
 855         return;
 856
 857 out_up:
 858         mutex_unlock(&scst_mutex);
 859         scst_resume_activity();
 860         goto out;
 861 }
 862
 863 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
 864 {
 865         int res;
 866
 867         TRACE_ENTRY();
 868
 869         res = scst_dev_handler_check(dev_type);
 870         if (res != 0)
 871                 goto out_err;
 872
 873         if (!dev_type->no_proc) {
 874                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
 875                 if (res < 0)
 876                         goto out_err;
 877         }
 878
 879         if (dev_type->type != -1) {
 880                 PRINT_INFO_PR("Virtual device handler %s for type %d "
 881                         "registered successfully", dev_type->name,
 882                         dev_type->type);
 883         } else {
 884                 PRINT_INFO_PR("Virtual device handler \"%s\" registered "
 885                         "successfully", dev_type->name);
 886         }
 887
 888 out:
 889         TRACE_EXIT_RES(res);
 890         return res;
 891
 892 out_err:
 893         PRINT_ERROR_PR("Failed to register virtual device handler \"%s\"",
 894                 dev_type->name);
 895         goto out;
 896 }
 897
 898 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
 899 {
 900         TRACE_ENTRY();
 901
 902         if (!dev_type->no_proc)
 903                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
 904
 905         PRINT_INFO_PR("Device handler \"%s\" unloaded", dev_type->name);
 906
 907         TRACE_EXIT();
 908         return;
 909 }
 910
 911 /* Called under scst_mutex and suspended activity */
 912 int scst_add_dev_threads(struct scst_device *dev, int num)
 913 {
 914         int i, res = 0;
 915         int n = 0;
 916         struct scst_cmd_thread_t *thr;
 917         char nm[12];
 918
 919         TRACE_ENTRY();
 920
 921         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
 922                 n++;
 923         }
 924
 925         for (i = 0; i < num; i++) {
 926                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
 927                 if (!thr) {
 928                         res = -ENOMEM;
 929                         PRINT_ERROR_PR("Failed to allocate thr %d", res);
 930                         goto out;
 931                 }
 932                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
 933                 nm[ARRAY_SIZE(nm)-1] = '\0';
 934                 thr->cmd_thread = kthread_run(scst_cmd_thread,
 935                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
 936                 if (IS_ERR(thr->cmd_thread)) {
 937                         res = PTR_ERR(thr->cmd_thread);
 938                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
 939                         kfree(thr);
 940                         goto out;
 941                 }
 942                 list_add(&thr->thread_list_entry, &dev->threads_list);
 943         }
 944
 945 out:
 946         TRACE_EXIT_RES(res);
 947         return res;
 948 }
 949
 950 /* Called under scst_mutex and suspended activity */
 951 static int scst_create_dev_threads(struct scst_device *dev)
 952 {
 953         int res = 0;
 954         int threads_num;
 955
 956         TRACE_ENTRY();
 957
 958         if (dev->handler->threads_num <= 0)
 959                 goto out;
 960
 961         threads_num = dev->handler->threads_num;
 962
 963         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
 964         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
 965         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
 966
 967         res = scst_add_dev_threads(dev, threads_num);
 968         if (res != 0)
 969                 goto out;
 970
 971         mutex_lock(&scst_suspend_mutex);
 972         list_add_tail(&dev->cmd_lists.lists_list_entry,
 973                 &scst_cmd_lists_list);
 974         mutex_unlock(&scst_suspend_mutex);
 975
 976         dev->p_cmd_lists = &dev->cmd_lists;
 977
 978 out:
 979         TRACE_EXIT_RES(res);
 980         return res;
 981 }
 982
 983 /* Called under scst_mutex and suspended activity */
 984 void scst_del_dev_threads(struct scst_device *dev, int num)
 985 {
 986         struct scst_cmd_thread_t *ct, *tmp;
 987         int i = 0;
 988
 989         TRACE_ENTRY();
 990
 991         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
 992                                 thread_list_entry) {
 993                 int rc = kthread_stop(ct->cmd_thread);
 994                 if (rc < 0) {
 995                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
 996                 }
 997                 list_del(&ct->thread_list_entry);
 998                 kfree(ct);
 999                 if ((num >0) && (++i >= num))
1000                         break;
1001         }
1002
1003         TRACE_EXIT();
1004         return;
1005 }
1006
1007 /* Called under scst_mutex and suspended activity */
1008 static void scst_stop_dev_threads(struct scst_device *dev)
1009 {
1010         TRACE_ENTRY();
1011
1012         if (list_empty(&dev->threads_list))
1013                 goto out;
1014
1015         scst_del_dev_threads(dev, -1);
1016
1017         if (dev->p_cmd_lists == &dev->cmd_lists) {
1018                 mutex_lock(&scst_suspend_mutex);
1019                 list_del(&dev->cmd_lists.lists_list_entry);
1020                 mutex_unlock(&scst_suspend_mutex);
1021         }
1022
1023 out:
1024         TRACE_EXIT();
1025         return;
1026 }
1027
1028 /* The activity supposed to be suspended and scst_mutex held */
1029 int scst_assign_dev_handler(struct scst_device *dev,
1030         struct scst_dev_type *handler)
1031 {
1032         int res = 0;
1033         struct scst_tgt_dev *tgt_dev;
1034         LIST_HEAD(attached_tgt_devs);
1035
1036         TRACE_ENTRY();
1037
1038         if (dev->handler == handler)
1039                 goto out;
1040
1041         if (dev->handler && dev->handler->detach_tgt) {
1042                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1043                                 dev_tgt_dev_list_entry) {
1044                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1045                                 tgt_dev);
1046                         dev->handler->detach_tgt(tgt_dev);
1047                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1048                 }
1049         }
1050
1051         if (dev->handler && dev->handler->detach) {
1052                 TRACE_DBG("%s", "Calling dev handler's detach()");
1053                 dev->handler->detach(dev);
1054                 TRACE_DBG("%s", "Old handler's detach() returned");
1055         }
1056
1057         scst_stop_dev_threads(dev);
1058
1059         dev->handler = handler;
1060
1061         if (handler) {
1062                 res = scst_create_dev_threads(dev);
1063                 if (res != 0)
1064                         goto out_null;
1065         }
1066
1067         if (handler && handler->attach) {
1068                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1069                 res = handler->attach(dev);
1070                 TRACE_DBG("New dev handler's attach() returned %d", res);
1071                 if (res != 0) {
1072                         PRINT_ERROR_PR("New device handler's %s attach() "
1073                                 "failed: %d", handler->name, res);
1074                 }
1075                 goto out_thr_null;
1076         }
1077
1078         if (handler && handler->attach_tgt) {
1079                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1080                                 dev_tgt_dev_list_entry) {
1081                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1082                                 tgt_dev);
1083                         res = handler->attach_tgt(tgt_dev);
1084                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1085                         if (res != 0) {
1086                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
1087                                     "failed: %d", handler->name, res);
1088                                 goto out_err_detach_tgt;
1089                         }
1090                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1091                                 &attached_tgt_devs);
1092                 }
1093         }
1094
1095 out_thr_null:
1096         if (res != 0)
1097                 scst_stop_dev_threads(dev);
1098
1099 out_null:
1100         if (res != 0)
1101                 dev->handler = NULL;
1102
1103 out:
1104         TRACE_EXIT_RES(res);
1105         return res;
1106
1107 out_err_detach_tgt:
1108         if (handler && handler->detach_tgt) {
1109                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1110                                  extra_tgt_dev_list_entry)
1111                 {
1112                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1113                                 tgt_dev);
1114                         handler->detach_tgt(tgt_dev);
1115                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1116                 }
1117         }
1118         if (handler && handler->detach) {
1119                 TRACE_DBG("%s", "Calling handler's detach()");
1120                 handler->detach(dev);
1121                 TRACE_DBG("%s", "Handler's detach() returned");
1122         }
1123         goto out_null;
1124 }
1125
1126 int scst_cmd_threads_count(void)
1127 {
1128         int i;
1129
1130         /* Just to lower the race window, when user can get just changed value */
1131         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1132         i = scst_threads_info.nr_cmd_threads;
1133         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1134         return i;
1135 }
1136
1137 static void scst_threads_info_init(void)
1138 {
1139         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1140         mutex_init(&scst_threads_info.cmd_threads_mutex);
1141         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1142 }
1143
1144 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1145 void __scst_del_cmd_threads(int num)
1146 {
1147         struct scst_cmd_thread_t *ct, *tmp;
1148         int i;
1149
1150         TRACE_ENTRY();
1151
1152         i = scst_threads_info.nr_cmd_threads;
1153         if (num <= 0 || num > i) {
1154                 PRINT_ERROR_PR("can not del %d cmd threads from %d", num, i);
1155                 return;
1156         }
1157
1158         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1159                                 thread_list_entry) {
1160                 int res;
1161
1162                 res = kthread_stop(ct->cmd_thread);
1163                 if (res < 0) {
1164                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1165                 }
1166                 list_del(&ct->thread_list_entry);
1167                 kfree(ct);
1168                 scst_threads_info.nr_cmd_threads--;
1169                 --num;
1170                 if (num == 0)
1171                         break;
1172         }
1173
1174         TRACE_EXIT();
1175         return;
1176 }
1177
1178 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1179 int __scst_add_cmd_threads(int num)
1180 {
1181         int res = 0, i;
1182         static int scst_thread_num = 0;
1183
1184         TRACE_ENTRY();
1185
1186         for (i = 0; i < num; i++) {
1187                 struct scst_cmd_thread_t *thr;
1188
1189                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1190                 if (!thr) {
1191                         res = -ENOMEM;
1192                         PRINT_ERROR_PR("fail to allocate thr %d", res);
1193                         goto out_error;
1194                 }
1195                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1196                         &scst_main_cmd_lists, "scsi_tgt%d",
1197                         scst_thread_num++);
1198                 if (IS_ERR(thr->cmd_thread)) {
1199                         res = PTR_ERR(thr->cmd_thread);
1200                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
1201                         kfree(thr);
1202                         goto out_error;
1203                 }
1204                 list_add(&thr->thread_list_entry,
1205                         &scst_threads_info.cmd_threads_list);
1206                 scst_threads_info.nr_cmd_threads++;
1207         }
1208         res = 0;
1209
1210 out:
1211         TRACE_EXIT_RES(res);
1212         return res;
1213
1214 out_error:
1215         if (i > 0)
1216                 __scst_del_cmd_threads(i - 1);
1217         goto out;
1218 }
1219
1220 int scst_add_cmd_threads(int num)
1221 {
1222         int res;
1223
1224         TRACE_ENTRY();
1225
1226         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1227         res = __scst_add_cmd_threads(num);
1228         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1229
1230         TRACE_EXIT_RES(res);
1231         return res;
1232 }
1233
1234 void scst_del_cmd_threads(int num)
1235 {
1236         TRACE_ENTRY();
1237
1238         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1239         __scst_del_cmd_threads(num);
1240         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1241
1242         TRACE_EXIT();
1243         return;
1244 }
1245
1246 static void scst_stop_all_threads(void)
1247 {
1248         TRACE_ENTRY();
1249
1250         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1251         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1252         if (scst_threads_info.mgmt_cmd_thread)
1253                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1254         if (scst_threads_info.mgmt_thread)
1255                 kthread_stop(scst_threads_info.mgmt_thread);
1256         if (scst_threads_info.init_cmd_thread)
1257                 kthread_stop(scst_threads_info.init_cmd_thread);
1258         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1259
1260         TRACE_EXIT();
1261         return;
1262 }
1263
1264 static int scst_start_all_threads(int num)
1265 {
1266         int res;
1267
1268         TRACE_ENTRY();
1269
1270         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1271         res = __scst_add_cmd_threads(num);
1272         if (res < 0)
1273                 goto out;
1274
1275         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1276                 NULL, "scsi_tgt_init");
1277         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1278                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1279                 PRINT_ERROR_PR("kthread_create() for init cmd failed: %d", res);
1280                 scst_threads_info.init_cmd_thread = NULL;
1281                 goto out;
1282         }
1283
1284         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1285                 NULL, "scsi_tgt_mc");
1286         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1287                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1288                 PRINT_ERROR_PR("kthread_create() for mcmd failed: %d", res);
1289                 scst_threads_info.mgmt_cmd_thread = NULL;
1290                 goto out;
1291         }
1292
1293         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1294                 NULL, "scsi_tgt_mgmt");
1295         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1296                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1297                 PRINT_ERROR_PR("kthread_create() for mgmt failed: %d", res);
1298                 scst_threads_info.mgmt_thread = NULL;
1299                 goto out;
1300         }
1301
1302 out:
1303         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1304         TRACE_EXIT_RES(res);
1305         return res;
1306 }
1307
1308 void scst_get(void)
1309 {
1310         __scst_get(0);
1311 }
1312
1313 void scst_put(void)
1314 {
1315         __scst_put();
1316 }
1317
1318 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1319 static int scst_add(struct class_device *cdev)
1320 #else
1321 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1322 #endif
1323 {
1324         struct scsi_device *scsidp;
1325         int res = 0;
1326
1327         TRACE_ENTRY();
1328
1329         scsidp = to_scsi_device(cdev->dev);
1330         res = scst_register_device(scsidp);
1331
1332         TRACE_EXIT();
1333         return res;
1334 }
1335
1336 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1337 static void scst_remove(struct class_device *cdev)
1338 #else
1339 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1340 #endif
1341 {
1342         struct scsi_device *scsidp;
1343
1344         TRACE_ENTRY();
1345
1346         scsidp = to_scsi_device(cdev->dev);
1347         scst_unregister_device(scsidp);
1348
1349         TRACE_EXIT();
1350         return;
1351 }
1352
1353 static struct class_interface scst_interface = {
1354         .add = scst_add,
1355         .remove = scst_remove,
1356 };
1357
1358 static int __init init_scst(void)
1359 {
1360         int res = 0, i;
1361         struct scst_cmd *cmd;
1362         int scst_num_cpus;
1363
1364         TRACE_ENTRY();
1365
1366 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1367         {
1368                 struct scsi_request *req;
1369                 BUILD_BUG_ON(sizeof(cmd->sense_buffer) !=
1370                         sizeof(req->sr_sense_buffer));
1371         }
1372 #else
1373         {
1374                 struct scsi_sense_hdr *shdr;
1375                 BUILD_BUG_ON((sizeof(cmd->sense_buffer) < sizeof(*shdr)) &&
1376                         (sizeof(cmd->sense_buffer) >= SCST_SENSE_BUFFERSIZE));
1377         }
1378 #endif
1379         {
1380                 struct scst_tgt_dev *t;
1381                 struct scst_cmd *c;
1382                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1383                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1384         }
1385
1386         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1387         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1388         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1389         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1390
1391         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1392         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1393         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1394         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1395                 &scst_cmd_lists_list);
1396
1397         scst_num_cpus = num_online_cpus();
1398
1399         /* ToDo: register_cpu_notifier() */
1400
1401         if (scst_threads == 0)
1402                 scst_threads = scst_num_cpus;
1403
1404         if (scst_threads < scst_num_cpus) {
1405                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
1406                         "CPUs count");
1407                 scst_threads = scst_num_cpus;
1408         }
1409
1410         scst_threads_info_init();
1411
1412 #define INIT_CACHEP(p, s, t, o) do {                                    \
1413                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
1414                                       SCST_SLAB_FLAGS, NULL, NULL);     \
1415                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
1416                           sizeof(struct t));                            \
1417                 if (p == NULL) { res = -ENOMEM; goto o; }               \
1418         } while (0)
1419
1420         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING,
1421                     scst_mgmt_cmd, out);
1422         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING,
1423                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
1424         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING,
1425                     scst_cmd, out_destroy_ua_cache);
1426         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
1427                     scst_session, out_destroy_cmd_cache);
1428         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
1429                     scst_tgt_dev, out_destroy_sess_cache);
1430         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
1431                     scst_acg_dev, out_destroy_tgt_cache);
1432
1433         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
1434                 mempool_free_slab, scst_mgmt_cachep);
1435         if (scst_mgmt_mempool == NULL) {
1436                 res = -ENOMEM;
1437                 goto out_destroy_acg_cache;
1438         }
1439
1440         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1441                 mempool_free_slab, scst_ua_cachep);
1442         if (scst_ua_mempool == NULL) {
1443                 res = -ENOMEM;
1444                 goto out_destroy_mgmt_mempool;
1445         }
1446
1447         if (scst_max_cmd_mem == 0) {
1448                 struct sysinfo si;
1449                 si_meminfo(&si);
1450 #if BITS_PER_LONG == 32
1451                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1452                                         (uint64_t)1 << 30);
1453 #else
1454                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1455 #endif
1456         } else
1457                 scst_max_cmd_mem <<= 20;
1458
1459         res = scst_sgv_pools_init(scst_max_cmd_mem, 0);
1460         if (res != 0)
1461                 goto out_destroy_ua_mempool;
1462
1463         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1464         if (scst_default_acg == NULL) {
1465                 res = -ENOMEM;
1466                 goto out_destroy_sgv_pool;
1467         }
1468
1469         res = scsi_register_interface(&scst_interface);
1470         if (res != 0)
1471                 goto out_free_acg;
1472
1473         scst_scsi_op_list_init();
1474
1475         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1476                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1477                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1478                 tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet,
1479                         (unsigned long)&scst_tasklets[i]);
1480         }
1481
1482         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1483                 scst_threads);
1484
1485         res = scst_start_all_threads(scst_threads);
1486         if (res < 0)
1487                 goto out_thread_free;
1488
1489         res = scst_proc_init_module();
1490         if (res != 0)
1491                 goto out_thread_free;
1492
1493
1494         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1495                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1496
1497 out:
1498         TRACE_EXIT_RES(res);
1499         return res;
1500
1501 out_thread_free:
1502         scst_stop_all_threads();
1503
1504         scsi_unregister_interface(&scst_interface);
1505
1506 out_free_acg:
1507         scst_destroy_acg(scst_default_acg);
1508
1509 out_destroy_sgv_pool:
1510         scst_sgv_pools_deinit();
1511
1512 out_destroy_ua_mempool:
1513         mempool_destroy(scst_ua_mempool);
1514
1515 out_destroy_mgmt_mempool:
1516         mempool_destroy(scst_mgmt_mempool);
1517
1518 out_destroy_acg_cache:
1519         kmem_cache_destroy(scst_acgd_cachep);
1520
1521 out_destroy_tgt_cache:
1522         kmem_cache_destroy(scst_tgtd_cachep);
1523
1524 out_destroy_sess_cache:
1525         kmem_cache_destroy(scst_sess_cachep);
1526
1527 out_destroy_cmd_cache:
1528         kmem_cache_destroy(scst_cmd_cachep);
1529
1530 out_destroy_ua_cache:
1531         kmem_cache_destroy(scst_ua_cachep);
1532
1533 out_destroy_mgmt_cache:
1534         kmem_cache_destroy(scst_mgmt_cachep);
1535         goto out;
1536 }
1537
1538 static void __exit exit_scst(void)
1539 {
1540 #ifdef CONFIG_LOCKDEP
1541         static /* To hide the lockdep's warning about non-static key */
1542 #endif
1543         DECLARE_MUTEX_LOCKED(shm);
1544
1545         TRACE_ENTRY();
1546
1547         /* ToDo: unregister_cpu_notifier() */
1548
1549         scst_proc_cleanup_module();
1550
1551         scst_stop_all_threads();
1552
1553         scsi_unregister_interface(&scst_interface);
1554         scst_destroy_acg(scst_default_acg);
1555
1556         scst_sgv_pools_deinit();
1557
1558 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
1559 #define DEINIT_CACHEP(p, s) do {                        \
1560                 if (kmem_cache_destroy(p)) {            \
1561                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1562                                 "error", s);            \
1563                 }                                       \
1564                 p = NULL;                               \
1565         } while (0)
1566 #else
1567 #define DEINIT_CACHEP(p, s) do {                        \
1568                 kmem_cache_destroy(p);                  \
1569                 p = NULL;                               \
1570         } while (0)
1571 #endif
1572
1573         mempool_destroy(scst_mgmt_mempool);
1574         mempool_destroy(scst_ua_mempool);
1575
1576         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1577         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1578         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1579         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1580         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1581         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1582
1583         PRINT_INFO_PR("%s", "SCST unloaded");
1584
1585         TRACE_EXIT();
1586         return;
1587 }
1588
1589 /*
1590  * Device Handler Side (i.e. scst_vdisk)
1591  */
1592 EXPORT_SYMBOL(scst_register_dev_driver);
1593 EXPORT_SYMBOL(scst_unregister_dev_driver);
1594 EXPORT_SYMBOL(scst_register);
1595 EXPORT_SYMBOL(scst_unregister);
1596
1597 EXPORT_SYMBOL(scst_register_virtual_device);
1598 EXPORT_SYMBOL(scst_unregister_virtual_device);
1599 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1600 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1601
1602 EXPORT_SYMBOL(scst_set_busy);
1603 EXPORT_SYMBOL(scst_set_cmd_error_status);
1604 EXPORT_SYMBOL(scst_set_cmd_error);
1605 EXPORT_SYMBOL(scst_set_resp_data_len);
1606
1607 EXPORT_SYMBOL(scst_process_active_cmd);
1608
1609 /*
1610  * Target Driver Side (i.e. HBA)
1611  */
1612 EXPORT_SYMBOL(scst_register_session);
1613 EXPORT_SYMBOL(scst_unregister_session);
1614
1615 EXPORT_SYMBOL(scst_register_target_template);
1616 EXPORT_SYMBOL(scst_unregister_target_template);
1617
1618 EXPORT_SYMBOL(scst_cmd_init_done);
1619 EXPORT_SYMBOL(scst_tgt_cmd_done);
1620 EXPORT_SYMBOL(scst_restart_cmd);
1621 EXPORT_SYMBOL(scst_rx_cmd);
1622 EXPORT_SYMBOL(scst_rx_data);
1623 EXPORT_SYMBOL(scst_rx_mgmt_fn);
1624
1625 EXPORT_SYMBOL(scst_find_cmd);
1626 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1627
1628 /*
1629  * Global Commands
1630  */
1631 EXPORT_SYMBOL(scst_suspend_activity);
1632 EXPORT_SYMBOL(scst_resume_activity);
1633
1634 EXPORT_SYMBOL(scst_add_cmd_threads);
1635 EXPORT_SYMBOL(scst_del_cmd_threads);
1636
1637 #if defined(DEBUG) || defined(TRACING)
1638 EXPORT_SYMBOL(scst_proc_log_entry_read);
1639 EXPORT_SYMBOL(scst_proc_log_entry_write);
1640 #endif
1641
1642 EXPORT_SYMBOL(scst_create_proc_entry);
1643 EXPORT_SYMBOL(scst_single_seq_open);
1644
1645 EXPORT_SYMBOL(__scst_get_buf);
1646 EXPORT_SYMBOL(scst_get);
1647 EXPORT_SYMBOL(scst_put);
1648
1649 EXPORT_SYMBOL(scst_alloc);
1650 EXPORT_SYMBOL(scst_free);
1651
1652 EXPORT_SYMBOL(scst_check_local_events);
1653
1654 /* Tgt_dev's threads local storage */
1655 EXPORT_SYMBOL(scst_add_thr_data);
1656 EXPORT_SYMBOL(scst_del_all_thr_data);
1657 EXPORT_SYMBOL(scst_dev_del_all_thr_data);
1658 EXPORT_SYMBOL(scst_find_thr_data);
1659
1660 /* SGV pool routines */
1661 EXPORT_SYMBOL(sgv_pool_create);
1662 EXPORT_SYMBOL(sgv_pool_destroy);
1663 EXPORT_SYMBOL(sgv_pool_set_allocator);
1664 EXPORT_SYMBOL(sgv_pool_alloc);
1665 EXPORT_SYMBOL(sgv_pool_free);
1666 EXPORT_SYMBOL(sgv_get_priv);
1667
1668 /* Generic parse() routines */
1669 EXPORT_SYMBOL(scst_calc_block_shift);
1670 EXPORT_SYMBOL(scst_sbc_generic_parse);
1671 EXPORT_SYMBOL(scst_cdrom_generic_parse);
1672 EXPORT_SYMBOL(scst_modisk_generic_parse);
1673 EXPORT_SYMBOL(scst_tape_generic_parse);
1674 EXPORT_SYMBOL(scst_changer_generic_parse);
1675 EXPORT_SYMBOL(scst_processor_generic_parse);
1676 EXPORT_SYMBOL(scst_raid_generic_parse);
1677
1678 /* Generic dev_done() routines */
1679 EXPORT_SYMBOL(scst_block_generic_dev_done);
1680 EXPORT_SYMBOL(scst_tape_generic_dev_done);
1681
1682 /*
1683  * Other Commands
1684  */
1685 EXPORT_SYMBOL(scst_get_cdb_info);
1686 EXPORT_SYMBOL(scst_cmd_get_tgt_priv_lock);
1687 EXPORT_SYMBOL(scst_cmd_set_tgt_priv_lock);
1688
1689 #ifdef DEBUG
1690 EXPORT_SYMBOL(scst_random);
1691 #endif
1692
1693 module_init(init_scst);
1694 module_exit(exit_scst);
1695
1696 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1697 MODULE_LICENSE("GPL");
1698 MODULE_DESCRIPTION("SCSI target core");
1699 MODULE_VERSION(SCST_VERSION_STRING);