scst/src/scst.c

   1 /*
   2  *  scst.c
   3  *
   4  *  Copyright (C) 2004-2006 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *                 and Leonid Stoljar
   6  *
   7  *  This program is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU General Public License
   9  *  as published by the Free Software Foundation, version 2
  10  *  of the License.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  *  GNU General Public License for more details.
  16  */
  17
  18 #include <linux/module.h>
  19
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/errno.h>
  23 #include <linux/list.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched.h>
  27 #include <asm/unistd.h>
  28 #include <asm/string.h>
  29
  30 #include "scst_debug.h"
  31 #include "scsi_tgt.h"
  32 #include "scst_priv.h"
  33 #include "scst_mem.h"
  34
  35 #include "scst_debug.c"
  36
  37 #if defined(DEBUG) || defined(TRACING)
  38 unsigned long trace_flag = SCST_DEFAULT_LOG_FLAGS;
  39 #endif
  40
  41 /*
  42  * All targets, devices and dev_types management is done under
  43  * this mutex.
  44  */
  45 DECLARE_MUTEX(scst_mutex);
  46
  47 int scst_num_cpus;
  48 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
  49 LIST_HEAD(scst_dev_wait_sess_list);
  50
  51 LIST_HEAD(scst_template_list);
  52 LIST_HEAD(scst_dev_list);
  53 LIST_HEAD(scst_dev_type_list);
  54
  55 kmem_cache_t *scst_mgmt_cachep;
  56 mempool_t *scst_mgmt_mempool;
  57 kmem_cache_t *scst_ua_cachep;
  58 mempool_t *scst_ua_mempool;
  59 kmem_cache_t *scst_tgtd_cachep;
  60 kmem_cache_t *scst_sess_cachep;
  61 kmem_cache_t *scst_acgd_cachep;
  62
  63 LIST_HEAD(scst_acg_list);
  64 struct scst_acg *scst_default_acg;
  65
  66 kmem_cache_t *scst_cmd_cachep;
  67
  68 unsigned long scst_flags;
  69 atomic_t scst_cmd_count = ATOMIC_INIT(0);
  70 spinlock_t scst_list_lock = SPIN_LOCK_UNLOCKED;
  71 LIST_HEAD(scst_active_cmd_list);
  72 LIST_HEAD(scst_init_cmd_list);
  73 LIST_HEAD(scst_cmd_list);
  74 DECLARE_WAIT_QUEUE_HEAD(scst_list_waitQ);
  75
  76 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
  77 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
  78
  79 struct tasklet_struct scst_tasklets[NR_CPUS];
  80
  81 struct scst_sgv_pools scst_sgv;
  82
  83 DECLARE_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn, 0);
  84
  85 unsigned long scst_max_cmd_mem;
  86
  87 LIST_HEAD(scst_mgmt_cmd_list);
  88 LIST_HEAD(scst_active_mgmt_cmd_list);
  89 LIST_HEAD(scst_delayed_mgmt_cmd_list);
  90 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
  91
  92 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
  93 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
  94 LIST_HEAD(scst_sess_mgmt_list);
  95
  96 struct semaphore *scst_shutdown_mutex = NULL;
  97
  98 int scst_threads;
  99 atomic_t scst_threads_count = ATOMIC_INIT(0);
 100 int scst_shut_threads_count;
 101 int scst_thread_num;
 102 static int suspend_count;
 103
 104 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
 105
 106 /*
 107  * This buffer and lock are intended to avoid memory allocation, which
 108  * could fail in improper places.
 109  */
 110 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
 111 uint8_t scst_temp_UA[SCSI_SENSE_BUFFERSIZE];
 112
 113 module_param_named(scst_threads, scst_threads, int, 0);
 114 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
 115
 116 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
 117 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
 118         "the SCST commands at any given time in Mb");
 119
 120 int scst_register_target_template(struct scst_tgt_template *vtt)
 121 {
 122         int res = 0;
 123         struct scst_tgt_template *t;
 124
 125         TRACE_ENTRY();
 126
 127         INIT_LIST_HEAD(&vtt->tgt_list);
 128
 129         if (!vtt->detect) {
 130                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 131                         "detect() method.", vtt->name);
 132                 res = -EINVAL;
 133                 goto out;
 134         }
 135
 136         if (!vtt->release) {
 137                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 138                         "release() method.", vtt->name);
 139                 res = -EINVAL;
 140                 goto out;
 141         }
 142
 143         if (!vtt->xmit_response) {
 144                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 145                         "xmit_response() method.", vtt->name);
 146                 res = -EINVAL;
 147                 goto out;
 148         }
 149
 150         if (!vtt->rdy_to_xfer) {
 151                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 152                         "rdy_to_xfer() method.", vtt->name);
 153                 res = -EINVAL;
 154                 goto out;
 155         }
 156
 157         if (!vtt->on_free_cmd) {
 158                 PRINT_ERROR_PR("Target driver %s doesn't have a "
 159                         "on_free_cmd() method.", vtt->name);
 160                 res = -EINVAL;
 161                 goto out;
 162         }
 163
 164         if (!vtt->no_proc_entry) {
 165                 res = scst_build_proc_target_dir_entries(vtt);
 166                 if (res < 0) {
 167                         goto out;
 168                 }
 169         }
 170
 171         if (down_interruptible(&scst_mutex) != 0)
 172                 goto out;
 173         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
 174                 if (strcmp(t->name, vtt->name) == 0) {
 175                         PRINT_ERROR_PR("Target driver %s already registered",
 176                                 vtt->name);
 177                         up(&scst_mutex);
 178                         goto out_cleanup;
 179                 }
 180         }
 181         /* That's OK to drop it. The race doesn't matter */
 182         up(&scst_mutex);
 183
 184         TRACE_DBG("%s", "Calling target driver's detect()");
 185         res = vtt->detect(vtt);
 186         TRACE_DBG("Target driver's detect() returned %d", res);
 187         if (res < 0) {
 188                 PRINT_ERROR_PR("%s", "The detect() routine failed");
 189                 res = -EINVAL;
 190                 goto out_cleanup;
 191         }
 192
 193         down(&scst_mutex);
 194         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
 195         up(&scst_mutex);
 196
 197         res = 0;
 198
 199 out:
 200         TRACE_EXIT_RES(res);
 201         return res;
 202
 203 out_cleanup:
 204         scst_cleanup_proc_target_dir_entries(vtt);
 205         goto out;
 206 }
 207
 208 void scst_unregister_target_template(struct scst_tgt_template *vtt)
 209 {
 210         struct scst_tgt *tgt;
 211
 212         TRACE_ENTRY();
 213
 214         if (down_interruptible(&scst_mutex) != 0)
 215                 goto out;
 216
 217 restart:
 218         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
 219                 up(&scst_mutex);
 220                 scst_unregister(tgt);
 221                 down(&scst_mutex);
 222                 goto restart;
 223         }
 224         list_del(&vtt->scst_template_list_entry);
 225         up(&scst_mutex);
 226
 227         scst_cleanup_proc_target_dir_entries(vtt);
 228
 229 out:
 230         TRACE_EXIT();
 231         return;
 232 }
 233
 234 struct scst_tgt *scst_register(struct scst_tgt_template *vtt)
 235 {
 236         struct scst_tgt *tgt;
 237
 238         TRACE_ENTRY();
 239
 240         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
 241         TRACE_MEM("kzalloc(GFP_KERNEL) for tgt (%zd): %p",
 242               sizeof(*tgt), tgt);
 243         if (tgt == NULL) {
 244                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
 245                 goto out;
 246         }
 247
 248         INIT_LIST_HEAD(&tgt->sess_list);
 249         init_waitqueue_head(&tgt->unreg_waitQ);
 250         tgt->tgtt = vtt;
 251         tgt->sg_tablesize = vtt->sg_tablesize;
 252         spin_lock_init(&tgt->tgt_lock);
 253         INIT_LIST_HEAD(&tgt->retry_cmd_list);
 254         atomic_set(&tgt->finished_cmds, 0);
 255         init_timer(&tgt->retry_timer);
 256         tgt->retry_timer.data = (unsigned long)tgt;
 257         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
 258
 259         down(&scst_mutex);
 260
 261         if (scst_build_proc_target_entries(tgt) < 0) {
 262                 tgt = NULL;
 263                 goto out_free;
 264         }
 265
 266         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
 267
 268         up(&scst_mutex);
 269
 270 out:
 271         TRACE_EXIT();
 272         return tgt;
 273
 274 out_free:
 275         TRACE_MEM("kfree() for tgt %p", tgt);
 276         kfree(tgt);
 277         goto out;
 278 }
 279
 280 static inline int test_sess_list(struct scst_tgt *tgt)
 281 {
 282         int res;
 283         down(&scst_mutex);
 284         res = list_empty(&tgt->sess_list);
 285         up(&scst_mutex);
 286         return res;
 287 }
 288
 289 void scst_unregister(struct scst_tgt *tgt)
 290 {
 291         struct scst_session *sess;
 292
 293         TRACE_ENTRY();
 294
 295         TRACE_DBG("%s", "Calling target driver's release()");
 296         tgt->tgtt->release(tgt);
 297         TRACE_DBG("%s", "Target driver's release() returned");
 298
 299         down(&scst_mutex);
 300         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
 301                 BUG_ON(!sess->shutting_down);
 302         }
 303         up(&scst_mutex);
 304
 305         TRACE_DBG("%s", "Waiting for sessions shutdown");
 306         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
 307         TRACE_DBG("%s", "wait_event() returned");
 308
 309         down(&scst_mutex);
 310         list_del(&tgt->tgt_list_entry);
 311         up(&scst_mutex);
 312
 313         scst_cleanup_proc_target_entries(tgt);
 314
 315         del_timer_sync(&tgt->retry_timer);
 316
 317         TRACE_MEM("kfree for tgt: %p", tgt);
 318         kfree(tgt);
 319
 320         TRACE_EXIT();
 321         return;
 322 }
 323
 324 /* scst_mutex supposed to be held */
 325 void __scst_suspend_activity(void)
 326 {
 327         TRACE_ENTRY();
 328
 329         suspend_count++;
 330         if (suspend_count > 1)
 331                 goto out;
 332
 333         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 334         smp_mb__after_set_bit();
 335
 336         TRACE_DBG("Waiting for all %d active commands to complete",
 337               atomic_read(&scst_cmd_count));
 338         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
 339         TRACE_DBG("%s", "wait_event() returned");
 340
 341 out:
 342         TRACE_EXIT();
 343         return;
 344 }
 345
 346 void scst_suspend_activity(void)
 347 {
 348         down(&scst_mutex);
 349         __scst_suspend_activity();
 350 }
 351
 352 /* scst_mutex supposed to be held */
 353 void __scst_resume_activity(void)
 354 {
 355         struct scst_session *sess, *tsess;
 356
 357         TRACE_ENTRY();
 358
 359         suspend_count--;
 360         if (suspend_count > 0)
 361                 goto out;
 362
 363         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
 364         smp_mb__after_clear_bit();
 365
 366         spin_lock_irq(&scst_list_lock);
 367         list_for_each_entry_safe(sess, tsess, &scst_dev_wait_sess_list,
 368                             dev_wait_sess_list_entry)
 369         {
 370                 sess->waiting = 0;
 371                 list_del(&sess->dev_wait_sess_list_entry);
 372         }
 373         spin_unlock_irq(&scst_list_lock);
 374
 375         wake_up_all(&scst_list_waitQ);
 376         wake_up_all(&scst_mgmt_cmd_list_waitQ);
 377
 378 out:
 379         TRACE_EXIT();
 380         return;
 381 }
 382
 383 void scst_resume_activity(void)
 384 {
 385         __scst_resume_activity();
 386         up(&scst_mutex);
 387 }
 388
 389 /* Called under scst_mutex */
 390 static int scst_register_device(struct scsi_device *scsidp)
 391 {
 392         int res = 0;
 393         struct scst_device *dev;
 394         struct scst_dev_type *dt;
 395
 396         TRACE_ENTRY();
 397
 398         dev = scst_alloc_device(GFP_KERNEL);
 399         if (dev == NULL) {
 400                 res = -ENOMEM;
 401                 goto out;
 402         }
 403
 404         dev->rq_disk = alloc_disk(1);
 405         if (dev->rq_disk == NULL) {
 406                 res = -ENOMEM;
 407                 scst_free_device(dev);
 408                 goto out;
 409         }
 410         dev->rq_disk->major = SCST_MAJOR;
 411
 412         dev->scsi_dev = scsidp;
 413
 414         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 415
 416         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 417                 if (dt->type == scsidp->type) {
 418                         res = scst_assign_dev_handler(dev, dt);
 419                         if (res != 0)
 420                                 goto out_free;
 421                         break;
 422                 }
 423         }
 424
 425 out:
 426         if (res == 0) {
 427                 PRINT_INFO_PR("Attached SCSI target mid-level at "
 428                     "scsi%d, channel %d, id %d, lun %d, type %d",
 429                     scsidp->host->host_no, scsidp->channel, scsidp->id,
 430                     scsidp->lun, scsidp->type);
 431         }
 432         else {
 433                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
 434                     "at scsi%d, channel %d, id %d, lun %d, type %d",
 435                     scsidp->host->host_no, scsidp->channel, scsidp->id,
 436                     scsidp->lun, scsidp->type);
 437         }
 438
 439         TRACE_EXIT_RES(res);
 440         return res;
 441
 442 out_free:
 443         put_disk(dev->rq_disk);
 444
 445         list_del(&dev->dev_list_entry);
 446         scst_assign_dev_handler(dev, NULL);
 447         goto out;
 448 }
 449
 450 /* Called under scst_mutex */
 451 static void scst_unregister_device(struct scsi_device *scsidp)
 452 {
 453         struct scst_device *d, *dev = NULL;
 454         struct scst_acg_dev *acg_dev, *aa;
 455
 456         TRACE_ENTRY();
 457
 458         __scst_suspend_activity();
 459
 460         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 461                 if (d->scsi_dev == scsidp) {
 462                         dev = d;
 463                         TRACE_DBG("Target device %p found", dev);
 464                         break;
 465                 }
 466         }
 467
 468         if (dev == NULL) {
 469                 PRINT_ERROR_PR("%s", "Target device not found");
 470                 goto out_unblock;
 471         }
 472
 473         list_del(&dev->dev_list_entry);
 474
 475         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
 476                                  dev_acg_dev_list_entry)
 477         {
 478                 scst_acg_remove_dev(acg_dev->acg, dev);
 479         }
 480
 481         scst_assign_dev_handler(dev, NULL);
 482
 483         put_disk(dev->rq_disk);
 484         scst_free_device(dev);
 485
 486 out_unblock:
 487         __scst_resume_activity();
 488
 489         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
 490              "id %d, lun %d, type %d", scsidp->host->host_no, scsidp->channel,
 491              scsidp->id, scsidp->lun, scsidp->type);
 492
 493         TRACE_EXIT();
 494         return;
 495 }
 496
 497 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
 498         const char *dev_name)
 499 {
 500         int res = -EINVAL, rc;
 501         struct scst_device *dev = NULL;
 502
 503         TRACE_ENTRY();
 504
 505         if (dev_handler == NULL) {
 506                 PRINT_ERROR_PR("%s: valid device handler must be supplied",
 507                         __FUNCTION__);
 508                 res = -EINVAL;
 509                 goto out;
 510         }
 511
 512         if (dev_name == NULL) {
 513                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
 514                 res = -EINVAL;
 515                 goto out;
 516         }
 517
 518         dev = scst_alloc_device(GFP_KERNEL);
 519         if (dev == NULL) {
 520                 res = -ENOMEM;
 521                 goto out;
 522         }
 523
 524         dev->scsi_dev = NULL;
 525         dev->virt_name = dev_name;
 526
 527         if (down_interruptible(&scst_mutex) != 0) {
 528                 res = -EINTR;
 529                 goto out_free_dev;
 530         }
 531
 532         dev->virt_id = scst_virt_dev_last_id++;
 533
 534         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
 535
 536         res = dev->virt_id;
 537
 538         rc = scst_assign_dev_handler(dev, dev_handler);
 539         if (rc != 0) {
 540                 res = rc;
 541                 goto out_free_del;
 542         }
 543
 544         up(&scst_mutex);
 545
 546 out:
 547         if (res > 0) {
 548                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
 549                     "device %s (id %d)", dev_name, dev->virt_id);
 550         }
 551         else {
 552                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
 553                     "virtual device %s", dev_name);
 554         }
 555
 556         TRACE_EXIT_RES(res);
 557         return res;
 558
 559 out_free_del:
 560         list_del(&dev->dev_list_entry);
 561         up(&scst_mutex);
 562
 563 out_free_dev:
 564         scst_free_device(dev);
 565         goto out;
 566 }
 567
 568 void scst_unregister_virtual_device(int id)
 569 {
 570         struct scst_device *d, *dev = NULL;
 571         struct scst_acg_dev *acg_dev, *aa;
 572
 573         TRACE_ENTRY();
 574
 575         if (down_interruptible(&scst_mutex) != 0)
 576                 goto out;
 577
 578         __scst_suspend_activity();
 579
 580         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
 581                 if (d->virt_id == id) {
 582                         dev = d;
 583                         TRACE_DBG("Target device %p found", dev);
 584                         break;
 585                 }
 586         }
 587
 588         if (dev == NULL) {
 589                 PRINT_ERROR_PR("%s", "Target device not found");
 590                 goto out_unblock;
 591         }
 592
 593         list_del(&dev->dev_list_entry);
 594
 595         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
 596                                  dev_acg_dev_list_entry)
 597         {
 598                 scst_acg_remove_dev(acg_dev->acg, dev);
 599         }
 600
 601         scst_assign_dev_handler(dev, NULL);
 602
 603         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
 604                 "(id %d)", dev->virt_name, dev->virt_id);
 605
 606         scst_free_device(dev);
 607
 608 out_unblock:
 609         __scst_resume_activity();
 610
 611         up(&scst_mutex);
 612
 613 out:
 614         TRACE_EXIT();
 615         return;
 616 }
 617
 618 int scst_register_dev_driver(struct scst_dev_type *dev_type)
 619 {
 620         struct scst_dev_type *dt;
 621         struct scst_device *dev;
 622         int res = 0;
 623         int exist;
 624
 625         TRACE_ENTRY();
 626
 627         if (dev_type->parse == NULL) {
 628                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
 629                         "parse() method.", dev_type->name);
 630                 res = -EINVAL;
 631                 goto out;
 632         }
 633
 634 #ifdef FILEIO_ONLY
 635         if (dev_type->exec == NULL) {
 636                 PRINT_ERROR_PR("Pass-through dev handlers (handler %s) not "
 637                         "supported. Recompile SCST with undefined FILEIO_ONLY",
 638                         dev_type->name);
 639                 res = -EINVAL;
 640                 goto out;
 641         }
 642 #endif
 643
 644         if (down_interruptible(&scst_mutex) != 0) {
 645                 res = -EINTR;
 646                 goto out;
 647         }
 648
 649         exist = 0;
 650         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
 651                 if (dt->type == dev_type->type) {
 652                         TRACE_DBG("Device type handler for type %d "
 653                                 "already exist", dt->type);
 654                         exist = 1;
 655                         break;
 656                 }
 657         }
 658
 659         res = scst_build_proc_dev_handler_dir_entries(dev_type);
 660         if (res < 0) {
 661                 goto out_up;
 662         }
 663
 664         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
 665
 666         if (exist)
 667                 goto out_up;
 668
 669         __scst_suspend_activity();
 670         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
 671                 if (dev->scsi_dev == NULL)
 672                         continue;
 673                 if (dev->scsi_dev->type == dev_type->type)
 674                         scst_assign_dev_handler(dev, dev_type);
 675         }
 676         __scst_resume_activity();
 677
 678 out_up:
 679         up(&scst_mutex);
 680
 681         if (res == 0) {
 682                 PRINT_INFO_PR("Device handler %s for type %d loaded "
 683                         "successfully", dev_type->name, dev_type->type);
 684         }
 685
 686 out:
 687         TRACE_EXIT_RES(res);
 688         return res;
 689 }
 690
 691 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
 692 {
 693         struct scst_device *dev;
 694
 695         TRACE_ENTRY();
 696
 697         if (down_interruptible(&scst_mutex) != 0)
 698                 goto out;
 699
 700         __scst_suspend_activity();
 701         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
 702                 if (dev->handler == dev_type) {
 703                         scst_assign_dev_handler(dev, NULL);
 704                         TRACE_DBG("Dev handler removed from device %p", dev);
 705                 }
 706         }
 707         __scst_resume_activity();
 708
 709         list_del(&dev_type->dev_type_list_entry);
 710
 711         up(&scst_mutex);
 712
 713         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
 714
 715         PRINT_INFO_PR("Device handler %s for type %d unloaded",
 716                    dev_type->name, dev_type->type);
 717
 718 out:
 719         TRACE_EXIT();
 720         return;
 721 }
 722
 723 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
 724 {
 725         int res = 0;
 726
 727         TRACE_ENTRY();
 728
 729         if (dev_type->parse == NULL) {
 730                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
 731                         "parse() method.", dev_type->name);
 732                 res = -EINVAL;
 733                 goto out;
 734         }
 735
 736         res = scst_build_proc_dev_handler_dir_entries(dev_type);
 737         if (res < 0) {
 738                 goto out;
 739         }
 740
 741         PRINT_INFO_PR("Device handler %s for type %d loaded "
 742                 "successfully", dev_type->name, dev_type->type);
 743
 744 out:
 745         TRACE_EXIT_RES(res);
 746         return res;
 747 }
 748
 749 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
 750 {
 751         TRACE_ENTRY();
 752
 753         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
 754
 755         PRINT_INFO_PR("Device handler %s for type %d unloaded",
 756                    dev_type->name, dev_type->type);
 757
 758         TRACE_EXIT();
 759         return;
 760 }
 761
 762 /* scst_mutex supposed to be held */
 763 int scst_assign_dev_handler(struct scst_device *dev,
 764         struct scst_dev_type *handler)
 765 {
 766         int res = 0;
 767         struct scst_tgt_dev *tgt_dev;
 768         LIST_HEAD(attached_tgt_devs);
 769
 770         TRACE_ENTRY();
 771
 772         if (dev->handler == handler)
 773                 goto out;
 774
 775         __scst_suspend_activity();
 776
 777         if (dev->handler && dev->handler->detach_tgt) {
 778                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
 779                         dev_tgt_dev_list_entry)
 780                 {
 781                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
 782                                 tgt_dev);
 783                         dev->handler->detach_tgt(tgt_dev);
 784                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
 785                 }
 786         }
 787
 788         if (dev->handler && dev->handler->detach) {
 789                 TRACE_DBG("%s", "Calling dev handler's detach()");
 790                 dev->handler->detach(dev);
 791                 TRACE_DBG("%s", "Old handler's detach() returned");
 792         }
 793
 794         dev->handler = handler;
 795
 796         if (handler && handler->attach) {
 797                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
 798                 res = handler->attach(dev);
 799                 TRACE_DBG("New dev handler's attach() returned %d", res);
 800                 if (res != 0) {
 801                         PRINT_ERROR_PR("New device handler's %s attach() "
 802                                 "failed: %d", handler->name, res);
 803                 }
 804                 goto out_resume;
 805         }
 806
 807         if (handler && handler->attach_tgt) {
 808                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
 809                         dev_tgt_dev_list_entry)
 810                 {
 811                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
 812                                 tgt_dev);
 813                         res = handler->attach_tgt(tgt_dev);
 814                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
 815                         if (res != 0) {
 816                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
 817                                     "failed: %d", handler->name, res);
 818                                 goto out_err_detach_tgt;
 819                         }
 820                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
 821                                 &attached_tgt_devs);
 822                 }
 823         }
 824
 825 out_resume:
 826         if (res != 0)
 827                 dev->handler = NULL;
 828         __scst_resume_activity();
 829
 830 out:
 831         TRACE_EXIT_RES(res);
 832         return res;
 833
 834 out_err_detach_tgt:
 835         if (handler && handler->detach_tgt) {
 836                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
 837                                  extra_tgt_dev_list_entry)
 838                 {
 839                         TRACE_DBG("Calling handler's detach_tgt(%p)",
 840                                 tgt_dev);
 841                         handler->detach_tgt(tgt_dev);
 842                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
 843                 }
 844         }
 845         if (handler && handler->detach) {
 846                 TRACE_DBG("%s", "Calling handler's detach()");
 847                 handler->detach(dev);
 848                 TRACE_DBG("%s", "Hhandler's detach() returned");
 849         }
 850         goto out_resume;
 851 }
 852
 853 int scst_add_threads(int num)
 854 {
 855         int res = 0, i;
 856
 857         TRACE_ENTRY();
 858
 859         for (i = 0; i < num; i++) {
 860                 res = kernel_thread(scst_cmd_thread, 0, SCST_THREAD_FLAGS);
 861                 if (res < 0) {
 862                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
 863                         goto out_error;
 864                 }
 865                 atomic_inc(&scst_threads_count);
 866         }
 867
 868         res = 0;
 869
 870 out:
 871         TRACE_EXIT_RES(res);
 872         return res;
 873
 874 out_error:
 875         if (i > 0)
 876                 scst_del_threads(i-1);
 877         goto out;
 878 }
 879
 880 void scst_del_threads(int num)
 881 {
 882         TRACE_ENTRY();
 883
 884         spin_lock_irq(&scst_list_lock);
 885         scst_shut_threads_count += num;
 886         spin_unlock_irq(&scst_list_lock);
 887
 888         wake_up_nr(&scst_list_waitQ, num);
 889
 890         TRACE_EXIT();
 891         return;
 892 }
 893
 894 void scst_get(void)
 895 {
 896         scst_inc_cmd_count();
 897 }
 898
 899 void scst_put(void)
 900 {
 901         scst_dec_cmd_count();
 902 }
 903
 904 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
 905 static int scst_add(struct class_device *cdev)
 906 #else
 907 static int scst_add(struct class_device *cdev, struct class_interface *intf)
 908 #endif
 909 {
 910         struct scsi_device *scsidp;
 911         int res = 0;
 912
 913         TRACE_ENTRY();
 914
 915         scsidp = to_scsi_device(cdev->dev);
 916
 917         down(&scst_mutex);
 918         res = scst_register_device(scsidp);
 919         up(&scst_mutex);
 920
 921         TRACE_EXIT();
 922         return res;
 923 }
 924
 925 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
 926 static void scst_remove(struct class_device *cdev)
 927 #else
 928 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
 929 #endif
 930 {
 931         struct scsi_device *scsidp;
 932
 933         TRACE_ENTRY();
 934
 935         scsidp = to_scsi_device(cdev->dev);
 936
 937         down(&scst_mutex);
 938         scst_unregister_device(scsidp);
 939         up(&scst_mutex);
 940
 941         TRACE_EXIT();
 942         return;
 943 }
 944
 945 static struct class_interface scst_interface = {
 946         .add = scst_add,
 947         .remove = scst_remove,
 948 };
 949
 950 static inline int get_cpus_count(void)
 951 {
 952 #ifdef CONFIG_SMP
 953         return cpus_weight(cpu_online_map);
 954 #else
 955         return 1;
 956 #endif
 957 }
 958
 959 static int __init init_scst(void)
 960 {
 961         int res = 0, i;
 962         struct scst_cmd *cmd;
 963
 964         TRACE_ENTRY();
 965
 966 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
 967         {
 968                 struct scsi_request *req;
 969                 BUILD_BUG_ON(sizeof(cmd->sense_buffer) !=
 970                         sizeof(req->sr_sense_buffer));
 971         }
 972 #else
 973         {
 974                 struct scsi_sense_hdr *shdr;
 975                 BUILD_BUG_ON((sizeof(cmd->sense_buffer) < sizeof(*shdr)) &&
 976                         (sizeof(cmd->sense_buffer) >= SCSI_SENSE_BUFFERSIZE));
 977         }
 978 #endif
 979
 980         scst_num_cpus = get_cpus_count();
 981
 982         /* ToDo: register_cpu_notifier() */
 983
 984         if (scst_threads == 0)
 985                 scst_threads = scst_num_cpus;
 986
 987         if (scst_threads < scst_num_cpus) {
 988                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
 989                         "CPUs count");
 990                 res = -EFAULT;
 991                 goto out;
 992         }
 993
 994 #define INIT_CACHEP(p, s, t, o) do {                                    \
 995                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
 996                                       SCST_SLAB_FLAGS, NULL, NULL);     \
 997                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
 998                           sizeof(struct t));                            \
 999                 if (p == NULL) { res = -ENOMEM; goto o; }               \
1000         } while (0)
1001
1002         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING,
1003                     scst_mgmt_cmd, out);
1004         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING,
1005                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
1006         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING,
1007                     scst_cmd, out_destroy_ua_cache);
1008         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
1009                     scst_session, out_destroy_cmd_cache);
1010         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
1011                     scst_tgt_dev, out_destroy_sess_cache);
1012         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
1013                     scst_acg_dev, out_destroy_tgt_cache);
1014
1015         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
1016                 mempool_free_slab, scst_mgmt_cachep);
1017         if (scst_mgmt_mempool == NULL) {
1018                 res = -ENOMEM;
1019                 goto out_destroy_acg_cache;
1020         }
1021
1022         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1023                 mempool_free_slab, scst_ua_cachep);
1024         if (scst_ua_mempool == NULL) {
1025                 res = -ENOMEM;
1026                 goto out_destroy_mgmt_mempool;
1027         }
1028
1029         res = scst_sgv_pools_init(&scst_sgv);
1030         if (res != 0)
1031                 goto out_destroy_ua_mempool;
1032
1033         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1034         if (scst_default_acg == NULL) {
1035                 res = -ENOMEM;
1036                 goto out_destroy_sgv_pool;
1037         }
1038
1039         res = scsi_register_interface(&scst_interface);
1040         if (res != 0)
1041                 goto out_free_acg;
1042
1043         scst_scsi_op_list_init();
1044
1045         res = scst_proc_init_module();
1046         if (res != 0)
1047                 goto out_unreg_interface;
1048
1049         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1050                 scst_threads);
1051
1052         for (i = 0; i < scst_threads; i++) {
1053                 res = kernel_thread(scst_cmd_thread, NULL, SCST_THREAD_FLAGS);
1054                 if (res < 0) {
1055                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
1056                         goto out_thread_free;
1057                 }
1058                 atomic_inc(&scst_threads_count);
1059         }
1060
1061         for (i = 0; i < sizeof(scst_tasklets) / sizeof(scst_tasklets[0]); i++)
1062                 tasklet_init(&scst_tasklets[i], (void *)scst_cmd_tasklet, 0);
1063
1064         res = kernel_thread(scst_mgmt_cmd_thread, NULL, SCST_THREAD_FLAGS);
1065         if (res < 0) {
1066                 PRINT_ERROR_PR("kernel_thread() for mcmd failed: %d", res);
1067                 goto out_thread_free;
1068         }
1069         atomic_inc(&scst_threads_count);
1070
1071         res = kernel_thread(scst_mgmt_thread, NULL, SCST_THREAD_FLAGS);
1072         if (res < 0) {
1073                 PRINT_ERROR_PR("kernel_thread() for mgmt failed: %d", res);
1074                 goto out_thread_free;
1075         }
1076         atomic_inc(&scst_threads_count);
1077
1078         if (scst_max_cmd_mem == 0) {
1079                 struct sysinfo si;
1080                 si_meminfo(&si);
1081 #if BITS_PER_LONG == 32
1082                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1083                                         (uint64_t)1 << 30);
1084 #else
1085                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1086 #endif
1087         } else
1088                 scst_max_cmd_mem <<= 20;
1089
1090         scst_cur_max_cmd_mem = scst_max_cmd_mem;
1091
1092         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1093                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1094
1095 out:
1096         TRACE_EXIT_RES(res);
1097         return res;
1098
1099 out_thread_free:
1100         if (atomic_read(&scst_threads_count)) {
1101                 DECLARE_MUTEX_LOCKED(shm);
1102                 scst_shutdown_mutex = &shm;
1103                 smp_mb();
1104                 set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1105
1106                 wake_up_all(&scst_list_waitQ);
1107                 wake_up_all(&scst_mgmt_cmd_list_waitQ);
1108                 wake_up_all(&scst_mgmt_waitQ);
1109
1110                 TRACE_DBG("Waiting for %d threads to complete",
1111                         atomic_read(&scst_threads_count));
1112                 down(&shm);
1113         }
1114
1115         scst_proc_cleanup_module();
1116
1117 out_unreg_interface:
1118         scsi_unregister_interface(&scst_interface);
1119
1120 out_free_acg:
1121         scst_destroy_acg(scst_default_acg);
1122
1123 out_destroy_sgv_pool:
1124         scst_sgv_pools_deinit(&scst_sgv);
1125
1126 out_destroy_ua_mempool:
1127         mempool_destroy(scst_ua_mempool);
1128
1129 out_destroy_mgmt_mempool:
1130         mempool_destroy(scst_mgmt_mempool);
1131
1132 out_destroy_acg_cache:
1133         kmem_cache_destroy(scst_acgd_cachep);
1134
1135 out_destroy_tgt_cache:
1136         kmem_cache_destroy(scst_tgtd_cachep);
1137
1138 out_destroy_sess_cache:
1139         kmem_cache_destroy(scst_sess_cachep);
1140
1141 out_destroy_cmd_cache:
1142         kmem_cache_destroy(scst_cmd_cachep);
1143
1144 out_destroy_ua_cache:
1145         kmem_cache_destroy(scst_ua_cachep);
1146
1147 out_destroy_mgmt_cache:
1148         kmem_cache_destroy(scst_mgmt_cachep);
1149         goto out;
1150 }
1151
1152 static void __exit exit_scst(void)
1153 {
1154 #ifdef CONFIG_LOCKDEP
1155         static /* To hide the lockdep's warning about non-static key */
1156 #endif
1157         DECLARE_MUTEX_LOCKED(shm);
1158
1159         TRACE_ENTRY();
1160
1161         /* ToDo: unregister_cpu_notifier() */
1162
1163         scst_shutdown_mutex = &shm;
1164         smp_mb();
1165         set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1166
1167         wake_up_all(&scst_list_waitQ);
1168         wake_up_all(&scst_mgmt_cmd_list_waitQ);
1169         wake_up_all(&scst_mgmt_waitQ);
1170
1171         if (atomic_read(&scst_threads_count)) {
1172                 TRACE_DBG("Waiting for %d threads to complete",
1173                       atomic_read(&scst_threads_count));
1174                 down(&shm);
1175
1176                 /*
1177                  * Wait for one sec. to allow the thread(s) actually exit,
1178                  * otherwise we can get Oops. Any better way?
1179                  */
1180                 {
1181                         unsigned long t = jiffies;
1182                         TRACE_DBG("%s", "Waiting 1 sec...");
1183                         while((jiffies - t) < HZ)
1184                                 schedule();
1185                 }
1186         }
1187
1188         if (test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
1189                 cancel_delayed_work(&scst_cmd_mem_work);
1190                 flush_scheduled_work();
1191         }
1192
1193         scst_proc_cleanup_module();
1194         scsi_unregister_interface(&scst_interface);
1195         scst_destroy_acg(scst_default_acg);
1196
1197         scst_sgv_pools_deinit(&scst_sgv);
1198
1199 #define DEINIT_CACHEP(p, s) do {                        \
1200                 if (kmem_cache_destroy(p)) {            \
1201                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1202                                 "error", s);            \
1203                 }                                       \
1204                 p = NULL;                               \
1205         } while (0)
1206
1207         mempool_destroy(scst_mgmt_mempool);
1208         mempool_destroy(scst_ua_mempool);
1209
1210         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1211         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1212         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1213         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1214         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1215         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1216
1217         PRINT_INFO_PR("%s", "SCST unloaded");
1218
1219         TRACE_EXIT();
1220         return;
1221 }
1222
1223 /*
1224  * Device Handler Side (i.e. scst_fileio)
1225  */
1226 EXPORT_SYMBOL(scst_register_dev_driver);
1227 EXPORT_SYMBOL(scst_unregister_dev_driver);
1228 EXPORT_SYMBOL(scst_register);
1229 EXPORT_SYMBOL(scst_unregister);
1230
1231 EXPORT_SYMBOL(scst_register_virtual_device);
1232 EXPORT_SYMBOL(scst_unregister_virtual_device);
1233 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1234 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1235
1236 EXPORT_SYMBOL(scst_set_busy);
1237 EXPORT_SYMBOL(scst_set_cmd_error_status);
1238 EXPORT_SYMBOL(scst_set_cmd_error);
1239 EXPORT_SYMBOL(scst_set_resp_data_len);
1240
1241 /*
1242  * Target Driver Side (i.e. HBA)
1243  */
1244 EXPORT_SYMBOL(scst_register_session);
1245 EXPORT_SYMBOL(scst_unregister_session);
1246
1247 EXPORT_SYMBOL(scst_register_target_template);
1248 EXPORT_SYMBOL(scst_unregister_target_template);
1249
1250 EXPORT_SYMBOL(scst_cmd_init_done);
1251 EXPORT_SYMBOL(scst_tgt_cmd_done);
1252 EXPORT_SYMBOL(scst_rx_cmd);
1253 EXPORT_SYMBOL(scst_rx_data);
1254 EXPORT_SYMBOL(scst_rx_mgmt_fn_tag);
1255 EXPORT_SYMBOL(scst_rx_mgmt_fn_lun);
1256
1257 EXPORT_SYMBOL(scst_find_cmd);
1258 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1259
1260 /*
1261  * Global Commands
1262  */
1263 EXPORT_SYMBOL(scst_suspend_activity);
1264 EXPORT_SYMBOL(scst_resume_activity);
1265
1266 EXPORT_SYMBOL(scst_add_threads);
1267 EXPORT_SYMBOL(scst_del_threads);
1268
1269 #if defined(DEBUG) || defined(TRACING)
1270 EXPORT_SYMBOL(scst_proc_log_entry_read);
1271 EXPORT_SYMBOL(scst_proc_log_entry_write);
1272 #endif
1273
1274 EXPORT_SYMBOL(__scst_get_buf);
1275 EXPORT_SYMBOL(scst_check_mem);
1276 EXPORT_SYMBOL(scst_get);
1277 EXPORT_SYMBOL(scst_put);
1278
1279 /*
1280  * Other Commands
1281  */
1282 EXPORT_SYMBOL(scst_get_cdb_info);
1283 EXPORT_SYMBOL(scst_cmd_get_tgt_specific_lock);
1284 EXPORT_SYMBOL(scst_cmd_set_tgt_specific_lock);
1285
1286 #ifdef DEBUG
1287 EXPORT_SYMBOL(scst_random);
1288 #endif
1289
1290 module_init(init_scst);
1291 module_exit(exit_scst);
1292
1293 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1294 MODULE_LICENSE("GPL");
1295 MODULE_DESCRIPTION("SCSI target core");
1296 MODULE_VERSION(SCST_VERSION_STRING);