be82e697a307e7cc985a1d84f5a95902a3856a71
[mirror/scst/.git] / scst / src / scst.c
1 /*
2  *  scst.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <asm/unistd.h>
28 #include <asm/string.h>
29 #include <linux/kthread.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
36 #warning HIGHMEM kernel configurations are fully supported, but not \
37         recommended for performance reasons. Consider change VMSPLIT \
38         option or use 64-bit configuration instead. See README file for \
39         details.
40 #endif
41
42 #ifdef SCST_HIGHMEM
43 #error SCST_HIGHMEM configuration isn't supported and broken, because there \
44         is no real point to support it, at least it definitely doesn't worth \
45         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
46         or in 64-bit configuration instead. See README file for details.
47 #endif
48
49 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
50 #warning Patch scst_exec_req_fifo.patch was not applied on your kernel and \
51         STRICT_SERIALIZING isn't defined. Pass-through dev handlers will \
52         not be supported.
53 #endif
54
55 /* All targets, devices and dev_types management is done under this mutex */
56 DECLARE_MUTEX(scst_mutex);
57
58 LIST_HEAD(scst_template_list);
59 LIST_HEAD(scst_dev_list);
60 LIST_HEAD(scst_dev_type_list);
61
62 spinlock_t scst_main_lock = SPIN_LOCK_UNLOCKED;
63
64 struct kmem_cache *scst_mgmt_cachep;
65 mempool_t *scst_mgmt_mempool;
66 struct kmem_cache *scst_ua_cachep;
67 mempool_t *scst_ua_mempool;
68 struct kmem_cache *scst_tgtd_cachep;
69 struct kmem_cache *scst_sess_cachep;
70 struct kmem_cache *scst_acgd_cachep;
71
72 LIST_HEAD(scst_acg_list);
73 struct scst_acg *scst_default_acg;
74
75 spinlock_t scst_init_lock = SPIN_LOCK_UNLOCKED;
76 DECLARE_WAIT_QUEUE_HEAD(scst_init_cmd_list_waitQ);
77 LIST_HEAD(scst_init_cmd_list);
78 unsigned int scst_init_poll_cnt;
79
80 struct kmem_cache *scst_cmd_cachep;
81
82 #if defined(DEBUG) || defined(TRACING)
83 unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
84 #endif
85
86 unsigned long scst_flags;
87 atomic_t scst_cmd_count = ATOMIC_INIT(0);
88
89 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
90 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
91 unsigned long scst_max_cmd_mem;
92
93 struct scst_sgv_pools scst_sgv;
94
95 struct scst_cmd_lists scst_main_cmd_lists;
96
97 struct scst_tasklet scst_tasklets[NR_CPUS];
98
99 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
100 DECLARE_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn, 0);
101 #else
102 DECLARE_DELAYED_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn);
103 #endif
104
105 spinlock_t scst_mcmd_lock = SPIN_LOCK_UNLOCKED;
106 LIST_HEAD(scst_active_mgmt_cmd_list);
107 LIST_HEAD(scst_delayed_mgmt_cmd_list);
108 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
109
110 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
111 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
112 LIST_HEAD(scst_sess_mgmt_list);
113
114 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
115
116 DECLARE_MUTEX(scst_suspend_mutex);
117 LIST_HEAD(scst_cmd_lists_list); /* protected by scst_suspend_mutex */
118
119 static int scst_threads;
120 struct scst_threads_info_t scst_threads_info;
121
122 static int suspend_count;
123
124 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
125
126 /* 
127  * This buffer and lock are intended to avoid memory allocation, which
128  * could fail in improper places.
129  */
130 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
131 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
132
133 module_param_named(scst_threads, scst_threads, int, 0);
134 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
135
136 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
137 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
138         "the SCST commands at any given time in Mb");
139
140 int scst_register_target_template(struct scst_tgt_template *vtt)
141 {
142         int res = 0;
143         struct scst_tgt_template *t;
144         static DECLARE_MUTEX(m);
145
146         TRACE_ENTRY();
147
148         INIT_LIST_HEAD(&vtt->tgt_list);
149
150         if (!vtt->detect) {
151                 PRINT_ERROR_PR("Target driver %s doesn't have a "
152                         "detect() method.", vtt->name);
153                 res = -EINVAL;
154                 goto out_err;
155         }
156         
157         if (!vtt->release) {
158                 PRINT_ERROR_PR("Target driver %s doesn't have a "
159                         "release() method.", vtt->name);
160                 res = -EINVAL;
161                 goto out_err;
162         }
163
164         if (!vtt->xmit_response) {
165                 PRINT_ERROR_PR("Target driver %s doesn't have a "
166                         "xmit_response() method.", vtt->name);
167                 res = -EINVAL;
168                 goto out_err;
169         }
170
171         if (vtt->threads_num < 0) {
172                 PRINT_ERROR_PR("Wrong threads_num value %d for "
173                         "target \"%s\"", vtt->threads_num,
174                         vtt->name);
175                 res = -EINVAL;
176                 goto out_err;
177         }
178
179         if (!vtt->no_proc_entry) {
180                 res = scst_build_proc_target_dir_entries(vtt);
181                 if (res < 0)
182                         goto out_err;
183         }
184
185         if (vtt->preprocessing_done == NULL)
186                 vtt->preprocessing_done_atomic = 1;
187
188         if (down_interruptible(&m) != 0)
189                 goto out_err;
190
191         if (down_interruptible(&scst_mutex) != 0)
192                 goto out_m_up;
193         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
194                 if (strcmp(t->name, vtt->name) == 0) {
195                         PRINT_ERROR_PR("Target driver %s already registered",
196                                 vtt->name);
197                         up(&scst_mutex);
198                         goto out_cleanup;
199                 }
200         }
201         up(&scst_mutex);
202
203         TRACE_DBG("%s", "Calling target driver's detect()");
204         res = vtt->detect(vtt);
205         TRACE_DBG("Target driver's detect() returned %d", res);
206         if (res < 0) {
207                 PRINT_ERROR_PR("%s", "The detect() routine failed");
208                 res = -EINVAL;
209                 goto out_cleanup;
210         }
211
212         down(&scst_mutex);
213         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
214         up(&scst_mutex);
215
216         res = 0;
217
218         PRINT_INFO_PR("Target template %s registered successfully", vtt->name);
219
220         up(&m);
221
222 out:
223         TRACE_EXIT_RES(res);
224         return res;
225
226 out_m_up:
227         up(&m);
228
229 out_cleanup:
230         scst_cleanup_proc_target_dir_entries(vtt);
231
232 out_err:
233         PRINT_ERROR_PR("Failed to register target template %s", vtt->name);
234         goto out;
235 }
236
237 void scst_unregister_target_template(struct scst_tgt_template *vtt)
238 {
239         struct scst_tgt *tgt;
240         struct scst_tgt_template *t;
241         int found = 0;
242
243         TRACE_ENTRY();
244
245         down(&scst_mutex);
246
247         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
248                 if (strcmp(t->name, vtt->name) == 0) {
249                         found = 1;
250                         break;
251                 }
252         }
253         if (!found) {
254                 PRINT_ERROR_PR("Target driver %s isn't registered", vtt->name);
255                 goto out_up;
256         }
257
258 restart:
259         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
260                 up(&scst_mutex);
261                 scst_unregister(tgt);
262                 down(&scst_mutex);
263                 goto restart;
264         }
265         list_del(&vtt->scst_template_list_entry);
266
267         PRINT_INFO_PR("Target template %s unregistered successfully", vtt->name);
268
269 out_up:
270         up(&scst_mutex);
271
272         scst_cleanup_proc_target_dir_entries(vtt);
273
274         TRACE_EXIT();
275         return;
276 }
277
278 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
279         const char *target_name)
280 {
281         struct scst_tgt *tgt;
282
283         TRACE_ENTRY();
284
285         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
286         if (tgt == NULL) {
287                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
288                 goto out_err;
289         }
290
291         INIT_LIST_HEAD(&tgt->sess_list);
292         init_waitqueue_head(&tgt->unreg_waitQ);
293         tgt->tgtt = vtt;
294         tgt->sg_tablesize = vtt->sg_tablesize;
295         spin_lock_init(&tgt->tgt_lock);
296         INIT_LIST_HEAD(&tgt->retry_cmd_list);
297         atomic_set(&tgt->finished_cmds, 0);
298         init_timer(&tgt->retry_timer);
299         tgt->retry_timer.data = (unsigned long)tgt;
300         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
301
302         scst_suspend_activity();
303         down(&scst_mutex);
304
305         if (target_name != NULL) {
306                 int len = strlen(target_name) + 1 +
307                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
308
309                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
310                 if (tgt->default_group_name == NULL) {
311                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
312                                 "group name failed");
313                         goto out_free_err;
314                 }
315                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
316                         target_name);
317         }
318
319         if (scst_build_proc_target_entries(tgt) < 0)
320                 goto out_free_name;
321         else
322                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
323
324         up(&scst_mutex);
325         scst_resume_activity();
326
327         PRINT_INFO_PR("Target %s for template %s registered successfully",
328                 target_name, vtt->name);
329
330 out:
331         TRACE_EXIT();
332         return tgt;
333
334 out_free_name:
335         if (tgt->default_group_name)
336                 kfree(tgt->default_group_name);
337
338 out_free_err:
339         up(&scst_mutex);
340         scst_resume_activity();
341
342         kfree(tgt);
343         tgt = NULL;
344
345 out_err:
346         PRINT_ERROR_PR("Failed to register target for template %s", vtt->name);
347         goto out;
348 }
349
350 static inline int test_sess_list(struct scst_tgt *tgt)
351 {
352         int res;
353         down(&scst_mutex);
354         res = list_empty(&tgt->sess_list);
355         up(&scst_mutex);
356         return res;
357 }
358
359 void scst_unregister(struct scst_tgt *tgt)
360 {
361         struct scst_session *sess;
362         struct scst_tgt_template *vtt = tgt->tgtt;
363
364         TRACE_ENTRY();
365
366         TRACE_DBG("%s", "Calling target driver's release()");
367         tgt->tgtt->release(tgt);
368         TRACE_DBG("%s", "Target driver's release() returned");
369
370         down(&scst_mutex);
371         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
372                 sBUG_ON(!sess->shutting_down);
373         }
374         up(&scst_mutex);
375
376         TRACE_DBG("%s", "Waiting for sessions shutdown");
377         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
378         TRACE_DBG("%s", "wait_event() returned");
379
380         scst_suspend_activity();
381         down(&scst_mutex);
382
383         list_del(&tgt->tgt_list_entry);
384
385         scst_cleanup_proc_target_entries(tgt);
386
387         if (tgt->default_group_name)
388                 kfree(tgt->default_group_name);
389
390         up(&scst_mutex);
391         scst_resume_activity();
392
393         del_timer_sync(&tgt->retry_timer);
394
395         kfree(tgt);
396
397         PRINT_INFO_PR("Target for template %s unregistered successfully",
398                 vtt->name);
399
400         TRACE_EXIT();
401         return;
402 }
403
404 void scst_suspend_activity(void)
405 {
406         TRACE_ENTRY();
407
408         down(&scst_suspend_mutex);
409
410         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
411         suspend_count++;
412         if (suspend_count > 1)
413                 goto out_up;
414
415         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
416         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
417         smp_mb__after_set_bit();
418
419         TRACE_MGMT_DBG("Waiting for %d active commands to complete",
420               atomic_read(&scst_cmd_count));
421         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
422         TRACE_MGMT_DBG("%s", "wait_event() returned");
423
424         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
425         smp_mb__after_clear_bit();
426
427         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
428               atomic_read(&scst_cmd_count));
429         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
430         TRACE_MGMT_DBG("%s", "wait_event() returned");
431
432 out_up:
433         up(&scst_suspend_mutex);
434
435         TRACE_EXIT();
436         return;
437 }
438
439 void scst_resume_activity(void)
440 {
441         struct scst_cmd_lists *l;
442
443         TRACE_ENTRY();
444
445         down(&scst_suspend_mutex);
446
447         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
448         suspend_count--;
449         if (suspend_count > 0)
450                 goto out_up;
451
452         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
453         smp_mb__after_clear_bit();
454
455         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
456                 wake_up_all(&l->cmd_list_waitQ);
457         }
458         wake_up_all(&scst_init_cmd_list_waitQ);
459
460         spin_lock_irq(&scst_mcmd_lock);
461         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
462                 struct scst_mgmt_cmd *m;
463                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
464                                 mgmt_cmd_list_entry);
465                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
466                         "mgmt cmd list", m);
467                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
468         }
469         spin_unlock_irq(&scst_mcmd_lock);
470         wake_up_all(&scst_mgmt_cmd_list_waitQ);
471
472 out_up:
473         up(&scst_suspend_mutex);
474
475         TRACE_EXIT();
476         return;
477 }
478
479 static int scst_register_device(struct scsi_device *scsidp)
480 {
481         int res = 0;
482         struct scst_device *dev;
483         struct scst_dev_type *dt;
484
485         TRACE_ENTRY();
486
487         scst_suspend_activity();
488         down(&scst_mutex);
489
490         res = scst_alloc_device(GFP_KERNEL, &dev);
491         if (res != 0)
492                 goto out_up;
493
494         dev->type = scsidp->type;
495
496         dev->rq_disk = alloc_disk(1);
497         if (dev->rq_disk == NULL) {
498                 res = -ENOMEM;
499                 goto out_free_dev;
500         }
501         dev->rq_disk->major = SCST_MAJOR;
502
503         dev->scsi_dev = scsidp;
504
505         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
506         
507         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
508                 if (dt->type == scsidp->type) {
509                         res = scst_assign_dev_handler(dev, dt);
510                         if (res != 0)
511                                 goto out_free;
512                         break;
513                 }
514         }
515
516 out_up:
517         up(&scst_mutex);
518         scst_resume_activity();
519
520         if (res == 0) {
521                 PRINT_INFO_PR("Attached SCSI target mid-level at "
522                     "scsi%d, channel %d, id %d, lun %d, type %d", 
523                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
524                     scsidp->lun, scsidp->type);
525         } 
526         else {
527                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
528                     "at scsi%d, channel %d, id %d, lun %d, type %d", 
529                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
530                     scsidp->lun, scsidp->type);
531         }
532
533         TRACE_EXIT_RES(res);
534         return res;
535
536 out_free:
537         list_del(&dev->dev_list_entry);
538         put_disk(dev->rq_disk);
539
540 out_free_dev:
541         scst_free_device(dev);
542         goto out_up;
543 }
544
545 static void scst_unregister_device(struct scsi_device *scsidp)
546 {
547         struct scst_device *d, *dev = NULL;
548         struct scst_acg_dev *acg_dev, *aa;
549
550         TRACE_ENTRY();
551         
552         scst_suspend_activity();
553         down(&scst_mutex);
554
555         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
556                 if (d->scsi_dev == scsidp) {
557                         dev = d;
558                         TRACE_DBG("Target device %p found", dev);
559                         break;
560                 }
561         }
562         if (dev == NULL) {
563                 PRINT_ERROR_PR("%s", "Target device not found");
564                 goto out_unblock;
565         }
566
567         list_del(&dev->dev_list_entry);
568         
569         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
570                                  dev_acg_dev_list_entry) 
571         {
572                 scst_acg_remove_dev(acg_dev->acg, dev);
573         }
574
575         scst_assign_dev_handler(dev, NULL);
576
577         put_disk(dev->rq_disk);
578         scst_free_device(dev);
579
580         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
581                 "id %d, lun %d, type %d", scsidp->host->host_no,
582                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
583
584 out_unblock:
585         up(&scst_mutex);
586         scst_resume_activity();
587
588         TRACE_EXIT();
589         return;
590 }
591
592 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
593 {
594         int res = 0;
595
596         if (dev_handler->parse == NULL) {
597                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
598                         "parse() method.", dev_handler->name);
599                 res = -EINVAL;
600                 goto out;
601         }
602
603         if (dev_handler->exec == NULL)
604                 dev_handler->exec_atomic = 1;
605
606         if (dev_handler->dev_done == NULL)
607                 dev_handler->dev_done_atomic = 1;
608
609 out:
610         TRACE_EXIT_RES(res);
611         return res;
612 }
613
614 int scst_register_virtual_device(struct scst_dev_type *dev_handler, 
615         const char *dev_name)
616 {
617         int res, rc;
618         struct scst_device *dev = NULL;
619
620         TRACE_ENTRY();
621         
622         if (dev_handler == NULL) {
623                 PRINT_ERROR_PR("%s: valid device handler must be supplied", 
624                         __FUNCTION__);
625                 res = -EINVAL;
626                 goto out;
627         }
628         
629         if (dev_name == NULL) {
630                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
631                 res = -EINVAL;
632                 goto out;
633         }
634
635         res = scst_dev_handler_check(dev_handler);
636         if (res != 0)
637                 goto out;
638
639         scst_suspend_activity();
640         if (down_interruptible(&scst_mutex) != 0) {
641                 res = -EINTR;
642                 goto out_resume;
643         }
644
645         res = scst_alloc_device(GFP_KERNEL, &dev);
646         if (res != 0)
647                 goto out_up;
648
649         dev->type = dev_handler->type;
650         dev->scsi_dev = NULL;
651         dev->virt_name = dev_name;
652         dev->virt_id = scst_virt_dev_last_id++;
653
654         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
655
656         res = dev->virt_id;
657
658         rc = scst_assign_dev_handler(dev, dev_handler);
659         if (rc != 0) {
660                 res = rc;
661                 goto out_free_del;
662         }
663
664 out_up:
665         up(&scst_mutex);
666
667 out_resume:
668         scst_resume_activity();
669
670 out:
671         if (res > 0) {
672                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
673                     "device %s (id %d)", dev_name, dev->virt_id);
674         } 
675         else {
676                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
677                     "virtual device %s", dev_name);
678         }
679
680         TRACE_EXIT_RES(res);
681         return res;
682
683 out_free_del:
684         list_del(&dev->dev_list_entry);
685         scst_free_device(dev);
686         goto out_up;
687 }
688
689 void scst_unregister_virtual_device(int id)
690 {
691         struct scst_device *d, *dev = NULL;
692         struct scst_acg_dev *acg_dev, *aa;
693
694         TRACE_ENTRY();
695
696         scst_suspend_activity();
697         down(&scst_mutex);
698
699         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
700                 if (d->virt_id == id) {
701                         dev = d;
702                         TRACE_DBG("Target device %p found", dev);
703                         break;
704                 }
705         }
706         if (dev == NULL) {
707                 PRINT_ERROR_PR("%s", "Target device not found");
708                 goto out_unblock;
709         }
710
711         list_del(&dev->dev_list_entry);
712         
713         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
714                                  dev_acg_dev_list_entry) 
715         {
716                 scst_acg_remove_dev(acg_dev->acg, dev);
717         }
718
719         scst_assign_dev_handler(dev, NULL);
720
721         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
722                 "(id %d)", dev->virt_name, dev->virt_id);
723
724         scst_free_device(dev);
725
726 out_unblock:
727         up(&scst_mutex);
728         scst_resume_activity();
729
730         TRACE_EXIT();
731         return;
732 }
733
734 int scst_register_dev_driver(struct scst_dev_type *dev_type)
735 {
736         struct scst_dev_type *dt;
737         struct scst_device *dev;
738         int res;
739         int exist;
740
741         TRACE_ENTRY();
742
743         res = scst_dev_handler_check(dev_type);
744         if (res != 0)
745                 goto out_err;
746
747 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
748         if (dev_type->exec == NULL) {
749                 PRINT_ERROR_PR("Pass-through dev handlers (handler \"%s\") not "
750                         "supported. Consider applying on your kernel patch "
751                         "scst_exec_req_fifo.patch or define STRICT_SERIALIZING",
752                         dev_type->name);
753                 res = -EINVAL;
754                 goto out_err;
755         }
756 #endif
757
758         scst_suspend_activity();
759         if (down_interruptible(&scst_mutex) != 0) {
760                 res = -EINTR;
761                 goto out_err;
762         }
763
764         exist = 0;
765         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
766                 if (strcmp(dt->name, dev_type->name) == 0) {
767                         PRINT_ERROR_PR("Device type handler \"%s\" already "
768                                 "exist", dt->name);
769                         exist = 1;
770                         break;
771                 }
772         }
773         if (exist)
774                 goto out_up;
775
776         res = scst_build_proc_dev_handler_dir_entries(dev_type);
777         if (res < 0) {
778                 goto out_up;
779         }
780
781         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
782
783         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
784                 if ((dev->scsi_dev == NULL) || (dev->handler != NULL))
785                         continue;
786                 if (dev->scsi_dev->type == dev_type->type)
787                         scst_assign_dev_handler(dev, dev_type);
788         }
789
790         up(&scst_mutex);
791         scst_resume_activity();
792
793         if (res == 0) {
794                 PRINT_INFO_PR("Device handler \"%s\" for type %d registered "
795                         "successfully", dev_type->name, dev_type->type);
796         }
797
798 out:
799         TRACE_EXIT_RES(res);
800         return res;
801
802 out_up:
803         up(&scst_mutex);
804
805 out_err:
806         scst_resume_activity();
807         PRINT_ERROR_PR("Failed to register device handler \"%s\" for type %d",
808                 dev_type->name, dev_type->type);
809         goto out;
810 }
811
812 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
813 {
814         struct scst_device *dev;
815         struct scst_dev_type *dt;
816         int found = 0;
817
818         TRACE_ENTRY();
819
820         scst_suspend_activity();
821         down(&scst_mutex);
822
823         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
824                 if (strcmp(dt->name, dev_type->name) == 0) {
825                         found = 1;
826                         break;
827                 }
828         }
829         if (!found) {
830                 PRINT_ERROR_PR("Dev handler \"%s\" isn't registered",
831                         dev_type->name);
832                 goto out_up;
833         }
834
835         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
836                 if (dev->handler == dev_type) {
837                         scst_assign_dev_handler(dev, NULL);
838                         TRACE_DBG("Dev handler removed from device %p", dev);
839                 }
840         }
841
842         list_del(&dev_type->dev_type_list_entry);
843
844         up(&scst_mutex);
845         scst_resume_activity();
846
847         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
848
849         PRINT_INFO_PR("Device handler \"%s\" for type %d unloaded",
850                    dev_type->name, dev_type->type);
851
852 out:
853         TRACE_EXIT();
854         return;
855
856 out_up:
857         up(&scst_mutex);
858         scst_resume_activity();
859         goto out;
860 }
861
862 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
863 {
864         int res;
865
866         TRACE_ENTRY();
867
868         res = scst_dev_handler_check(dev_type);
869         if (res != 0)
870                 goto out_err;
871
872         if (!dev_type->no_proc) {
873                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
874                 if (res < 0)
875                         goto out_err;
876         }
877
878         if (dev_type->type != -1) {
879                 PRINT_INFO_PR("Virtual device handler %s for type %d "
880                         "registered successfully", dev_type->name,
881                         dev_type->type);
882         } else {
883                 PRINT_INFO_PR("Virtual device handler \"%s\" registered "
884                         "successfully", dev_type->name);
885         }
886
887 out:
888         TRACE_EXIT_RES(res);
889         return res;
890
891 out_err:
892         PRINT_ERROR_PR("Failed to register virtual device handler \"%s\"",
893                 dev_type->name);
894         goto out;
895 }
896
897 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
898 {
899         TRACE_ENTRY();
900
901         if (!dev_type->no_proc)
902                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
903
904         PRINT_INFO_PR("Device handler \"%s\" unloaded", dev_type->name);
905
906         TRACE_EXIT();
907         return;
908 }
909
910 /* Called under scst_mutex and suspended activity */
911 int scst_add_dev_threads(struct scst_device *dev, int num)
912 {
913         int i, res = 0;
914         int n = 0;
915         struct scst_cmd_thread_t *thr;
916         char nm[12];
917
918         TRACE_ENTRY();
919
920         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
921                 n++;
922         }
923
924         for (i = 0; i < num; i++) {
925                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
926                 if (!thr) {
927                         res = -ENOMEM;
928                         PRINT_ERROR_PR("Failed to allocate thr %d", res);
929                         goto out;
930                 }
931                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
932                 nm[ARRAY_SIZE(nm)-1] = '\0';
933                 thr->cmd_thread = kthread_run(scst_cmd_thread,
934                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
935                 if (IS_ERR(thr->cmd_thread)) {
936                         res = PTR_ERR(thr->cmd_thread);
937                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
938                         kfree(thr);
939                         goto out;
940                 }
941                 list_add(&thr->thread_list_entry, &dev->threads_list);
942         }
943
944 out:
945         TRACE_EXIT_RES(res);
946         return res;
947 }
948
949 /* Called under scst_mutex and suspended activity */
950 static int scst_create_dev_threads(struct scst_device *dev)
951 {
952         int res = 0;
953         int threads_num;
954
955         TRACE_ENTRY();
956
957         if (dev->handler->threads_num <= 0)
958                 goto out;
959
960         threads_num = dev->handler->threads_num;
961
962         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
963         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
964         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
965
966         res = scst_add_dev_threads(dev, threads_num);
967         if (res != 0)
968                 goto out;
969
970         down(&scst_suspend_mutex);
971         list_add_tail(&dev->cmd_lists.lists_list_entry,
972                 &scst_cmd_lists_list);
973         up(&scst_suspend_mutex);
974
975         dev->p_cmd_lists = &dev->cmd_lists;
976
977 out:
978         TRACE_EXIT_RES(res);
979         return res;
980 }
981
982 /* Called under scst_mutex and suspended activity */
983 void scst_del_dev_threads(struct scst_device *dev, int num)
984 {
985         struct scst_cmd_thread_t *ct, *tmp;
986         int i = 0;
987
988         TRACE_ENTRY();
989
990         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
991                                 thread_list_entry) {
992                 int rc = kthread_stop(ct->cmd_thread);
993                 if (rc < 0) {
994                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
995                 }
996                 list_del(&ct->thread_list_entry);
997                 kfree(ct);
998                 if ((num >0) && (++i >= num))
999                         break;
1000         }
1001
1002         TRACE_EXIT();
1003         return;
1004 }
1005
1006 /* Called under scst_mutex and suspended activity */
1007 static void scst_stop_dev_threads(struct scst_device *dev)
1008 {
1009         TRACE_ENTRY();
1010
1011         if (list_empty(&dev->threads_list))
1012                 goto out;
1013
1014         scst_del_dev_threads(dev, -1);
1015
1016         if (dev->p_cmd_lists == &dev->cmd_lists) {
1017                 down(&scst_suspend_mutex);
1018                 list_del(&dev->cmd_lists.lists_list_entry);
1019                 up(&scst_suspend_mutex);
1020         }
1021
1022 out:
1023         TRACE_EXIT();
1024         return;
1025 }
1026
1027 /* The activity supposed to be suspended and scst_mutex held */
1028 int scst_assign_dev_handler(struct scst_device *dev, 
1029         struct scst_dev_type *handler)
1030 {
1031         int res = 0;
1032         struct scst_tgt_dev *tgt_dev;
1033         LIST_HEAD(attached_tgt_devs);
1034         
1035         TRACE_ENTRY();
1036         
1037         if (dev->handler == handler)
1038                 goto out;
1039         
1040         if (dev->handler && dev->handler->detach_tgt) {
1041                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1042                                 dev_tgt_dev_list_entry) {
1043                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1044                                 tgt_dev);
1045                         dev->handler->detach_tgt(tgt_dev);
1046                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1047                 }
1048         }
1049
1050         if (dev->handler && dev->handler->detach) {
1051                 TRACE_DBG("%s", "Calling dev handler's detach()");
1052                 dev->handler->detach(dev);
1053                 TRACE_DBG("%s", "Old handler's detach() returned");
1054         }
1055
1056         scst_stop_dev_threads(dev);
1057
1058         dev->handler = handler;
1059
1060         if (handler) {
1061                 res = scst_create_dev_threads(dev);
1062                 if (res != 0)
1063                         goto out_null;
1064         }
1065
1066         if (handler && handler->attach) {
1067                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1068                 res = handler->attach(dev);
1069                 TRACE_DBG("New dev handler's attach() returned %d", res);
1070                 if (res != 0) {
1071                         PRINT_ERROR_PR("New device handler's %s attach() "
1072                                 "failed: %d", handler->name, res);
1073                 }
1074                 goto out_thr_null;
1075         }
1076         
1077         if (handler && handler->attach_tgt) {
1078                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1079                                 dev_tgt_dev_list_entry) {
1080                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1081                                 tgt_dev);
1082                         res = handler->attach_tgt(tgt_dev);
1083                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1084                         if (res != 0) {
1085                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
1086                                     "failed: %d", handler->name, res);
1087                                 goto out_err_detach_tgt;
1088                         }
1089                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1090                                 &attached_tgt_devs);
1091                 }
1092         }
1093
1094 out_thr_null:
1095         if (res != 0)
1096                 scst_stop_dev_threads(dev);
1097
1098 out_null:
1099         if (res != 0)
1100                 dev->handler = NULL;
1101         
1102 out:
1103         TRACE_EXIT_RES(res);
1104         return res;
1105
1106 out_err_detach_tgt:
1107         if (handler && handler->detach_tgt) {
1108                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1109                                  extra_tgt_dev_list_entry) 
1110                 {
1111                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1112                                 tgt_dev);
1113                         handler->detach_tgt(tgt_dev);
1114                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1115                 }
1116         }
1117         if (handler && handler->detach) {
1118                 TRACE_DBG("%s", "Calling handler's detach()");
1119                 handler->detach(dev);
1120                 TRACE_DBG("%s", "Handler's detach() returned");
1121         }
1122         goto out_null;
1123 }
1124
1125 int scst_cmd_threads_count(void)
1126 {
1127         int i;
1128
1129         /* Just to lower the race window, when user can get just changed value */
1130         down(&scst_threads_info.cmd_threads_mutex);
1131         i = scst_threads_info.nr_cmd_threads;
1132         up(&scst_threads_info.cmd_threads_mutex);
1133         return i;
1134 }
1135
1136 static void scst_threads_info_init(void)
1137 {
1138         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1139         init_MUTEX(&scst_threads_info.cmd_threads_mutex);
1140         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1141 }
1142
1143 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1144 void __scst_del_cmd_threads(int num)
1145 {
1146         struct scst_cmd_thread_t *ct, *tmp;
1147         int i;
1148
1149         TRACE_ENTRY();
1150
1151         i = scst_threads_info.nr_cmd_threads;
1152         if (num <= 0 || num > i) {
1153                 PRINT_ERROR_PR("can not del %d cmd threads from %d", num, i);
1154                 return;
1155         }
1156
1157         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1158                                 thread_list_entry) {
1159                 int res;
1160
1161                 res = kthread_stop(ct->cmd_thread);
1162                 if (res < 0) {
1163                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1164                 }
1165                 list_del(&ct->thread_list_entry);
1166                 kfree(ct);
1167                 scst_threads_info.nr_cmd_threads--;
1168                 --num;
1169                 if (num == 0)
1170                         break;
1171         }
1172
1173         TRACE_EXIT();
1174         return;
1175 }
1176
1177 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1178 int __scst_add_cmd_threads(int num)
1179 {
1180         int res = 0, i;
1181         static int scst_thread_num = 0;
1182         
1183         TRACE_ENTRY();
1184
1185         for (i = 0; i < num; i++) {
1186                 struct scst_cmd_thread_t *thr;
1187
1188                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1189                 if (!thr) {
1190                         res = -ENOMEM;
1191                         PRINT_ERROR_PR("fail to allocate thr %d", res);
1192                         goto out_error;
1193                 }
1194                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1195                         &scst_main_cmd_lists, "scsi_tgt%d",
1196                         scst_thread_num++);
1197                 if (IS_ERR(thr->cmd_thread)) {
1198                         res = PTR_ERR(thr->cmd_thread);
1199                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
1200                         kfree(thr);
1201                         goto out_error;
1202                 }
1203                 list_add(&thr->thread_list_entry,
1204                         &scst_threads_info.cmd_threads_list);
1205                 scst_threads_info.nr_cmd_threads++;
1206         }
1207         res = 0;
1208
1209 out:
1210         TRACE_EXIT_RES(res);
1211         return res;
1212
1213 out_error:
1214         if (i > 0)
1215                 __scst_del_cmd_threads(i - 1);
1216         goto out;
1217 }
1218
1219 int scst_add_cmd_threads(int num)
1220 {
1221         int res;
1222
1223         TRACE_ENTRY();
1224
1225         down(&scst_threads_info.cmd_threads_mutex);
1226         res = __scst_add_cmd_threads(num);
1227         up(&scst_threads_info.cmd_threads_mutex);
1228
1229         TRACE_EXIT_RES(res);
1230         return res;
1231 }
1232
1233 void scst_del_cmd_threads(int num)
1234 {
1235         TRACE_ENTRY();
1236
1237         down(&scst_threads_info.cmd_threads_mutex);
1238         __scst_del_cmd_threads(num);
1239         up(&scst_threads_info.cmd_threads_mutex);
1240
1241         TRACE_EXIT();
1242         return;
1243 }
1244
1245 static void scst_stop_all_threads(void)
1246 {
1247         TRACE_ENTRY();
1248
1249         down(&scst_threads_info.cmd_threads_mutex);
1250         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1251         if (scst_threads_info.mgmt_cmd_thread)
1252                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1253         if (scst_threads_info.mgmt_thread)
1254                 kthread_stop(scst_threads_info.mgmt_thread);
1255         if (scst_threads_info.init_cmd_thread)
1256                 kthread_stop(scst_threads_info.init_cmd_thread);
1257         up(&scst_threads_info.cmd_threads_mutex);
1258
1259         TRACE_EXIT();
1260         return;
1261 }
1262
1263 static int scst_start_all_threads(int num)
1264 {
1265         int res;
1266
1267         TRACE_ENTRY();
1268
1269         down(&scst_threads_info.cmd_threads_mutex);             
1270         res = __scst_add_cmd_threads(num);
1271         if (res < 0)
1272                 goto out;
1273
1274         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1275                 NULL, "scsi_tgt_init");
1276         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1277                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1278                 PRINT_ERROR_PR("kthread_create() for init cmd failed: %d", res);
1279                 scst_threads_info.init_cmd_thread = NULL;
1280                 goto out;
1281         }
1282
1283         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1284                 NULL, "scsi_tgt_mc");
1285         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1286                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1287                 PRINT_ERROR_PR("kthread_create() for mcmd failed: %d", res);
1288                 scst_threads_info.mgmt_cmd_thread = NULL;
1289                 goto out;
1290         }
1291
1292         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1293                 NULL, "scsi_tgt_mgmt");
1294         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1295                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1296                 PRINT_ERROR_PR("kthread_create() for mgmt failed: %d", res);
1297                 scst_threads_info.mgmt_thread = NULL;
1298                 goto out;
1299         }
1300
1301 out:
1302         up(&scst_threads_info.cmd_threads_mutex);
1303         TRACE_EXIT_RES(res);
1304         return res;     
1305 }
1306
1307 void scst_get(void)
1308 {
1309         __scst_get(0);
1310 }
1311
1312 void scst_put(void)
1313 {
1314         __scst_put();
1315 }
1316
1317 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1318 static int scst_add(struct class_device *cdev)
1319 #else
1320 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1321 #endif
1322 {
1323         struct scsi_device *scsidp;
1324         int res = 0;
1325
1326         TRACE_ENTRY();
1327         
1328         scsidp = to_scsi_device(cdev->dev);
1329         res = scst_register_device(scsidp);
1330
1331         TRACE_EXIT();
1332         return res;
1333 }
1334
1335 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1336 static void scst_remove(struct class_device *cdev)
1337 #else
1338 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1339 #endif
1340 {
1341         struct scsi_device *scsidp;
1342
1343         TRACE_ENTRY();
1344
1345         scsidp = to_scsi_device(cdev->dev);
1346         scst_unregister_device(scsidp);
1347
1348         TRACE_EXIT();
1349         return;
1350 }
1351
1352 static struct class_interface scst_interface = {
1353         .add = scst_add,
1354         .remove = scst_remove,
1355 };
1356
1357 static int __init init_scst(void)
1358 {
1359         int res = 0, i;
1360         struct scst_cmd *cmd;
1361         int scst_num_cpus;
1362
1363         TRACE_ENTRY();
1364
1365 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1366         {
1367                 struct scsi_request *req;
1368                 BUILD_BUG_ON(sizeof(cmd->sense_buffer) !=
1369                         sizeof(req->sr_sense_buffer));
1370         }
1371 #else
1372         {
1373                 struct scsi_sense_hdr *shdr;
1374                 BUILD_BUG_ON((sizeof(cmd->sense_buffer) < sizeof(*shdr)) &&
1375                         (sizeof(cmd->sense_buffer) >= SCST_SENSE_BUFFERSIZE));
1376         }
1377 #endif
1378         {
1379                 struct scst_tgt_dev *t;
1380                 struct scst_cmd *c;
1381                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1382                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1383         }
1384
1385         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1386         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1387         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1388         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1389
1390         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1391         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1392         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1393         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1394                 &scst_cmd_lists_list);
1395
1396         scst_num_cpus = num_online_cpus();
1397
1398         /* ToDo: register_cpu_notifier() */
1399         
1400         if (scst_threads == 0)
1401                 scst_threads = scst_num_cpus;
1402                 
1403         if (scst_threads < scst_num_cpus) {
1404                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
1405                         "CPUs count");
1406                 scst_threads = scst_num_cpus;
1407         }
1408
1409         scst_threads_info_init();
1410
1411 #define INIT_CACHEP(p, s, t, o) do {                                    \
1412                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
1413                                       SCST_SLAB_FLAGS, NULL, NULL);     \
1414                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
1415                           sizeof(struct t));                            \
1416                 if (p == NULL) { res = -ENOMEM; goto o; }               \
1417         } while (0)
1418           
1419         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING, 
1420                     scst_mgmt_cmd, out);
1421         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING, 
1422                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
1423         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING, 
1424                     scst_cmd, out_destroy_ua_cache);
1425         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
1426                     scst_session, out_destroy_cmd_cache);
1427         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
1428                     scst_tgt_dev, out_destroy_sess_cache);
1429         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
1430                     scst_acg_dev, out_destroy_tgt_cache);
1431
1432         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
1433                 mempool_free_slab, scst_mgmt_cachep);
1434         if (scst_mgmt_mempool == NULL) {
1435                 res = -ENOMEM;
1436                 goto out_destroy_acg_cache;
1437         }
1438
1439         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1440                 mempool_free_slab, scst_ua_cachep);
1441         if (scst_ua_mempool == NULL) {
1442                 res = -ENOMEM;
1443                 goto out_destroy_mgmt_mempool;
1444         }
1445
1446         res = scst_sgv_pools_init(&scst_sgv);
1447         if (res != 0)
1448                 goto out_destroy_ua_mempool;
1449
1450         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1451         if (scst_default_acg == NULL) {
1452                 res = -ENOMEM;
1453                 goto out_destroy_sgv_pool;
1454         }
1455         
1456         res = scsi_register_interface(&scst_interface);
1457         if (res != 0)
1458                 goto out_free_acg;
1459
1460         scst_scsi_op_list_init();
1461
1462         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1463                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1464                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1465                 tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet,
1466                         (unsigned long)&scst_tasklets[i]);
1467         }
1468
1469         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1470                 scst_threads);
1471
1472         res = scst_start_all_threads(scst_threads);
1473         if (res < 0)
1474                 goto out_thread_free;
1475
1476         res = scst_proc_init_module();
1477         if (res != 0)
1478                 goto out_thread_free;
1479
1480         if (scst_max_cmd_mem == 0) {
1481                 struct sysinfo si;
1482                 si_meminfo(&si);
1483 #if BITS_PER_LONG == 32
1484                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1485                                         (uint64_t)1 << 30);
1486 #else
1487                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1488 #endif
1489         } else
1490                 scst_max_cmd_mem <<= 20;
1491
1492         scst_cur_max_cmd_mem = scst_max_cmd_mem;
1493
1494         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1495                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1496
1497 out:
1498         TRACE_EXIT_RES(res);
1499         return res;
1500
1501 out_thread_free:
1502         scst_stop_all_threads();
1503
1504         scsi_unregister_interface(&scst_interface);
1505
1506 out_free_acg:
1507         scst_destroy_acg(scst_default_acg);
1508
1509 out_destroy_sgv_pool:
1510         scst_sgv_pools_deinit(&scst_sgv);
1511
1512 out_destroy_ua_mempool:
1513         mempool_destroy(scst_ua_mempool);
1514
1515 out_destroy_mgmt_mempool:
1516         mempool_destroy(scst_mgmt_mempool);
1517
1518 out_destroy_acg_cache:
1519         kmem_cache_destroy(scst_acgd_cachep);
1520
1521 out_destroy_tgt_cache:
1522         kmem_cache_destroy(scst_tgtd_cachep);
1523
1524 out_destroy_sess_cache:
1525         kmem_cache_destroy(scst_sess_cachep);
1526
1527 out_destroy_cmd_cache:
1528         kmem_cache_destroy(scst_cmd_cachep);
1529
1530 out_destroy_ua_cache:
1531         kmem_cache_destroy(scst_ua_cachep);
1532
1533 out_destroy_mgmt_cache:
1534         kmem_cache_destroy(scst_mgmt_cachep);
1535         goto out;
1536 }
1537
1538 static void __exit exit_scst(void)
1539 {
1540 #ifdef CONFIG_LOCKDEP
1541         static /* To hide the lockdep's warning about non-static key */
1542 #endif
1543         DECLARE_MUTEX_LOCKED(shm);
1544
1545         TRACE_ENTRY();
1546         
1547         /* ToDo: unregister_cpu_notifier() */
1548
1549         if (test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
1550                 cancel_delayed_work(&scst_cmd_mem_work);
1551                 flush_scheduled_work();
1552         }
1553
1554         scst_proc_cleanup_module();
1555
1556         scst_stop_all_threads();
1557
1558         scsi_unregister_interface(&scst_interface);
1559         scst_destroy_acg(scst_default_acg);
1560
1561         scst_sgv_pools_deinit(&scst_sgv);
1562
1563 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
1564 #define DEINIT_CACHEP(p, s) do {                        \
1565                 if (kmem_cache_destroy(p)) {            \
1566                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1567                                 "error", s);            \
1568                 }                                       \
1569                 p = NULL;                               \
1570         } while (0)
1571 #else
1572 #define DEINIT_CACHEP(p, s) do {                        \
1573                 kmem_cache_destroy(p);                  \
1574                 p = NULL;                               \
1575         } while (0)
1576 #endif
1577
1578         mempool_destroy(scst_mgmt_mempool);
1579         mempool_destroy(scst_ua_mempool);
1580
1581         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1582         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1583         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1584         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1585         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1586         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1587
1588         PRINT_INFO_PR("%s", "SCST unloaded");
1589
1590         TRACE_EXIT();
1591         return;
1592 }
1593
1594 /*
1595  * Device Handler Side (i.e. scst_vdisk)
1596  */
1597 EXPORT_SYMBOL(scst_register_dev_driver);
1598 EXPORT_SYMBOL(scst_unregister_dev_driver);
1599 EXPORT_SYMBOL(scst_register);
1600 EXPORT_SYMBOL(scst_unregister);
1601
1602 EXPORT_SYMBOL(scst_register_virtual_device);
1603 EXPORT_SYMBOL(scst_unregister_virtual_device);
1604 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1605 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1606
1607 EXPORT_SYMBOL(scst_set_busy);
1608 EXPORT_SYMBOL(scst_set_cmd_error_status);
1609 EXPORT_SYMBOL(scst_set_cmd_error);
1610 EXPORT_SYMBOL(scst_set_resp_data_len);
1611
1612 EXPORT_SYMBOL(scst_process_active_cmd);
1613
1614 /*
1615  * Target Driver Side (i.e. HBA)
1616  */
1617 EXPORT_SYMBOL(scst_register_session);
1618 EXPORT_SYMBOL(scst_unregister_session);
1619
1620 EXPORT_SYMBOL(scst_register_target_template);
1621 EXPORT_SYMBOL(scst_unregister_target_template);
1622
1623 EXPORT_SYMBOL(scst_cmd_init_done);
1624 EXPORT_SYMBOL(scst_tgt_cmd_done);
1625 EXPORT_SYMBOL(scst_restart_cmd);
1626 EXPORT_SYMBOL(scst_rx_cmd);
1627 EXPORT_SYMBOL(scst_rx_data);
1628 EXPORT_SYMBOL(scst_rx_mgmt_fn_tag);
1629 EXPORT_SYMBOL(scst_rx_mgmt_fn_lun);
1630
1631 EXPORT_SYMBOL(scst_find_cmd);
1632 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1633
1634 /*
1635  * Global Commands
1636  */
1637 EXPORT_SYMBOL(scst_suspend_activity);
1638 EXPORT_SYMBOL(scst_resume_activity);
1639
1640 EXPORT_SYMBOL(scst_add_cmd_threads);
1641 EXPORT_SYMBOL(scst_del_cmd_threads);
1642
1643 #if defined(DEBUG) || defined(TRACING)
1644 EXPORT_SYMBOL(scst_proc_log_entry_read);
1645 EXPORT_SYMBOL(scst_proc_log_entry_write);
1646 #endif
1647
1648 EXPORT_SYMBOL(scst_create_proc_entry);
1649 EXPORT_SYMBOL(scst_single_seq_open);
1650
1651 EXPORT_SYMBOL(__scst_get_buf);
1652 EXPORT_SYMBOL(scst_check_mem);
1653 EXPORT_SYMBOL(scst_get);
1654 EXPORT_SYMBOL(scst_put);
1655
1656 EXPORT_SYMBOL(scst_alloc);
1657 EXPORT_SYMBOL(scst_free);
1658
1659 /* Tgt_dev's threads local storage */
1660 EXPORT_SYMBOL(scst_add_thr_data);
1661 EXPORT_SYMBOL(scst_del_all_thr_data);
1662 EXPORT_SYMBOL(scst_dev_del_all_thr_data);
1663 EXPORT_SYMBOL(scst_find_thr_data);
1664
1665 /* SGV pool routines */
1666 EXPORT_SYMBOL(sgv_pool_create);
1667 EXPORT_SYMBOL(sgv_pool_destroy);
1668 EXPORT_SYMBOL(sgv_pool_set_allocator);
1669 EXPORT_SYMBOL(sgv_pool_alloc);
1670 EXPORT_SYMBOL(sgv_pool_free);
1671 EXPORT_SYMBOL(sgv_get_priv);
1672
1673 /* Generic parse() routines */
1674 EXPORT_SYMBOL(scst_calc_block_shift);
1675 EXPORT_SYMBOL(scst_sbc_generic_parse);
1676 EXPORT_SYMBOL(scst_cdrom_generic_parse);
1677 EXPORT_SYMBOL(scst_modisk_generic_parse);
1678 EXPORT_SYMBOL(scst_tape_generic_parse);
1679 EXPORT_SYMBOL(scst_changer_generic_parse);
1680 EXPORT_SYMBOL(scst_processor_generic_parse);
1681 EXPORT_SYMBOL(scst_raid_generic_parse);
1682
1683 /* Generic dev_done() routines */
1684 EXPORT_SYMBOL(scst_block_generic_dev_done);
1685 EXPORT_SYMBOL(scst_tape_generic_dev_done);
1686
1687 /*
1688  * Other Commands
1689  */
1690 EXPORT_SYMBOL(scst_get_cdb_info);
1691 EXPORT_SYMBOL(scst_cmd_get_tgt_priv_lock);
1692 EXPORT_SYMBOL(scst_cmd_set_tgt_priv_lock);
1693
1694 #ifdef DEBUG
1695 EXPORT_SYMBOL(scst_random);
1696 #endif
1697
1698 module_init(init_scst);
1699 module_exit(exit_scst);
1700
1701 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1702 MODULE_LICENSE("GPL");
1703 MODULE_DESCRIPTION("SCSI target core");
1704 MODULE_VERSION(SCST_VERSION_STRING);