Patch from Alexey Obitotskiy <alexeyo1@open-e.com> with 2 fixes and cleanups implemen...
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/module.h>
20
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/list.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/sched.h>
28 #include <linux/unistd.h>
29 #include <linux/string.h>
30 #include <linux/kthread.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34 #include "scst_mem.h"
35
36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
37 #warning "HIGHMEM kernel configurations are fully supported, but not\
38  recommended for performance reasons. Consider changing VMSPLIT\
39  option or use a 64-bit configuration instead. See README file for\
40  details."
41 #endif
42
43 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
44 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
45 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
46 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
47  your kernel and CONFIG_SCST_STRICT_SERIALIZING isn't defined.\
48  Pass-through dev handlers will not work."
49 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
50 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
51 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
52  your kernel. Pass-through dev handlers will not work."
53 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
54 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
55
56 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
57 #if !defined(SCST_IO_CONTEXT)
58 #warning "Patch io_context-<kernel-version> was not applied\
59  on your kernel. SCST will be working with not the best performance."
60 #endif
61 #else
62 #warning "There is no patch io_context-<kernel-version>\
63  for your kernel version. For performance reasons it is strongly recommended\
64  to upgrade your kernel to version >= 2.6.26.x."
65 #endif
66
67 /**
68  ** SCST global variables. They are all uninitialized to have their layout in
69  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
70  ** variable separately from nonzero-initialized ones.
71  **/
72
73 /*
74  * All targets, devices and dev_types management is done under this mutex.
75  *
76  * It must NOT be used in any works (schedule_work(), etc.), because
77  * otherwise a deadlock (double lock, actually) is possible, e.g., with
78  * scst_user detach_tgt(), which is called under scst_mutex and calls
79  * flush_scheduled_work().
80  */
81 struct mutex scst_mutex;
82 EXPORT_SYMBOL(scst_mutex);
83
84  /* All 3 protected by scst_mutex */
85 struct list_head scst_template_list;
86 struct list_head scst_dev_list;
87 struct list_head scst_dev_type_list;
88
89 spinlock_t scst_main_lock;
90
91 static struct kmem_cache *scst_mgmt_cachep;
92 mempool_t *scst_mgmt_mempool;
93 static struct kmem_cache *scst_mgmt_stub_cachep;
94 mempool_t *scst_mgmt_stub_mempool;
95 static struct kmem_cache *scst_ua_cachep;
96 mempool_t *scst_ua_mempool;
97 static struct kmem_cache *scst_sense_cachep;
98 mempool_t *scst_sense_mempool;
99 static struct kmem_cache *scst_aen_cachep;
100 mempool_t *scst_aen_mempool;
101 struct kmem_cache *scst_tgtd_cachep;
102 struct kmem_cache *scst_sess_cachep;
103 struct kmem_cache *scst_acgd_cachep;
104
105 #ifdef CONFIG_SCST_PROC
106 struct list_head scst_acg_list;
107 struct scst_acg *scst_default_acg;
108 #endif
109
110 spinlock_t scst_init_lock;
111 wait_queue_head_t scst_init_cmd_list_waitQ;
112 struct list_head scst_init_cmd_list;
113 unsigned int scst_init_poll_cnt;
114
115 struct kmem_cache *scst_cmd_cachep;
116
117 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
118 unsigned long scst_trace_flag;
119 #endif
120
121 unsigned long scst_flags;
122 atomic_t scst_cmd_count;
123
124 struct scst_cmd_lists scst_main_cmd_lists;
125
126 struct scst_tasklet scst_tasklets[NR_CPUS];
127
128 spinlock_t scst_mcmd_lock;
129 struct list_head scst_active_mgmt_cmd_list;
130 struct list_head scst_delayed_mgmt_cmd_list;
131 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
132
133 wait_queue_head_t scst_mgmt_waitQ;
134 spinlock_t scst_mgmt_lock;
135 struct list_head scst_sess_init_list;
136 struct list_head scst_sess_shut_list;
137
138 wait_queue_head_t scst_dev_cmd_waitQ;
139
140 static struct mutex scst_suspend_mutex;
141 /* protected by scst_suspend_mutex */
142 static struct list_head scst_cmd_lists_list;
143
144 static int scst_threads;
145 struct mutex scst_global_threads_mutex;
146 u32 scst_nr_global_threads;
147 static struct list_head scst_global_threads_list;
148 static struct task_struct *scst_init_cmd_thread;
149 static struct task_struct *scst_mgmt_thread;
150 static struct task_struct *scst_mgmt_cmd_thread;
151
152 static int suspend_count;
153
154 static int scst_virt_dev_last_id; /* protected by scst_mutex */
155
156 static unsigned int scst_max_cmd_mem;
157 unsigned int scst_max_dev_cmd_mem;
158
159 module_param_named(scst_threads, scst_threads, int, 0);
160 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
161
162 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, S_IRUGO);
163 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
164         "all SCSI commands of all devices at any given time in MB");
165
166 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, S_IRUGO);
167 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
168         "by all SCSI commands of a device at any given time in MB");
169
170 struct scst_dev_type scst_null_devtype = {
171         .name = "none",
172 };
173
174 static void __scst_resume_activity(void);
175
176 int __scst_register_target_template(struct scst_tgt_template *vtt,
177         const char *version)
178 {
179         int res = 0;
180         struct scst_tgt_template *t;
181         static DEFINE_MUTEX(m);
182
183         TRACE_ENTRY();
184
185         INIT_LIST_HEAD(&vtt->tgt_list);
186
187         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
188                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
189                 res = -EINVAL;
190                 goto out_err;
191         }
192
193         if (!vtt->detect) {
194                 PRINT_ERROR("Target driver %s must have "
195                         "detect() method.", vtt->name);
196                 res = -EINVAL;
197                 goto out_err;
198         }
199
200         if (!vtt->release) {
201                 PRINT_ERROR("Target driver %s must have "
202                         "release() method.", vtt->name);
203                 res = -EINVAL;
204                 goto out_err;
205         }
206
207         if (!vtt->xmit_response) {
208                 PRINT_ERROR("Target driver %s must have "
209                         "xmit_response() method.", vtt->name);
210                 res = -EINVAL;
211                 goto out_err;
212         }
213
214         if (vtt->threads_num < 0) {
215                 PRINT_ERROR("Wrong threads_num value %d for "
216                         "target \"%s\"", vtt->threads_num,
217                         vtt->name);
218                 res = -EINVAL;
219                 goto out_err;
220         }
221
222 #ifndef CONFIG_SCST_PROC
223         if (!vtt->enable_tgt || !vtt->is_tgt_enabled) {
224                 PRINT_WARNING("Target driver %s doesn't have enable_tgt() "
225                         "and/or is_tgt_enabled() method(s). This is unsafe "
226                         "and can lead that initiators connected on the "
227                         "initialization time can see an unexpected set of "
228                         "devices or no devices at all!", vtt->name);
229         }
230 #else
231         if (!vtt->no_proc_entry) {
232                 res = scst_build_proc_target_dir_entries(vtt);
233                 if (res < 0)
234                         goto out_err;
235         }
236 #endif
237
238         res = scst_create_tgtt_sysfs(vtt);
239         if (res)
240                 goto out_sysfs_err;
241
242         if (vtt->rdy_to_xfer == NULL)
243                 vtt->rdy_to_xfer_atomic = 1;
244
245         if (mutex_lock_interruptible(&m) != 0)
246                 goto out_sysfs_err;
247
248         if (mutex_lock_interruptible(&scst_mutex) != 0)
249                 goto out_m_err;
250         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
251                 if (strcmp(t->name, vtt->name) == 0) {
252                         PRINT_ERROR("Target driver %s already registered",
253                                 vtt->name);
254                         mutex_unlock(&scst_mutex);
255                         goto out_m_err;
256                 }
257         }
258         mutex_unlock(&scst_mutex);
259
260         TRACE_DBG("%s", "Calling target driver's detect()");
261         res = vtt->detect(vtt);
262         TRACE_DBG("Target driver's detect() returned %d", res);
263         if (res < 0) {
264                 PRINT_ERROR("%s", "The detect() routine failed");
265                 res = -EINVAL;
266                 goto out_m_err;
267         }
268
269         mutex_lock(&scst_mutex);
270         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
271         mutex_unlock(&scst_mutex);
272
273         res = 0;
274
275         PRINT_INFO("Target template %s registered successfully", vtt->name);
276
277         mutex_unlock(&m);
278
279 out:
280         TRACE_EXIT_RES(res);
281         return res;
282
283 out_m_err:
284         mutex_unlock(&m);
285
286 out_sysfs_err:
287 #ifdef CONFIG_SCST_PROC
288         scst_cleanup_proc_target_dir_entries(vtt);
289 #endif
290         scst_tgtt_sysfs_put(vtt);
291
292 out_err:
293         PRINT_ERROR("Failed to register target template %s", vtt->name);
294         goto out;
295 }
296 EXPORT_SYMBOL(__scst_register_target_template);
297
298 void scst_unregister_target_template(struct scst_tgt_template *vtt)
299 {
300         struct scst_tgt *tgt;
301         struct scst_tgt_template *t;
302         int found = 0;
303
304         TRACE_ENTRY();
305
306         mutex_lock(&scst_mutex);
307
308         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
309                 if (strcmp(t->name, vtt->name) == 0) {
310                         found = 1;
311                         break;
312                 }
313         }
314         if (!found) {
315                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
316                 goto out_err_up;
317         }
318
319 restart:
320         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
321                 mutex_unlock(&scst_mutex);
322                 scst_unregister(tgt);
323                 mutex_lock(&scst_mutex);
324                 goto restart;
325         }
326         list_del(&vtt->scst_template_list_entry);
327
328         mutex_unlock(&scst_mutex);
329
330 #ifdef CONFIG_SCST_PROC
331         scst_cleanup_proc_target_dir_entries(vtt);
332 #endif
333
334         scst_tgtt_sysfs_put(vtt);
335
336         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
337
338 out:
339         TRACE_EXIT();
340         return;
341
342 out_err_up:
343         mutex_unlock(&scst_mutex);
344         goto out;
345 }
346 EXPORT_SYMBOL(scst_unregister_target_template);
347
348 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
349         const char *target_name)
350 {
351         struct scst_tgt *tgt;
352         int rc = 0;
353
354         TRACE_ENTRY();
355
356         tgt = scst_alloc_tgt(vtt);
357         if (tgt == NULL) {
358                 rc = -ENOMEM;
359                 goto out_err;
360         }
361
362         rc = scst_suspend_activity(true);
363         if (rc != 0)
364                 goto out_free_tgt_err;
365
366         if (mutex_lock_interruptible(&scst_mutex) != 0) {
367                 rc = -EINTR;
368                 goto out_resume_free;
369         }
370
371         if (target_name != NULL) {
372 #ifdef CONFIG_SCST_PROC
373                 int len = strlen(target_name) +
374                         strlen(SCST_DEFAULT_ACG_NAME) + 1 + 1;
375
376                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
377                 if (tgt->default_group_name == NULL) {
378                         TRACE(TRACE_OUT_OF_MEM, "Allocation of default "
379                                 "group name failed (tgt %s)", target_name);
380                         rc = -ENOMEM;
381                         goto out_unlock_resume;
382                 }
383                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
384                         target_name);
385 #endif
386
387                 tgt->tgt_name = kmalloc(strlen(target_name) + 1, GFP_KERNEL);
388                 if (tgt->tgt_name == NULL) {
389                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name %s failed",
390                                 target_name);
391                         rc = -ENOMEM;
392 #ifdef CONFIG_SCST_PROC
393                         goto out_free_def_name;
394 #else
395                         goto out_unlock_resume;
396 #endif
397                 }
398                 strcpy(tgt->tgt_name, target_name);
399         } else {
400                 static int tgt_num; /* protected by scst_mutex */
401                 int len = strlen(vtt->name) +
402                         strlen(SCST_DEFAULT_TGT_NAME_SUFFIX) + 11 + 1;
403
404                 tgt->tgt_name = kmalloc(len, GFP_KERNEL);
405                 if (tgt->tgt_name == NULL) {
406                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name failed "
407                                 "(template name %s)", vtt->name);
408                         rc = -ENOMEM;
409 #ifdef CONFIG_SCST_PROC
410                         goto out_free_def_name;
411 #else
412                         goto out_unlock_resume;
413 #endif
414                 }
415                 sprintf(tgt->tgt_name, "%s%s%d", vtt->name,
416                         SCST_DEFAULT_TGT_NAME_SUFFIX, tgt_num++);
417         }
418
419         tgt->default_acg = scst_alloc_add_acg(NULL, tgt->tgt_name);
420         if (tgt->default_acg == NULL)
421                 goto out_free_tgt_name;
422
423         INIT_LIST_HEAD(&tgt->acg_list);
424
425 #ifdef CONFIG_SCST_PROC
426         rc = scst_build_proc_target_entries(tgt);
427         if (rc < 0)
428                 goto out_clear_acg;
429 #endif
430
431         rc = scst_create_tgt_sysfs(tgt);
432         if (rc < 0)
433 #ifdef CONFIG_SCST_PROC
434                 goto out_clean_proc;
435 #else
436                 goto out_clear_acg;
437 #endif
438
439         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
440
441         mutex_unlock(&scst_mutex);
442         scst_resume_activity();
443
444         PRINT_INFO("Target %s (%p) for template %s registered successfully",
445                 target_name, tgt, vtt->name);
446
447 out:
448         TRACE_EXIT();
449         return tgt;
450
451 #ifdef CONFIG_SCST_PROC
452 out_clean_proc:
453         scst_cleanup_proc_target_entries(tgt);
454 #endif
455
456 out_clear_acg:
457         scst_clear_acg(tgt->default_acg);
458
459 out_free_tgt_name:
460         kfree(tgt->tgt_name);
461
462 #ifdef CONFIG_SCST_PROC
463 out_free_def_name:
464         kfree(tgt->default_group_name);
465 #endif
466
467 out_unlock_resume:
468         mutex_unlock(&scst_mutex);
469
470 out_resume_free:
471         scst_resume_activity();
472
473 out_free_tgt_err:
474         scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
475         tgt = NULL;
476
477 out_err:
478         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
479                 target_name, vtt->name, rc);
480         goto out;
481 }
482 EXPORT_SYMBOL(scst_register);
483
484 static inline int test_sess_list(struct scst_tgt *tgt)
485 {
486         int res;
487         mutex_lock(&scst_mutex);
488         res = list_empty(&tgt->sess_list);
489         mutex_unlock(&scst_mutex);
490         return res;
491 }
492
493 void scst_unregister(struct scst_tgt *tgt)
494 {
495         struct scst_session *sess;
496         struct scst_tgt_template *vtt = tgt->tgtt;
497         struct scst_acg *acg, *acg_tmp;
498
499         TRACE_ENTRY();
500
501         scst_tgt_sysfs_prepare_put(tgt);
502
503         TRACE_DBG("%s", "Calling target driver's release()");
504         tgt->tgtt->release(tgt);
505         TRACE_DBG("%s", "Target driver's release() returned");
506
507         mutex_lock(&scst_mutex);
508 again:
509         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
510                 if (sess->shut_phase == SCST_SESS_SPH_READY) {
511                         /*
512                          * Sometimes it's hard for target driver to track all
513                          * its sessions (see scst_local, for example), so let's
514                          * help it.
515                          */
516                         mutex_unlock(&scst_mutex);
517                         scst_unregister_session(sess, 0, NULL);
518                         mutex_lock(&scst_mutex);
519                         goto again;
520                 }
521         }
522         mutex_unlock(&scst_mutex);
523
524         TRACE_DBG("%s", "Waiting for sessions shutdown");
525         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
526         TRACE_DBG("%s", "wait_event() returned");
527
528         scst_suspend_activity(false);
529         mutex_lock(&scst_mutex);
530
531         list_del(&tgt->tgt_list_entry);
532
533 #ifdef CONFIG_SCST_PROC
534         scst_cleanup_proc_target_entries(tgt);
535 #endif
536
537         mutex_unlock(&scst_mutex);
538         scst_resume_activity();
539
540         scst_clear_acg(tgt->default_acg);
541
542         list_for_each_entry_safe(acg, acg_tmp, &tgt->acg_list, acg_list_entry) {
543                 scst_acg_sysfs_put(acg);
544         }
545
546         del_timer_sync(&tgt->retry_timer);
547
548         scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
549
550         PRINT_INFO("Target %p for template %s unregistered successfully",
551                 tgt, vtt->name);
552
553         TRACE_EXIT();
554         return;
555 }
556 EXPORT_SYMBOL(scst_unregister);
557
558 static int scst_susp_wait(bool interruptible)
559 {
560         int res = 0;
561
562         TRACE_ENTRY();
563
564         if (interruptible) {
565                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
566                         (atomic_read(&scst_cmd_count) == 0),
567                         SCST_SUSPENDING_TIMEOUT);
568                 if (res <= 0) {
569                         __scst_resume_activity();
570                         if (res == 0)
571                                 res = -EBUSY;
572                 } else
573                         res = 0;
574         } else
575                 wait_event(scst_dev_cmd_waitQ,
576                            atomic_read(&scst_cmd_count) == 0);
577
578         TRACE_MGMT_DBG("wait_event() returned %d", res);
579
580         TRACE_EXIT_RES(res);
581         return res;
582 }
583
584 int scst_suspend_activity(bool interruptible)
585 {
586         int res = 0;
587         bool rep = false;
588
589         TRACE_ENTRY();
590
591         if (interruptible) {
592                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
593                         res = -EINTR;
594                         goto out;
595                 }
596         } else
597                 mutex_lock(&scst_suspend_mutex);
598
599         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
600         suspend_count++;
601         if (suspend_count > 1)
602                 goto out_up;
603
604         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
605         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
606         /*
607          * Assignment of SCST_FLAG_SUSPENDING and SCST_FLAG_SUSPENDED must be
608          * ordered with scst_cmd_count. Otherwise lockless logic in
609          * scst_translate_lun() and scst_mgmt_translate_lun() won't work.
610          */
611         smp_mb__after_set_bit();
612
613         /*
614          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
615          * information about scst_user behavior.
616          *
617          * ToDo: make the global suspending unneeded (switch to per-device
618          * reference counting? That would mean to switch off from lockless
619          * implementation of scst_translate_lun().. )
620          */
621
622         if (atomic_read(&scst_cmd_count) != 0) {
623                 PRINT_INFO("Waiting for %d active commands to complete... This "
624                         "might take few minutes for disks or few hours for "
625                         "tapes, if you use long executed commands, like "
626                         "REWIND or FORMAT. In case, if you have a hung user "
627                         "space device (i.e. made using scst_user module) not "
628                         "responding to any commands, if might take virtually "
629                         "forever until the corresponding user space "
630                         "program recovers and starts responding or gets "
631                         "killed.", atomic_read(&scst_cmd_count));
632                 rep = true;
633         }
634
635         res = scst_susp_wait(interruptible);
636         if (res != 0)
637                 goto out_clear;
638
639         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
640         /* See comment about smp_mb() above */
641         smp_mb__after_clear_bit();
642
643         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
644                 atomic_read(&scst_cmd_count));
645
646         res = scst_susp_wait(interruptible);
647         if (res != 0)
648                 goto out_clear;
649
650         if (rep)
651                 PRINT_INFO("%s", "All active commands completed");
652
653 out_up:
654         mutex_unlock(&scst_suspend_mutex);
655
656 out:
657         TRACE_EXIT_RES(res);
658         return res;
659
660 out_clear:
661         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
662         /* See comment about smp_mb() above */
663         smp_mb__after_clear_bit();
664         goto out_up;
665 }
666 EXPORT_SYMBOL(scst_suspend_activity);
667
668 static void __scst_resume_activity(void)
669 {
670         struct scst_cmd_lists *l;
671
672         TRACE_ENTRY();
673
674         suspend_count--;
675         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
676         if (suspend_count > 0)
677                 goto out;
678
679         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
680         /*
681          * The barrier is needed to make sure all woken up threads see the
682          * cleared flag. Not sure if it's really needed, but let's be safe.
683          */
684         smp_mb__after_clear_bit();
685
686         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
687                 wake_up_all(&l->cmd_list_waitQ);
688         }
689         wake_up_all(&scst_init_cmd_list_waitQ);
690
691         spin_lock_irq(&scst_mcmd_lock);
692         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
693                 struct scst_mgmt_cmd *m;
694                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
695                                 mgmt_cmd_list_entry);
696                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
697                         "mgmt cmd list", m);
698                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
699         }
700         spin_unlock_irq(&scst_mcmd_lock);
701         wake_up_all(&scst_mgmt_cmd_list_waitQ);
702
703 out:
704         TRACE_EXIT();
705         return;
706 }
707
708 void scst_resume_activity(void)
709 {
710         TRACE_ENTRY();
711
712         mutex_lock(&scst_suspend_mutex);
713         __scst_resume_activity();
714         mutex_unlock(&scst_suspend_mutex);
715
716         TRACE_EXIT();
717         return;
718 }
719 EXPORT_SYMBOL(scst_resume_activity);
720
721 static int scst_register_device(struct scsi_device *scsidp)
722 {
723         int res = 0;
724         struct scst_device *dev;
725         struct scst_dev_type *dt;
726
727         TRACE_ENTRY();
728
729         res = scst_suspend_activity(true);
730         if (res != 0)
731                 goto out_err;
732
733         if (mutex_lock_interruptible(&scst_mutex) != 0) {
734                 res = -EINTR;
735                 goto out_resume;
736         }
737
738         res = scst_alloc_device(GFP_KERNEL, &dev);
739         if (res != 0)
740                 goto out_up;
741
742         dev->type = scsidp->type;
743
744         dev->virt_name = kmalloc(50, GFP_KERNEL);
745         if (dev->virt_name == NULL) {
746                 PRINT_ERROR("%s", "Unable to alloc device name");
747                 res = -ENOMEM;
748                 goto out_free_dev;
749         }
750         snprintf(dev->virt_name, 50, "%d:%d:%d:%d", scsidp->host->host_no,
751                 scsidp->channel, scsidp->id, scsidp->lun);
752
753 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
754         dev->rq_disk = alloc_disk(1);
755         if (dev->rq_disk == NULL) {
756                 PRINT_ERROR("Unable to alloc disk object for device %s",
757                         dev->virt_name);
758                 res = -ENOMEM;
759                 /* virt_name will be freed in scst_free_dev() */
760                 goto out_free_dev;
761         }
762         dev->rq_disk->major = SCST_MAJOR;
763 #endif
764
765         dev->scsi_dev = scsidp;
766
767         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
768
769         res = scst_create_device_sysfs(dev);
770         if (res != 0)
771                 goto out_free;
772
773         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
774                 if (dt->type == scsidp->type) {
775                         res = scst_assign_dev_handler(dev, dt);
776                         if (res != 0)
777                                 goto out_free;
778                         break;
779                 }
780         }
781
782 out_up:
783         mutex_unlock(&scst_mutex);
784
785 out_resume:
786         scst_resume_activity();
787
788 out_err:
789         if (res == 0) {
790                 PRINT_INFO("Attached to scsi%d, channel %d, id %d, lun %d, "
791                         "type %d", scsidp->host->host_no, scsidp->channel,
792                         scsidp->id, scsidp->lun, scsidp->type);
793         } else {
794                 PRINT_ERROR("Failed to attach to scsi%d, channel %d, id %d, "
795                         "lun %d, type %d", scsidp->host->host_no,
796                         scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
797         }
798
799         TRACE_EXIT_RES(res);
800         return res;
801
802 out_free:
803         list_del(&dev->dev_list_entry);
804 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
805         put_disk(dev->rq_disk);
806 #endif
807
808 out_free_dev:
809         mutex_unlock(&scst_mutex);
810         scst_resume_activity();
811         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
812         goto out_err;
813 }
814
815 static void scst_unregister_device(struct scsi_device *scsidp)
816 {
817         struct scst_device *d, *dev = NULL;
818         struct scst_acg_dev *acg_dev, *aa;
819
820         TRACE_ENTRY();
821
822         scst_suspend_activity(false);
823         mutex_lock(&scst_mutex);
824
825         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
826                 if (d->scsi_dev == scsidp) {
827                         dev = d;
828                         TRACE_DBG("Target device %p found", dev);
829                         break;
830                 }
831         }
832         if (dev == NULL) {
833                 PRINT_ERROR("%s", "Target device not found");
834                 goto out_resume;
835         }
836
837         list_del(&dev->dev_list_entry);
838
839         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
840                                  dev_acg_dev_list_entry) {
841                 scst_acg_remove_dev(acg_dev->acg, dev, true);
842         }
843
844         scst_assign_dev_handler(dev, &scst_null_devtype);
845
846 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
847         put_disk(dev->rq_disk);
848 #endif
849
850         mutex_unlock(&scst_mutex);
851         scst_resume_activity();
852
853         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
854
855         PRINT_INFO("Detached from scsi%d, channel %d, id %d, lun %d, type %d",
856                 scsidp->host->host_no, scsidp->channel, scsidp->id,
857                 scsidp->lun, scsidp->type);
858
859 out:
860         TRACE_EXIT();
861         return;
862
863 out_resume:
864         mutex_unlock(&scst_mutex);
865         scst_resume_activity();
866         goto out;
867 }
868
869 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
870 {
871         int res = 0;
872
873         if (dev_handler->parse == NULL) {
874                 PRINT_ERROR("scst dev handler %s must have "
875                         "parse() method.", dev_handler->name);
876                 res = -EINVAL;
877                 goto out;
878         }
879
880         if (dev_handler->exec == NULL) {
881 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
882                 dev_handler->exec_atomic = 1;
883 #else
884                 dev_handler->exec_atomic = 0;
885 #endif
886         }
887
888         if (dev_handler->dev_done == NULL)
889                 dev_handler->dev_done_atomic = 1;
890
891 out:
892         TRACE_EXIT_RES(res);
893         return res;
894 }
895
896 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
897         const char *dev_name)
898 {
899         int res, rc;
900         struct scst_device *dev = NULL;
901
902         TRACE_ENTRY();
903
904         if (dev_handler == NULL) {
905                 PRINT_ERROR("%s: valid device handler must be supplied",
906                             __func__);
907                 res = -EINVAL;
908                 goto out;
909         }
910
911         if (dev_name == NULL) {
912                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
913                 res = -EINVAL;
914                 goto out;
915         }
916
917         res = scst_dev_handler_check(dev_handler);
918         if (res != 0)
919                 goto out;
920
921         res = scst_suspend_activity(true);
922         if (res != 0)
923                 goto out;
924
925         if (mutex_lock_interruptible(&scst_mutex) != 0) {
926                 res = -EINTR;
927                 goto out_resume;
928         }
929
930         res = scst_alloc_device(GFP_KERNEL, &dev);
931         if (res != 0)
932                 goto out_up;
933
934         dev->type = dev_handler->type;
935         dev->scsi_dev = NULL;
936         dev->virt_name = kstrdup(dev_name, GFP_KERNEL);
937         if (dev->virt_name == NULL) {
938                 PRINT_ERROR("Unable to allocate virt_name for dev %s",
939                         dev_name);
940                 res = -ENOMEM;
941                 goto out_release;
942         }
943         dev->virt_id = scst_virt_dev_last_id++;
944
945         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
946
947         res = dev->virt_id;
948
949         rc = scst_create_device_sysfs(dev);
950         if (rc != 0) {
951                 res = rc;
952                 goto out_free_del;
953         }
954
955         rc = scst_assign_dev_handler(dev, dev_handler);
956         if (rc != 0) {
957                 res = rc;
958                 goto out_free_del;
959         }
960
961 out_up:
962         mutex_unlock(&scst_mutex);
963
964 out_resume:
965         scst_resume_activity();
966
967 out:
968         if (res > 0)
969                 PRINT_INFO("Attached to virtual device %s (id %d)",
970                         dev_name, dev->virt_id);
971         else
972                 PRINT_INFO("Failed to attach to virtual device %s", dev_name);
973
974         TRACE_EXIT_RES(res);
975         return res;
976
977 out_free_del:
978         list_del(&dev->dev_list_entry);
979
980 out_release:
981         mutex_unlock(&scst_mutex);
982         scst_resume_activity();
983         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
984         goto out;
985 }
986 EXPORT_SYMBOL(scst_register_virtual_device);
987
988 void scst_unregister_virtual_device(int id)
989 {
990         struct scst_device *d, *dev = NULL;
991         struct scst_acg_dev *acg_dev, *aa;
992
993         TRACE_ENTRY();
994
995         scst_suspend_activity(false);
996         mutex_lock(&scst_mutex);
997
998         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
999                 if (d->virt_id == id) {
1000                         dev = d;
1001                         TRACE_DBG("Target device %p (id %d) found", dev, id);
1002                         break;
1003                 }
1004         }
1005         if (dev == NULL) {
1006                 PRINT_ERROR("Target virtual device (id %d) not found", id);
1007                 goto out_unblock;
1008         }
1009
1010         list_del(&dev->dev_list_entry);
1011
1012         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
1013                                  dev_acg_dev_list_entry) {
1014                 scst_acg_remove_dev(acg_dev->acg, dev, true);
1015         }
1016
1017         scst_assign_dev_handler(dev, &scst_null_devtype);
1018
1019         PRINT_INFO("Detached from virtual device %s (id %d)",
1020                 dev->virt_name, dev->virt_id);
1021
1022 out_unblock:
1023         mutex_unlock(&scst_mutex);
1024         scst_resume_activity();
1025
1026         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
1027
1028         TRACE_EXIT();
1029         return;
1030 }
1031 EXPORT_SYMBOL(scst_unregister_virtual_device);
1032
1033 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
1034         const char *version)
1035 {
1036         struct scst_dev_type *dt;
1037         struct scst_device *dev;
1038         int res;
1039         int exist;
1040
1041         TRACE_ENTRY();
1042
1043         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1044                 PRINT_ERROR("Incorrect version of dev handler %s",
1045                         dev_type->name);
1046                 res = -EINVAL;
1047                 goto out_error;
1048         }
1049
1050         res = scst_dev_handler_check(dev_type);
1051         if (res != 0)
1052                 goto out_error;
1053
1054 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
1055         if (dev_type->exec == NULL) {
1056 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
1057 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
1058                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1059                         "supported. Consider applying on your kernel patch "
1060                         "scst_exec_req_fifo-<kernel-version> or define "
1061                         "CONFIG_SCST_STRICT_SERIALIZING", dev_type->name);
1062                 res = -EINVAL;
1063                 goto out_error;
1064 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
1065 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1066                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1067                         "supported. Consider applying on your kernel patch "
1068                         "scst_exec_req_fifo-<kernel-version>", dev_type->name);
1069                 res = -EINVAL;
1070                 goto out_error;
1071 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1072         }
1073 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
1074
1075         res = scst_suspend_activity(true);
1076         if (res != 0)
1077                 goto out_error;
1078
1079         if (mutex_lock_interruptible(&scst_mutex) != 0) {
1080                 res = -EINTR;
1081                 goto out_err_res;
1082         }
1083
1084         exist = 0;
1085         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1086                 if (strcmp(dt->name, dev_type->name) == 0) {
1087                         PRINT_ERROR("Device type handler \"%s\" already "
1088                                 "exist", dt->name);
1089                         exist = 1;
1090                         break;
1091                 }
1092         }
1093         if (exist)
1094                 goto out_up;
1095
1096 #ifdef CONFIG_SCST_PROC
1097         if (!dev_type->no_proc) {
1098                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1099                 if (res < 0)
1100                         goto out_up;
1101         }
1102 #endif
1103
1104         res = scst_create_devt_sysfs(dev_type);
1105         if (res < 0)
1106                 goto out_free;
1107
1108         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
1109
1110         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1111                 if (dev->scsi_dev == NULL || dev->handler != &scst_null_devtype)
1112                         continue;
1113                 if (dev->scsi_dev->type == dev_type->type)
1114                         scst_assign_dev_handler(dev, dev_type);
1115         }
1116
1117         mutex_unlock(&scst_mutex);
1118         scst_resume_activity();
1119
1120         if (res == 0) {
1121                 PRINT_INFO("Device handler \"%s\" for type %d registered "
1122                         "successfully", dev_type->name, dev_type->type);
1123         }
1124
1125 out:
1126         TRACE_EXIT_RES(res);
1127         return res;
1128
1129 out_free:
1130 #ifdef CONFIG_SCST_PROC
1131         if (!dev_type->no_proc)
1132                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1133 #endif
1134         scst_devt_sysfs_put(dev_type);
1135
1136 out_up:
1137         mutex_unlock(&scst_mutex);
1138
1139 out_err_res:
1140         scst_resume_activity();
1141
1142 out_error:
1143         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
1144                 dev_type->name, dev_type->type);
1145         goto out;
1146 }
1147 EXPORT_SYMBOL(__scst_register_dev_driver);
1148
1149 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
1150 {
1151         struct scst_device *dev;
1152         struct scst_dev_type *dt;
1153         int found = 0;
1154
1155         TRACE_ENTRY();
1156
1157         scst_suspend_activity(false);
1158         mutex_lock(&scst_mutex);
1159
1160         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1161                 if (strcmp(dt->name, dev_type->name) == 0) {
1162                         found = 1;
1163                         break;
1164                 }
1165         }
1166         if (!found) {
1167                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
1168                         dev_type->name);
1169                 goto out_up;
1170         }
1171
1172         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1173                 if (dev->handler == dev_type) {
1174                         scst_assign_dev_handler(dev, &scst_null_devtype);
1175                         TRACE_DBG("Dev handler removed from device %p", dev);
1176                 }
1177         }
1178
1179         list_del(&dev_type->dev_type_list_entry);
1180
1181         mutex_unlock(&scst_mutex);
1182         scst_resume_activity();
1183
1184 #ifdef CONFIG_SCST_PROC
1185         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1186 #endif
1187
1188         scst_devt_sysfs_put(dev_type);
1189
1190         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
1191                    dev_type->name, dev_type->type);
1192
1193 out:
1194         TRACE_EXIT();
1195         return;
1196
1197 out_up:
1198         mutex_unlock(&scst_mutex);
1199         scst_resume_activity();
1200         goto out;
1201 }
1202 EXPORT_SYMBOL(scst_unregister_dev_driver);
1203
1204 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1205         const char *version)
1206 {
1207         int res;
1208
1209         TRACE_ENTRY();
1210
1211         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1212                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1213                         dev_type->name);
1214                 res = -EINVAL;
1215                 goto out_err;
1216         }
1217
1218         res = scst_dev_handler_check(dev_type);
1219         if (res != 0)
1220                 goto out_err;
1221
1222 #ifdef CONFIG_SCST_PROC
1223         if (!dev_type->no_proc) {
1224                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1225                 if (res < 0)
1226                         goto out_err;
1227         }
1228 #endif
1229
1230         res = scst_create_devt_sysfs(dev_type);
1231         if (res < 0)
1232                 goto out_free;
1233
1234         if (dev_type->type != -1) {
1235                 PRINT_INFO("Virtual device handler %s for type %d "
1236                         "registered successfully", dev_type->name,
1237                         dev_type->type);
1238         } else {
1239                 PRINT_INFO("Virtual device handler \"%s\" registered "
1240                         "successfully", dev_type->name);
1241         }
1242
1243 out:
1244         TRACE_EXIT_RES(res);
1245         return res;
1246
1247 out_free:
1248 #ifdef CONFIG_SCST_PROC
1249         if (!dev_type->no_proc)
1250                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1251 #endif
1252
1253         scst_devt_sysfs_put(dev_type);
1254
1255 out_err:
1256         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1257                 dev_type->name);
1258         goto out;
1259 }
1260 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1261
1262 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1263 {
1264         TRACE_ENTRY();
1265
1266 #ifdef CONFIG_SCST_PROC
1267         if (!dev_type->no_proc)
1268                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1269 #endif
1270
1271         scst_devt_sysfs_put(dev_type);
1272
1273         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1274
1275         TRACE_EXIT();
1276         return;
1277 }
1278 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1279
1280 /* Called under scst_mutex */
1281 int scst_add_dev_threads(struct scst_device *dev, int num)
1282 {
1283         int i, res = 0;
1284         int n = 0;
1285         struct scst_cmd_thread_t *thr;
1286         char nm[12];
1287
1288         TRACE_ENTRY();
1289
1290         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1291                 n++;
1292         }
1293
1294         for (i = 0; i < num; i++) {
1295                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1296                 if (!thr) {
1297                         res = -ENOMEM;
1298                         PRINT_ERROR("Failed to allocate thr %d", res);
1299                         goto out_del;
1300                 }
1301                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1302                 nm[ARRAY_SIZE(nm)-1] = '\0';
1303                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1304                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1305                 if (IS_ERR(thr->cmd_thread)) {
1306                         res = PTR_ERR(thr->cmd_thread);
1307                         PRINT_ERROR("kthread_create() failed: %d", res);
1308                         kfree(thr);
1309                         goto out_del;
1310                 }
1311
1312                 list_add(&thr->thread_list_entry, &dev->threads_list);
1313
1314                 /*
1315                  * ToDo: better to use tgt_dev_io_context instead, but we
1316                  * are not ready for that yet.
1317                  */
1318                 __exit_io_context(thr->cmd_thread->io_context);
1319                 thr->cmd_thread->io_context = ioc_task_link(dev->dev_io_ctx);
1320                 TRACE_DBG("Setting dev io ctx %p on thr %d", dev->dev_io_ctx,
1321                         thr->cmd_thread->pid);
1322
1323                 wake_up_process(thr->cmd_thread);
1324         }
1325
1326 out:
1327         TRACE_EXIT_RES(res);
1328         return res;
1329
1330 out_del:
1331         scst_del_dev_threads(dev, i);
1332         goto out;
1333 }
1334
1335 /* Called under scst_mutex and suspended activity */
1336 static int scst_create_dev_threads(struct scst_device *dev)
1337 {
1338         int res = 0;
1339         int threads_num;
1340
1341         TRACE_ENTRY();
1342
1343         if (dev->handler->threads_num <= 0)
1344                 goto out;
1345
1346         threads_num = dev->handler->threads_num;
1347
1348         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1349         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1350         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1351
1352         res = scst_add_dev_threads(dev, threads_num);
1353         if (res != 0)
1354                 goto out;
1355
1356         mutex_lock(&scst_suspend_mutex);
1357         list_add_tail(&dev->cmd_lists.lists_list_entry,
1358                 &scst_cmd_lists_list);
1359         mutex_unlock(&scst_suspend_mutex);
1360
1361         dev->p_cmd_lists = &dev->cmd_lists;
1362
1363 out:
1364         TRACE_EXIT_RES(res);
1365         return res;
1366 }
1367
1368 /* Called under scst_mutex */
1369 void scst_del_dev_threads(struct scst_device *dev, int num)
1370 {
1371         struct scst_cmd_thread_t *ct, *tmp;
1372         int i = 0;
1373
1374         TRACE_ENTRY();
1375
1376         if (num == 0)
1377                 goto out;
1378
1379         list_for_each_entry_safe_reverse(ct, tmp, &dev->threads_list,
1380                                 thread_list_entry) {
1381                 int rc;
1382                 struct scst_tgt_dev *tgt_dev;
1383
1384                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1385                                 dev_tgt_dev_list_entry) {
1386                         struct scst_thr_data_hdr *td;
1387                         td = __scst_find_thr_data(tgt_dev, ct->cmd_thread);
1388                         if (td != NULL) {
1389                                 scst_thr_data_put(td);
1390                                 break;
1391                         }
1392                 }
1393
1394                 rc = kthread_stop(ct->cmd_thread);
1395                 if (rc < 0)
1396                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1397
1398                 list_del(&ct->thread_list_entry);
1399                 kfree(ct);
1400
1401                 if ((num > 0) && (++i >= num))
1402                         break;
1403         }
1404
1405 out:
1406         TRACE_EXIT();
1407         return;
1408 }
1409
1410 /* Called under scst_mutex and suspended activity */
1411 static void scst_stop_dev_threads(struct scst_device *dev)
1412 {
1413         TRACE_ENTRY();
1414
1415         if (list_empty(&dev->threads_list))
1416                 goto out;
1417
1418         scst_del_dev_threads(dev, -1);
1419
1420         if (dev->p_cmd_lists == &dev->cmd_lists) {
1421                 mutex_lock(&scst_suspend_mutex);
1422                 list_del(&dev->cmd_lists.lists_list_entry);
1423                 mutex_unlock(&scst_suspend_mutex);
1424         }
1425
1426 out:
1427         TRACE_EXIT();
1428         return;
1429 }
1430
1431 /* The activity supposed to be suspended and scst_mutex held */
1432 int scst_assign_dev_handler(struct scst_device *dev,
1433         struct scst_dev_type *handler)
1434 {
1435         int res = 0;
1436         struct scst_tgt_dev *tgt_dev;
1437         LIST_HEAD(attached_tgt_devs);
1438
1439         TRACE_ENTRY();
1440
1441         sBUG_ON(handler == NULL);
1442
1443         if (dev->handler == handler)
1444                 goto out;
1445
1446         if (dev->handler == NULL)
1447                 goto assign;
1448
1449         if (dev->handler->detach_tgt) {
1450                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1451                                 dev_tgt_dev_list_entry) {
1452                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1453                                 tgt_dev);
1454                         dev->handler->detach_tgt(tgt_dev);
1455                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1456                 }
1457         }
1458
1459         if (dev->handler->detach) {
1460                 TRACE_DBG("%s", "Calling dev handler's detach()");
1461                 dev->handler->detach(dev);
1462                 TRACE_DBG("%s", "Old handler's detach() returned");
1463         }
1464
1465         scst_stop_dev_threads(dev);
1466
1467         scst_devt_dev_sysfs_put(dev);
1468
1469 assign:
1470         dev->handler = handler;
1471
1472         if (handler == NULL)
1473                 goto out;
1474
1475         res = scst_create_devt_dev_sysfs(dev);
1476         if (res != 0)
1477                 goto out_null;
1478
1479         res = scst_create_dev_threads(dev);
1480         if (res != 0)
1481                 goto out_remove_sysfs;
1482
1483         if (handler->attach) {
1484                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1485                 res = handler->attach(dev);
1486                 TRACE_DBG("New dev handler's attach() returned %d", res);
1487                 if (res != 0) {
1488                         PRINT_ERROR("New device handler's %s attach() "
1489                                 "failed: %d", handler->name, res);
1490                         goto out_thr_null;
1491                 }
1492         }
1493
1494         if (handler->attach_tgt) {
1495                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1496                                 dev_tgt_dev_list_entry) {
1497                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1498                                 tgt_dev);
1499                         res = handler->attach_tgt(tgt_dev);
1500                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1501                         if (res != 0) {
1502                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1503                                     "failed: %d", handler->name, res);
1504                                 goto out_err_detach_tgt;
1505                         }
1506                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1507                                 &attached_tgt_devs);
1508                 }
1509         }
1510
1511 out:
1512         TRACE_EXIT_RES(res);
1513         return res;
1514
1515 out_err_detach_tgt:
1516         if (handler && handler->detach_tgt) {
1517                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1518                                  extra_tgt_dev_list_entry) {
1519                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1520                                 tgt_dev);
1521                         handler->detach_tgt(tgt_dev);
1522                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1523                 }
1524         }
1525         if (handler && handler->detach) {
1526                 TRACE_DBG("%s", "Calling handler's detach()");
1527                 handler->detach(dev);
1528                 TRACE_DBG("%s", "Handler's detach() returned");
1529         }
1530
1531 out_thr_null:
1532         scst_stop_dev_threads(dev);
1533
1534 out_remove_sysfs:
1535         scst_devt_dev_sysfs_put(dev);
1536
1537 out_null:
1538         dev->handler = &scst_null_devtype;
1539         goto out;
1540 }
1541
1542 int scst_global_threads_count(void)
1543 {
1544         int i;
1545
1546         /*
1547          * Just to lower the race window, when user can get just changed value
1548          */
1549         mutex_lock(&scst_global_threads_mutex);
1550         i = scst_nr_global_threads;
1551         mutex_unlock(&scst_global_threads_mutex);
1552         return i;
1553 }
1554
1555 static void scst_threads_info_init(void)
1556 {
1557         mutex_init(&scst_global_threads_mutex);
1558         INIT_LIST_HEAD(&scst_global_threads_list);
1559 }
1560
1561 /* scst_global_threads_mutex supposed to be held */
1562 void __scst_del_global_threads(int num)
1563 {
1564         struct scst_cmd_thread_t *ct, *tmp;
1565
1566         TRACE_ENTRY();
1567
1568         if (num == 0)
1569                 goto out;
1570
1571         list_for_each_entry_safe(ct, tmp, &scst_global_threads_list,
1572                                 thread_list_entry) {
1573                 int res;
1574
1575                 res = kthread_stop(ct->cmd_thread);
1576                 if (res < 0)
1577                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1578                 list_del(&ct->thread_list_entry);
1579                 kfree(ct);
1580                 scst_nr_global_threads--;
1581                 --num;
1582                 if (num == 0)
1583                         break;
1584         }
1585
1586 out:
1587         TRACE_EXIT();
1588         return;
1589 }
1590
1591 /* scst_global_threads_mutex supposed to be held */
1592 int __scst_add_global_threads(int num)
1593 {
1594         int res = 0, i;
1595         static int scst_thread_num;
1596
1597         TRACE_ENTRY();
1598
1599         for (i = 0; i < num; i++) {
1600                 struct scst_cmd_thread_t *thr;
1601
1602                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1603                 if (!thr) {
1604                         res = -ENOMEM;
1605                         PRINT_ERROR("fail to allocate thr %d", res);
1606                         goto out_error;
1607                 }
1608                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1609                         &scst_main_cmd_lists, "scsi_tgt%d",
1610                         scst_thread_num++);
1611                 if (IS_ERR(thr->cmd_thread)) {
1612                         res = PTR_ERR(thr->cmd_thread);
1613                         PRINT_ERROR("kthread_create() failed: %d", res);
1614                         kfree(thr);
1615                         goto out_error;
1616                 }
1617
1618                 list_add(&thr->thread_list_entry, &scst_global_threads_list);
1619                 scst_nr_global_threads++;
1620
1621                 wake_up_process(thr->cmd_thread);
1622         }
1623         res = 0;
1624
1625 out:
1626         TRACE_EXIT_RES(res);
1627         return res;
1628
1629 out_error:
1630         __scst_del_global_threads(i);
1631         goto out;
1632 }
1633
1634 int scst_add_global_threads(int num)
1635 {
1636         int res;
1637
1638         TRACE_ENTRY();
1639
1640         mutex_lock(&scst_global_threads_mutex);
1641         res = __scst_add_global_threads(num);
1642         mutex_unlock(&scst_global_threads_mutex);
1643
1644         TRACE_EXIT_RES(res);
1645         return res;
1646 }
1647 EXPORT_SYMBOL(scst_add_global_threads);
1648
1649 void scst_del_global_threads(int num)
1650 {
1651         TRACE_ENTRY();
1652
1653         mutex_lock(&scst_global_threads_mutex);
1654         __scst_del_global_threads(num);
1655         mutex_unlock(&scst_global_threads_mutex);
1656
1657         TRACE_EXIT();
1658         return;
1659 }
1660 EXPORT_SYMBOL(scst_del_global_threads);
1661
1662 static void scst_stop_all_threads(void)
1663 {
1664         TRACE_ENTRY();
1665
1666         mutex_lock(&scst_global_threads_mutex);
1667         __scst_del_global_threads(-1);
1668         if (scst_mgmt_cmd_thread)
1669                 kthread_stop(scst_mgmt_cmd_thread);
1670         if (scst_mgmt_thread)
1671                 kthread_stop(scst_mgmt_thread);
1672         if (scst_init_cmd_thread)
1673                 kthread_stop(scst_init_cmd_thread);
1674         mutex_unlock(&scst_global_threads_mutex);
1675
1676         TRACE_EXIT();
1677         return;
1678 }
1679
1680 static int scst_start_all_threads(int num)
1681 {
1682         int res;
1683
1684         TRACE_ENTRY();
1685
1686         mutex_lock(&scst_global_threads_mutex);
1687         res = __scst_add_global_threads(num);
1688         if (res < 0)
1689                 goto out;
1690
1691         scst_init_cmd_thread = kthread_run(scst_init_thread,
1692                 NULL, "scsi_tgt_init");
1693         if (IS_ERR(scst_init_cmd_thread)) {
1694                 res = PTR_ERR(scst_init_cmd_thread);
1695                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1696                 scst_init_cmd_thread = NULL;
1697                 goto out;
1698         }
1699
1700         scst_mgmt_cmd_thread = kthread_run(scst_tm_thread,
1701                 NULL, "scsi_tm");
1702         if (IS_ERR(scst_mgmt_cmd_thread)) {
1703                 res = PTR_ERR(scst_mgmt_cmd_thread);
1704                 PRINT_ERROR("kthread_create() for TM failed: %d", res);
1705                 scst_mgmt_cmd_thread = NULL;
1706                 goto out;
1707         }
1708
1709         scst_mgmt_thread = kthread_run(scst_global_mgmt_thread,
1710                 NULL, "scsi_tgt_mgmt");
1711         if (IS_ERR(scst_mgmt_thread)) {
1712                 res = PTR_ERR(scst_mgmt_thread);
1713                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1714                 scst_mgmt_thread = NULL;
1715                 goto out;
1716         }
1717
1718 out:
1719         mutex_unlock(&scst_global_threads_mutex);
1720         TRACE_EXIT_RES(res);
1721         return res;
1722 }
1723
1724 void scst_get(void)
1725 {
1726         __scst_get(0);
1727 }
1728 EXPORT_SYMBOL(scst_get);
1729
1730 void scst_put(void)
1731 {
1732         __scst_put();
1733 }
1734 EXPORT_SYMBOL(scst_put);
1735
1736 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1737 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1738 #else
1739 static int scst_add(struct device *cdev, struct class_interface *intf)
1740 #endif
1741 {
1742         struct scsi_device *scsidp;
1743         int res = 0;
1744
1745         TRACE_ENTRY();
1746
1747 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1748         scsidp = to_scsi_device(cdev->dev);
1749 #else
1750         scsidp = to_scsi_device(cdev->parent);
1751 #endif
1752
1753         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1754                 res = scst_register_device(scsidp);
1755
1756         TRACE_EXIT();
1757         return res;
1758 }
1759
1760 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1761 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1762 #else
1763 static void scst_remove(struct device *cdev, struct class_interface *intf)
1764 #endif
1765 {
1766         struct scsi_device *scsidp;
1767
1768         TRACE_ENTRY();
1769
1770 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1771         scsidp = to_scsi_device(cdev->dev);
1772 #else
1773         scsidp = to_scsi_device(cdev->parent);
1774 #endif
1775
1776         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1777                 scst_unregister_device(scsidp);
1778
1779         TRACE_EXIT();
1780         return;
1781 }
1782
1783 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1784 static struct class_interface scst_interface = {
1785         .add = scst_add,
1786         .remove = scst_remove,
1787 };
1788 #else
1789 static struct class_interface scst_interface = {
1790         .add_dev = scst_add,
1791         .remove_dev = scst_remove,
1792 };
1793 #endif
1794
1795 static void __init scst_print_config(void)
1796 {
1797         char buf[128];
1798         int i, j;
1799
1800         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1801         j = i;
1802
1803 #ifdef CONFIG_SCST_STRICT_SERIALIZING
1804         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1805 #endif
1806
1807 #ifdef CONFIG_SCST_EXTRACHECKS
1808         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1809                 (j == i) ? "" : ", ");
1810 #endif
1811
1812 #ifdef CONFIG_SCST_TRACING
1813         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1814                 (j == i) ? "" : ", ");
1815 #endif
1816
1817 #ifdef CONFIG_SCST_DEBUG
1818         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1819                 (j == i) ? "" : ", ");
1820 #endif
1821
1822 #ifdef CONFIG_SCST_DEBUG_TM
1823         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1824                 (j == i) ? "" : ", ");
1825 #endif
1826
1827 #ifdef CONFIG_SCST_DEBUG_RETRY
1828         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1829                 (j == i) ? "" : ", ");
1830 #endif
1831
1832 #ifdef CONFIG_SCST_DEBUG_OOM
1833         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1834                 (j == i) ? "" : ", ");
1835 #endif
1836
1837 #ifdef CONFIG_SCST_DEBUG_SN
1838         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1839                 (j == i) ? "" : ", ");
1840 #endif
1841
1842 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
1843         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1844                 (j == i) ? "" : ", ");
1845 #endif
1846
1847 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1848         i += snprintf(&buf[i], sizeof(buf) - i,
1849                 "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1850                 (j == i) ? "" : ", ");
1851 #endif
1852
1853 #ifdef CONFIG_SCST_STRICT_SECURITY
1854         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1855                 (j == i) ? "" : ", ");
1856 #endif
1857
1858         if (j != i)
1859                 PRINT_INFO("%s", buf);
1860 }
1861
1862 static int __init init_scst(void)
1863 {
1864         int res, i;
1865         int scst_num_cpus;
1866
1867         TRACE_ENTRY();
1868
1869 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1870         {
1871                 struct scsi_request *req;
1872                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1873                         sizeof(req->sr_sense_buffer));
1874         }
1875 #else
1876         {
1877                 struct scsi_sense_hdr *shdr;
1878                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1879         }
1880 #endif
1881         {
1882                 struct scst_tgt_dev *t;
1883                 struct scst_cmd *c;
1884                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1885                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1886         }
1887
1888 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
1889 #if !defined(SCST_IO_CONTEXT)
1890         PRINT_WARNING("%s", "Patch io_context was not applied on "
1891                 "your kernel. SCST will be working with not the best "
1892                 "performance.");
1893 #endif
1894 #else
1895         PRINT_WARNING("%s", "There is no patch io_context for your kernel "
1896                 "version. For performance reasons it is strongly recommended "
1897                 "to upgrade your kernel to version >= 2.6.27.x.");
1898 #endif
1899
1900         mutex_init(&scst_mutex);
1901         INIT_LIST_HEAD(&scst_template_list);
1902         INIT_LIST_HEAD(&scst_dev_list);
1903         INIT_LIST_HEAD(&scst_dev_type_list);
1904         spin_lock_init(&scst_main_lock);
1905 #ifdef CONFIG_SCST_PROC
1906         INIT_LIST_HEAD(&scst_acg_list);
1907 #endif
1908         spin_lock_init(&scst_init_lock);
1909         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1910         INIT_LIST_HEAD(&scst_init_cmd_list);
1911 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1912         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1913 #endif
1914         atomic_set(&scst_cmd_count, 0);
1915         spin_lock_init(&scst_mcmd_lock);
1916         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1917         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1918         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1919         init_waitqueue_head(&scst_mgmt_waitQ);
1920         spin_lock_init(&scst_mgmt_lock);
1921         INIT_LIST_HEAD(&scst_sess_init_list);
1922         INIT_LIST_HEAD(&scst_sess_shut_list);
1923         init_waitqueue_head(&scst_dev_cmd_waitQ);
1924         mutex_init(&scst_suspend_mutex);
1925         INIT_LIST_HEAD(&scst_cmd_lists_list);
1926         scst_virt_dev_last_id = 1;
1927         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1928         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1929         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1930         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1931                 &scst_cmd_lists_list);
1932
1933         res = scst_lib_init();
1934         if (res != 0)
1935                 goto out;
1936
1937         scst_num_cpus = num_online_cpus();
1938
1939         /* ToDo: register_cpu_notifier() */
1940
1941         if (scst_threads == 0)
1942                 scst_threads = scst_num_cpus;
1943
1944         if (scst_threads < 1) {
1945                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1946                 scst_threads = scst_num_cpus;
1947         }
1948
1949         scst_threads_info_init();
1950
1951 #define INIT_CACHEP(p, s, o) do {                                       \
1952                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1953                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1954                           sizeof(struct s));                            \
1955                 if (p == NULL) {                                        \
1956                         res = -ENOMEM;                                  \
1957                         goto o;                                         \
1958                 }                                                       \
1959         } while (0)
1960
1961         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit);
1962         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1963                         out_destroy_mgmt_cache);
1964         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1965                         out_destroy_mgmt_stub_cache);
1966         {
1967                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1968                 INIT_CACHEP(scst_sense_cachep, scst_sense,
1969                             out_destroy_ua_cache);
1970         }
1971         INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
1972         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
1973         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1974         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1975         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1976
1977         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1978                 mempool_free_slab, scst_mgmt_cachep);
1979         if (scst_mgmt_mempool == NULL) {
1980                 res = -ENOMEM;
1981                 goto out_destroy_acg_cache;
1982         }
1983
1984         /*
1985          * All mgmt stubs, UAs and sense buffers are bursty and loosing them
1986          * may have fatal consequences, so let's have big pools for them.
1987          */
1988
1989         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1990                 mempool_free_slab, scst_mgmt_stub_cachep);
1991         if (scst_mgmt_stub_mempool == NULL) {
1992                 res = -ENOMEM;
1993                 goto out_destroy_mgmt_mempool;
1994         }
1995
1996         scst_ua_mempool = mempool_create(512, mempool_alloc_slab,
1997                 mempool_free_slab, scst_ua_cachep);
1998         if (scst_ua_mempool == NULL) {
1999                 res = -ENOMEM;
2000                 goto out_destroy_mgmt_stub_mempool;
2001         }
2002
2003         scst_sense_mempool = mempool_create(1024, mempool_alloc_slab,
2004                 mempool_free_slab, scst_sense_cachep);
2005         if (scst_sense_mempool == NULL) {
2006                 res = -ENOMEM;
2007                 goto out_destroy_ua_mempool;
2008         }
2009
2010         scst_aen_mempool = mempool_create(100, mempool_alloc_slab,
2011                 mempool_free_slab, scst_aen_cachep);
2012         if (scst_aen_mempool == NULL) {
2013                 res = -ENOMEM;
2014                 goto out_destroy_sense_mempool;
2015         }
2016
2017         res = scst_sysfs_init();
2018         if (res != 0)
2019                 goto out_destroy_aen_mempool;
2020
2021         if (scst_max_cmd_mem == 0) {
2022                 struct sysinfo si;
2023                 si_meminfo(&si);
2024 #if BITS_PER_LONG == 32
2025                 scst_max_cmd_mem = min(
2026                         (((uint64_t)(si.totalram - si.totalhigh) << PAGE_SHIFT)
2027                                 >> 20) >> 2, (uint64_t)1 << 30);
2028 #else
2029                 scst_max_cmd_mem = (((si.totalram - si.totalhigh) << PAGE_SHIFT)
2030                                         >> 20) >> 2;
2031 #endif
2032         }
2033
2034         if (scst_max_dev_cmd_mem != 0) {
2035                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
2036                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
2037                                 "scst_max_cmd_mem (%d)",
2038                                 scst_max_dev_cmd_mem,
2039                                 scst_max_cmd_mem);
2040                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
2041                 }
2042         } else
2043                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
2044
2045         res = scst_sgv_pools_init(
2046                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
2047         if (res != 0)
2048                 goto out_sysfs_cleanup;
2049
2050 #ifdef CONFIG_SCST_PROC
2051         scst_default_acg = scst_alloc_add_acg(NULL, SCST_DEFAULT_ACG_NAME);
2052         if (scst_default_acg == NULL) {
2053                 res = -ENOMEM;
2054                 goto out_destroy_sgv_pool;
2055         }
2056 #endif
2057
2058         res = scsi_register_interface(&scst_interface);
2059         if (res != 0)
2060 #ifdef CONFIG_SCST_PROC
2061                 goto out_free_acg;
2062 #else
2063                 goto out_destroy_sgv_pool;
2064 #endif
2065
2066         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
2067                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
2068                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
2069                 tasklet_init(&scst_tasklets[i].tasklet,
2070                              (void *)scst_cmd_tasklet,
2071                              (unsigned long)&scst_tasklets[i]);
2072         }
2073
2074         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
2075                 scst_threads);
2076
2077         res = scst_start_all_threads(scst_threads);
2078         if (res < 0)
2079                 goto out_thread_free;
2080
2081 #ifdef CONFIG_SCST_PROC
2082         res = scst_proc_init_module();
2083         if (res != 0)
2084                 goto out_thread_free;
2085 #endif
2086
2087         PRINT_INFO("SCST version %s loaded successfully (max mem for "
2088                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
2089                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
2090
2091         scst_print_config();
2092
2093 out:
2094         TRACE_EXIT_RES(res);
2095         return res;
2096
2097 out_thread_free:
2098         scst_stop_all_threads();
2099
2100         scsi_unregister_interface(&scst_interface);
2101
2102 #ifdef CONFIG_SCST_PROC
2103 out_free_acg:
2104         scst_destroy_acg(scst_default_acg);
2105 #endif
2106
2107 out_destroy_sgv_pool:
2108         scst_sgv_pools_deinit();
2109
2110 out_sysfs_cleanup:
2111         scst_sysfs_cleanup();
2112
2113 out_destroy_aen_mempool:
2114         mempool_destroy(scst_aen_mempool);
2115
2116 out_destroy_sense_mempool:
2117         mempool_destroy(scst_sense_mempool);
2118
2119 out_destroy_ua_mempool:
2120         mempool_destroy(scst_ua_mempool);
2121
2122 out_destroy_mgmt_stub_mempool:
2123         mempool_destroy(scst_mgmt_stub_mempool);
2124
2125 out_destroy_mgmt_mempool:
2126         mempool_destroy(scst_mgmt_mempool);
2127
2128 out_destroy_acg_cache:
2129         kmem_cache_destroy(scst_acgd_cachep);
2130
2131 out_destroy_tgt_cache:
2132         kmem_cache_destroy(scst_tgtd_cachep);
2133
2134 out_destroy_sess_cache:
2135         kmem_cache_destroy(scst_sess_cachep);
2136
2137 out_destroy_cmd_cache:
2138         kmem_cache_destroy(scst_cmd_cachep);
2139
2140 out_destroy_aen_cache:
2141         kmem_cache_destroy(scst_aen_cachep);
2142
2143 out_destroy_sense_cache:
2144         kmem_cache_destroy(scst_sense_cachep);
2145
2146 out_destroy_ua_cache:
2147         kmem_cache_destroy(scst_ua_cachep);
2148
2149 out_destroy_mgmt_stub_cache:
2150         kmem_cache_destroy(scst_mgmt_stub_cachep);
2151
2152 out_destroy_mgmt_cache:
2153         kmem_cache_destroy(scst_mgmt_cachep);
2154
2155 out_lib_exit:
2156         scst_lib_exit();
2157         goto out;
2158 }
2159
2160 static void __exit exit_scst(void)
2161 {
2162         TRACE_ENTRY();
2163
2164         /* ToDo: unregister_cpu_notifier() */
2165
2166 #ifdef CONFIG_SCST_PROC
2167         scst_proc_cleanup_module();
2168 #endif
2169         scst_sysfs_cleanup();
2170
2171         scst_stop_all_threads();
2172
2173         scsi_unregister_interface(&scst_interface);
2174 #ifdef CONFIG_SCST_PROC
2175         scst_destroy_acg(scst_default_acg);
2176 #endif
2177
2178         scst_sgv_pools_deinit();
2179
2180 #define DEINIT_CACHEP(p) do {           \
2181                 kmem_cache_destroy(p);  \
2182                 p = NULL;               \
2183         } while (0)
2184
2185         mempool_destroy(scst_mgmt_mempool);
2186         mempool_destroy(scst_mgmt_stub_mempool);
2187         mempool_destroy(scst_ua_mempool);
2188         mempool_destroy(scst_sense_mempool);
2189         mempool_destroy(scst_aen_mempool);
2190
2191         DEINIT_CACHEP(scst_mgmt_cachep);
2192         DEINIT_CACHEP(scst_mgmt_stub_cachep);
2193         DEINIT_CACHEP(scst_ua_cachep);
2194         DEINIT_CACHEP(scst_sense_cachep);
2195         DEINIT_CACHEP(scst_aen_cachep);
2196         DEINIT_CACHEP(scst_cmd_cachep);
2197         DEINIT_CACHEP(scst_sess_cachep);
2198         DEINIT_CACHEP(scst_tgtd_cachep);
2199         DEINIT_CACHEP(scst_acgd_cachep);
2200
2201         scst_lib_exit();
2202
2203         PRINT_INFO("%s", "SCST unloaded");
2204
2205         TRACE_EXIT();
2206         return;
2207 }
2208
2209 module_init(init_scst);
2210 module_exit(exit_scst);
2211
2212 MODULE_AUTHOR("Vladislav Bolkhovitin");
2213 MODULE_LICENSE("GPL");
2214 MODULE_DESCRIPTION("SCSI target core");
2215 MODULE_VERSION(SCST_VERSION_STRING);