f647b50fad205d087ccf7d10e60884d01c8539d9
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/module.h>
20
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/list.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/sched.h>
28 #include <linux/unistd.h>
29 #include <linux/string.h>
30 #include <linux/kthread.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34 #include "scst_mem.h"
35
36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
37 #warning "HIGHMEM kernel configurations are fully supported, but not\
38  recommended for performance reasons. Consider changing VMSPLIT\
39  option or use a 64-bit configuration instead. See README file for\
40  details."
41 #endif
42
43 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
44 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
45 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
46 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
47  your kernel and CONFIG_SCST_STRICT_SERIALIZING isn't defined.\
48  Pass-through dev handlers will not work."
49 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
50 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
51 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
52  your kernel. Pass-through dev handlers will not work."
53 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
54 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
55
56 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
57 #if !defined(SCST_IO_CONTEXT)
58 #warning "Patch io_context-<kernel-version> was not applied\
59  on your kernel. SCST will be working with not the best performance."
60 #endif
61 #else
62 #warning "There is no patch io_context-<kernel-version>\
63  for your kernel version. For performance reasons it is strongly recommended\
64  to upgrade your kernel to version >= 2.6.26.x."
65 #endif
66
67 /**
68  ** SCST global variables. They are all uninitialized to have their layout in
69  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
70  ** variable separately from nonzero-initialized ones.
71  **/
72
73 /*
74  * All targets, devices and dev_types management is done under this mutex.
75  *
76  * It must NOT be used in any works (schedule_work(), etc.), because
77  * otherwise a deadlock (double lock, actually) is possible, e.g., with
78  * scst_user detach_tgt(), which is called under scst_mutex and calls
79  * flush_scheduled_work().
80  */
81 struct mutex scst_mutex;
82 EXPORT_SYMBOL(scst_mutex);
83
84  /* All 3 protected by scst_mutex */
85 struct list_head scst_template_list;
86 struct list_head scst_dev_list;
87 struct list_head scst_dev_type_list;
88
89 spinlock_t scst_main_lock;
90
91 static struct kmem_cache *scst_mgmt_cachep;
92 mempool_t *scst_mgmt_mempool;
93 static struct kmem_cache *scst_mgmt_stub_cachep;
94 mempool_t *scst_mgmt_stub_mempool;
95 static struct kmem_cache *scst_ua_cachep;
96 mempool_t *scst_ua_mempool;
97 static struct kmem_cache *scst_sense_cachep;
98 mempool_t *scst_sense_mempool;
99 static struct kmem_cache *scst_aen_cachep;
100 mempool_t *scst_aen_mempool;
101 struct kmem_cache *scst_tgtd_cachep;
102 struct kmem_cache *scst_sess_cachep;
103 struct kmem_cache *scst_acgd_cachep;
104
105 struct list_head scst_acg_list;
106 #ifdef CONFIG_SCST_PROC
107 struct scst_acg *scst_default_acg;
108 #endif
109
110 spinlock_t scst_init_lock;
111 wait_queue_head_t scst_init_cmd_list_waitQ;
112 struct list_head scst_init_cmd_list;
113 unsigned int scst_init_poll_cnt;
114
115 struct kmem_cache *scst_cmd_cachep;
116
117 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
118 unsigned long scst_trace_flag;
119 #endif
120
121 unsigned long scst_flags;
122 atomic_t scst_cmd_count;
123
124 struct scst_cmd_lists scst_main_cmd_lists;
125
126 struct scst_tasklet scst_tasklets[NR_CPUS];
127
128 spinlock_t scst_mcmd_lock;
129 struct list_head scst_active_mgmt_cmd_list;
130 struct list_head scst_delayed_mgmt_cmd_list;
131 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
132
133 wait_queue_head_t scst_mgmt_waitQ;
134 spinlock_t scst_mgmt_lock;
135 struct list_head scst_sess_init_list;
136 struct list_head scst_sess_shut_list;
137
138 wait_queue_head_t scst_dev_cmd_waitQ;
139
140 static struct mutex scst_suspend_mutex;
141 /* protected by scst_suspend_mutex */
142 static struct list_head scst_cmd_lists_list;
143
144 static int scst_threads;
145 struct mutex scst_global_threads_mutex;
146 u32 scst_nr_global_threads;
147 static struct list_head scst_global_threads_list;
148 static struct task_struct *scst_init_cmd_thread;
149 static struct task_struct *scst_mgmt_thread;
150 static struct task_struct *scst_mgmt_cmd_thread;
151
152 static int suspend_count;
153
154 static int scst_virt_dev_last_id; /* protected by scst_mutex */
155
156 static unsigned int scst_max_cmd_mem;
157 unsigned int scst_max_dev_cmd_mem;
158
159 module_param_named(scst_threads, scst_threads, int, 0);
160 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
161
162 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, S_IRUGO);
163 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
164         "all SCSI commands of all devices at any given time in MB");
165
166 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, S_IRUGO);
167 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
168         "by all SCSI commands of a device at any given time in MB");
169
170 struct scst_dev_type scst_null_devtype = {
171         .name = "none",
172 };
173
174 static void __scst_resume_activity(void);
175
176 int __scst_register_target_template(struct scst_tgt_template *vtt,
177         const char *version)
178 {
179         int res = 0;
180         struct scst_tgt_template *t;
181         static DEFINE_MUTEX(m);
182
183         TRACE_ENTRY();
184
185         INIT_LIST_HEAD(&vtt->tgt_list);
186
187         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
188                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
189                 res = -EINVAL;
190                 goto out_err;
191         }
192
193         if (!vtt->detect) {
194                 PRINT_ERROR("Target driver %s must have "
195                         "detect() method.", vtt->name);
196                 res = -EINVAL;
197                 goto out_err;
198         }
199
200         if (!vtt->release) {
201                 PRINT_ERROR("Target driver %s must have "
202                         "release() method.", vtt->name);
203                 res = -EINVAL;
204                 goto out_err;
205         }
206
207         if (!vtt->xmit_response) {
208                 PRINT_ERROR("Target driver %s must have "
209                         "xmit_response() method.", vtt->name);
210                 res = -EINVAL;
211                 goto out_err;
212         }
213
214         if (vtt->threads_num < 0) {
215                 PRINT_ERROR("Wrong threads_num value %d for "
216                         "target \"%s\"", vtt->threads_num,
217                         vtt->name);
218                 res = -EINVAL;
219                 goto out_err;
220         }
221
222 #ifndef CONFIG_SCST_PROC
223         if (!vtt->enable_tgt || !vtt->is_tgt_enabled) {
224                 PRINT_WARNING("Target driver %s doesn't have enable_tgt() "
225                         "and/or is_tgt_enabled() method(s). This is unsafe "
226                         "and can lead that initiators connected on the "
227                         "initialization time can see an unexpected set of "
228                         "devices or no devices at all!", vtt->name);
229         }
230 #else
231         if (!vtt->no_proc_entry) {
232                 res = scst_build_proc_target_dir_entries(vtt);
233                 if (res < 0)
234                         goto out_err;
235         }
236 #endif
237
238         res = scst_create_tgtt_sysfs(vtt);
239         if (res)
240                 goto out_sysfs_err;
241
242         if (vtt->rdy_to_xfer == NULL)
243                 vtt->rdy_to_xfer_atomic = 1;
244
245         if (mutex_lock_interruptible(&m) != 0)
246                 goto out_sysfs_err;
247
248         if (mutex_lock_interruptible(&scst_mutex) != 0)
249                 goto out_m_err;
250         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
251                 if (strcmp(t->name, vtt->name) == 0) {
252                         PRINT_ERROR("Target driver %s already registered",
253                                 vtt->name);
254                         mutex_unlock(&scst_mutex);
255                         goto out_m_err;
256                 }
257         }
258         mutex_unlock(&scst_mutex);
259
260         TRACE_DBG("%s", "Calling target driver's detect()");
261         res = vtt->detect(vtt);
262         TRACE_DBG("Target driver's detect() returned %d", res);
263         if (res < 0) {
264                 PRINT_ERROR("%s", "The detect() routine failed");
265                 res = -EINVAL;
266                 goto out_m_err;
267         }
268
269         mutex_lock(&scst_mutex);
270         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
271         mutex_unlock(&scst_mutex);
272
273         res = 0;
274
275         PRINT_INFO("Target template %s registered successfully", vtt->name);
276
277         mutex_unlock(&m);
278
279 out:
280         TRACE_EXIT_RES(res);
281         return res;
282
283 out_m_err:
284         mutex_unlock(&m);
285
286 out_sysfs_err:
287 #ifdef CONFIG_SCST_PROC
288         scst_cleanup_proc_target_dir_entries(vtt);
289 #endif
290         scst_tgtt_sysfs_put(vtt);
291
292 out_err:
293         PRINT_ERROR("Failed to register target template %s", vtt->name);
294         goto out;
295 }
296 EXPORT_SYMBOL(__scst_register_target_template);
297
298 void scst_unregister_target_template(struct scst_tgt_template *vtt)
299 {
300         struct scst_tgt *tgt;
301         struct scst_tgt_template *t;
302         int found = 0;
303
304         TRACE_ENTRY();
305
306         mutex_lock(&scst_mutex);
307
308         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
309                 if (strcmp(t->name, vtt->name) == 0) {
310                         found = 1;
311                         break;
312                 }
313         }
314         if (!found) {
315                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
316                 goto out_err_up;
317         }
318
319 restart:
320         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
321                 mutex_unlock(&scst_mutex);
322                 scst_unregister(tgt);
323                 mutex_lock(&scst_mutex);
324                 goto restart;
325         }
326         list_del(&vtt->scst_template_list_entry);
327
328         mutex_unlock(&scst_mutex);
329
330 #ifdef CONFIG_SCST_PROC
331         scst_cleanup_proc_target_dir_entries(vtt);
332 #endif
333
334         scst_tgtt_sysfs_put(vtt);
335
336         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
337
338 out:
339         TRACE_EXIT();
340         return;
341
342 out_err_up:
343         mutex_unlock(&scst_mutex);
344         goto out;
345 }
346 EXPORT_SYMBOL(scst_unregister_target_template);
347
348 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
349         const char *target_name)
350 {
351         struct scst_tgt *tgt;
352         int rc = 0;
353
354         TRACE_ENTRY();
355
356         tgt = scst_alloc_tgt(vtt);
357         if (tgt == NULL) {
358                 rc = -ENOMEM;
359                 goto out_err;
360         }
361
362         rc = scst_suspend_activity(true);
363         if (rc != 0)
364                 goto out_free_tgt_err;
365
366         if (mutex_lock_interruptible(&scst_mutex) != 0) {
367                 rc = -EINTR;
368                 goto out_resume_free;
369         }
370
371         if (target_name != NULL) {
372 #ifdef CONFIG_SCST_PROC
373                 int len = strlen(target_name) +
374                         strlen(SCST_DEFAULT_ACG_NAME) + 1 + 1;
375
376                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
377                 if (tgt->default_group_name == NULL) {
378                         TRACE(TRACE_OUT_OF_MEM, "Allocation of default "
379                                 "group name failed (tgt %s)", target_name);
380                         rc = -ENOMEM;
381                         goto out_unlock_resume;
382                 }
383                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
384                         target_name);
385 #endif
386
387                 tgt->tgt_name = kmalloc(strlen(target_name) + 1, GFP_KERNEL);
388                 if (tgt->tgt_name == NULL) {
389                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name %s failed",
390                                 target_name);
391                         rc = -ENOMEM;
392 #ifdef CONFIG_SCST_PROC
393                         goto out_free_def_name;
394 #else
395                         goto out_unlock_resume;
396 #endif
397                 }
398                 strcpy(tgt->tgt_name, target_name);
399         } else {
400                 static int tgt_num; /* protected by scst_mutex */
401                 int len = strlen(vtt->name) +
402                         strlen(SCST_DEFAULT_TGT_NAME_SUFFIX) + 11 + 1;
403
404                 tgt->tgt_name = kmalloc(len, GFP_KERNEL);
405                 if (tgt->tgt_name == NULL) {
406                         TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name failed "
407                                 "(template name %s)", vtt->name);
408                         rc = -ENOMEM;
409 #ifdef CONFIG_SCST_PROC
410                         goto out_free_def_name;
411 #else
412                         goto out_unlock_resume;
413 #endif
414                 }
415                 sprintf(tgt->tgt_name, "%s%s%d", vtt->name,
416                         SCST_DEFAULT_TGT_NAME_SUFFIX, tgt_num++);
417         }
418
419         tgt->default_acg = scst_alloc_add_acg(tgt->tgt_name);
420         if (tgt->default_acg == NULL)
421                 goto out_free_tgt_name;
422
423 #ifdef CONFIG_SCST_PROC
424         rc = scst_build_proc_target_entries(tgt);
425         if (rc < 0)
426                 goto out_free_acg;
427 #endif
428
429         rc = scst_create_tgt_sysfs(tgt);
430         if (rc < 0)
431 #ifdef CONFIG_SCST_PROC
432                 goto out_clean_proc;
433 #else
434                 goto out_free_acg;
435 #endif
436
437         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
438
439         mutex_unlock(&scst_mutex);
440         scst_resume_activity();
441
442         PRINT_INFO("Target %s (%p) for template %s registered successfully",
443                 target_name, tgt, vtt->name);
444
445 out:
446         TRACE_EXIT();
447         return tgt;
448
449 #ifdef CONFIG_SCST_PROC
450 out_clean_proc:
451         scst_cleanup_proc_target_entries(tgt);
452 #endif
453
454 out_free_acg:
455         scst_destroy_acg(tgt->default_acg);
456
457 out_free_tgt_name:
458         kfree(tgt->tgt_name);
459
460 #ifdef CONFIG_SCST_PROC
461 out_free_def_name:
462         kfree(tgt->default_group_name);
463 #endif
464
465 out_unlock_resume:
466         mutex_unlock(&scst_mutex);
467
468 out_resume_free:
469         scst_resume_activity();
470
471 out_free_tgt_err:
472         scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
473         tgt = NULL;
474
475 out_err:
476         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
477                 target_name, vtt->name, rc);
478         goto out;
479 }
480 EXPORT_SYMBOL(scst_register);
481
482 static inline int test_sess_list(struct scst_tgt *tgt)
483 {
484         int res;
485         mutex_lock(&scst_mutex);
486         res = list_empty(&tgt->sess_list);
487         mutex_unlock(&scst_mutex);
488         return res;
489 }
490
491 void scst_unregister(struct scst_tgt *tgt)
492 {
493         struct scst_session *sess;
494         struct scst_tgt_template *vtt = tgt->tgtt;
495
496         TRACE_ENTRY();
497
498         scst_tgt_sysfs_prepare_put(tgt);
499
500         TRACE_DBG("%s", "Calling target driver's release()");
501         tgt->tgtt->release(tgt);
502         TRACE_DBG("%s", "Target driver's release() returned");
503
504         mutex_lock(&scst_mutex);
505 again:
506         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
507                 if (sess->shut_phase == SCST_SESS_SPH_READY) {
508                         /*
509                          * Sometimes it's hard for target driver to track all
510                          * its sessions (see scst_local, for example), so let's
511                          * help it.
512                          */
513                         mutex_unlock(&scst_mutex);
514                         scst_unregister_session(sess, 0, NULL);
515                         mutex_lock(&scst_mutex);
516                         goto again;
517                 }
518         }
519         mutex_unlock(&scst_mutex);
520
521         TRACE_DBG("%s", "Waiting for sessions shutdown");
522         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
523         TRACE_DBG("%s", "wait_event() returned");
524
525         scst_suspend_activity(false);
526         mutex_lock(&scst_mutex);
527
528         list_del(&tgt->tgt_list_entry);
529
530 #ifdef CONFIG_SCST_PROC
531         scst_cleanup_proc_target_entries(tgt);
532 #endif
533
534         mutex_unlock(&scst_mutex);
535         scst_resume_activity();
536
537         /*
538          * It should be before freeing of tgt_name, because acg_name
539          * points to it.
540          */
541         scst_destroy_acg(tgt->default_acg);
542
543         kfree(tgt->tgt_name);
544 #ifdef CONFIG_SCST_PROC
545         kfree(tgt->default_group_name);
546 #endif
547
548         del_timer_sync(&tgt->retry_timer);
549
550         scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
551
552         PRINT_INFO("Target %p for template %s unregistered successfully",
553                 tgt, vtt->name);
554
555         TRACE_EXIT();
556         return;
557 }
558 EXPORT_SYMBOL(scst_unregister);
559
560 static int scst_susp_wait(bool interruptible)
561 {
562         int res = 0;
563
564         TRACE_ENTRY();
565
566         if (interruptible) {
567                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
568                         (atomic_read(&scst_cmd_count) == 0),
569                         SCST_SUSPENDING_TIMEOUT);
570                 if (res <= 0) {
571                         __scst_resume_activity();
572                         if (res == 0)
573                                 res = -EBUSY;
574                 } else
575                         res = 0;
576         } else
577                 wait_event(scst_dev_cmd_waitQ,
578                            atomic_read(&scst_cmd_count) == 0);
579
580         TRACE_MGMT_DBG("wait_event() returned %d", res);
581
582         TRACE_EXIT_RES(res);
583         return res;
584 }
585
586 int scst_suspend_activity(bool interruptible)
587 {
588         int res = 0;
589         bool rep = false;
590
591         TRACE_ENTRY();
592
593         if (interruptible) {
594                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
595                         res = -EINTR;
596                         goto out;
597                 }
598         } else
599                 mutex_lock(&scst_suspend_mutex);
600
601         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
602         suspend_count++;
603         if (suspend_count > 1)
604                 goto out_up;
605
606         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
607         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
608         /*
609          * Assignment of SCST_FLAG_SUSPENDING and SCST_FLAG_SUSPENDED must be
610          * ordered with scst_cmd_count. Otherwise lockless logic in
611          * scst_translate_lun() and scst_mgmt_translate_lun() won't work.
612          */
613         smp_mb__after_set_bit();
614
615         /*
616          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
617          * information about scst_user behavior.
618          *
619          * ToDo: make the global suspending unneeded (switch to per-device
620          * reference counting? That would mean to switch off from lockless
621          * implementation of scst_translate_lun().. )
622          */
623
624         if (atomic_read(&scst_cmd_count) != 0) {
625                 PRINT_INFO("Waiting for %d active commands to complete... This "
626                         "might take few minutes for disks or few hours for "
627                         "tapes, if you use long executed commands, like "
628                         "REWIND or FORMAT. In case, if you have a hung user "
629                         "space device (i.e. made using scst_user module) not "
630                         "responding to any commands, if might take virtually "
631                         "forever until the corresponding user space "
632                         "program recovers and starts responding or gets "
633                         "killed.", atomic_read(&scst_cmd_count));
634                 rep = true;
635         }
636
637         res = scst_susp_wait(interruptible);
638         if (res != 0)
639                 goto out_clear;
640
641         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
642         /* See comment about smp_mb() above */
643         smp_mb__after_clear_bit();
644
645         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
646                 atomic_read(&scst_cmd_count));
647
648         res = scst_susp_wait(interruptible);
649         if (res != 0)
650                 goto out_clear;
651
652         if (rep)
653                 PRINT_INFO("%s", "All active commands completed");
654
655 out_up:
656         mutex_unlock(&scst_suspend_mutex);
657
658 out:
659         TRACE_EXIT_RES(res);
660         return res;
661
662 out_clear:
663         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
664         /* See comment about smp_mb() above */
665         smp_mb__after_clear_bit();
666         goto out_up;
667 }
668 EXPORT_SYMBOL(scst_suspend_activity);
669
670 static void __scst_resume_activity(void)
671 {
672         struct scst_cmd_lists *l;
673
674         TRACE_ENTRY();
675
676         suspend_count--;
677         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
678         if (suspend_count > 0)
679                 goto out;
680
681         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
682         /*
683          * The barrier is needed to make sure all woken up threads see the
684          * cleared flag. Not sure if it's really needed, but let's be safe.
685          */
686         smp_mb__after_clear_bit();
687
688         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
689                 wake_up_all(&l->cmd_list_waitQ);
690         }
691         wake_up_all(&scst_init_cmd_list_waitQ);
692
693         spin_lock_irq(&scst_mcmd_lock);
694         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
695                 struct scst_mgmt_cmd *m;
696                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
697                                 mgmt_cmd_list_entry);
698                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
699                         "mgmt cmd list", m);
700                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
701         }
702         spin_unlock_irq(&scst_mcmd_lock);
703         wake_up_all(&scst_mgmt_cmd_list_waitQ);
704
705 out:
706         TRACE_EXIT();
707         return;
708 }
709
710 void scst_resume_activity(void)
711 {
712         TRACE_ENTRY();
713
714         mutex_lock(&scst_suspend_mutex);
715         __scst_resume_activity();
716         mutex_unlock(&scst_suspend_mutex);
717
718         TRACE_EXIT();
719         return;
720 }
721 EXPORT_SYMBOL(scst_resume_activity);
722
723 static int scst_register_device(struct scsi_device *scsidp)
724 {
725         int res = 0;
726         struct scst_device *dev;
727         struct scst_dev_type *dt;
728
729         TRACE_ENTRY();
730
731         res = scst_suspend_activity(true);
732         if (res != 0)
733                 goto out_err;
734
735         if (mutex_lock_interruptible(&scst_mutex) != 0) {
736                 res = -EINTR;
737                 goto out_resume;
738         }
739
740         res = scst_alloc_device(GFP_KERNEL, &dev);
741         if (res != 0)
742                 goto out_up;
743
744         dev->type = scsidp->type;
745
746         dev->virt_name = kmalloc(50, GFP_KERNEL);
747         if (dev->virt_name == NULL) {
748                 PRINT_ERROR("%s", "Unable to alloc device name");
749                 res = -ENOMEM;
750                 goto out_free_dev;
751         }
752         snprintf(dev->virt_name, 50, "%d:%d:%d:%d", scsidp->host->host_no,
753                 scsidp->channel, scsidp->id, scsidp->lun);
754
755 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
756         dev->rq_disk = alloc_disk(1);
757         if (dev->rq_disk == NULL) {
758                 PRINT_ERROR("Unable to alloc disk object for device %s",
759                         dev->virt_name);
760                 res = -ENOMEM;
761                 /* virt_name will be freed in scst_free_dev() */
762                 goto out_free_dev;
763         }
764         dev->rq_disk->major = SCST_MAJOR;
765 #endif
766
767         dev->scsi_dev = scsidp;
768
769         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
770
771         res = scst_create_device_sysfs(dev);
772         if (res != 0)
773                 goto out_free;
774
775         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
776                 if (dt->type == scsidp->type) {
777                         res = scst_assign_dev_handler(dev, dt);
778                         if (res != 0)
779                                 goto out_free;
780                         break;
781                 }
782         }
783
784 out_up:
785         mutex_unlock(&scst_mutex);
786
787 out_resume:
788         scst_resume_activity();
789
790 out_err:
791         if (res == 0) {
792                 PRINT_INFO("Attached to scsi%d, channel %d, id %d, lun %d, "
793                         "type %d", scsidp->host->host_no, scsidp->channel,
794                         scsidp->id, scsidp->lun, scsidp->type);
795         } else {
796                 PRINT_ERROR("Failed to attach to scsi%d, channel %d, id %d, "
797                         "lun %d, type %d", scsidp->host->host_no,
798                         scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
799         }
800
801         TRACE_EXIT_RES(res);
802         return res;
803
804 out_free:
805         list_del(&dev->dev_list_entry);
806 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
807         put_disk(dev->rq_disk);
808 #endif
809
810 out_free_dev:
811         mutex_unlock(&scst_mutex);
812         scst_resume_activity();
813         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
814         goto out_err;
815 }
816
817 static void scst_unregister_device(struct scsi_device *scsidp)
818 {
819         struct scst_device *d, *dev = NULL;
820         struct scst_acg_dev *acg_dev, *aa;
821
822         TRACE_ENTRY();
823
824         scst_suspend_activity(false);
825         mutex_lock(&scst_mutex);
826
827         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
828                 if (d->scsi_dev == scsidp) {
829                         dev = d;
830                         TRACE_DBG("Target device %p found", dev);
831                         break;
832                 }
833         }
834         if (dev == NULL) {
835                 PRINT_ERROR("%s", "Target device not found");
836                 goto out_resume;
837         }
838
839         list_del(&dev->dev_list_entry);
840
841         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
842                                  dev_acg_dev_list_entry) {
843                 scst_acg_remove_dev(acg_dev->acg, dev, true);
844         }
845
846         scst_assign_dev_handler(dev, &scst_null_devtype);
847
848 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
849         put_disk(dev->rq_disk);
850 #endif
851
852         mutex_unlock(&scst_mutex);
853         scst_resume_activity();
854
855         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
856
857         PRINT_INFO("Detached from scsi%d, channel %d, id %d, lun %d, type %d",
858                 scsidp->host->host_no, scsidp->channel, scsidp->id,
859                 scsidp->lun, scsidp->type);
860
861 out:
862         TRACE_EXIT();
863         return;
864
865 out_resume:
866         mutex_unlock(&scst_mutex);
867         scst_resume_activity();
868         goto out;
869 }
870
871 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
872 {
873         int res = 0;
874
875         if (dev_handler->parse == NULL) {
876                 PRINT_ERROR("scst dev handler %s must have "
877                         "parse() method.", dev_handler->name);
878                 res = -EINVAL;
879                 goto out;
880         }
881
882         if (dev_handler->exec == NULL) {
883 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
884                 dev_handler->exec_atomic = 1;
885 #else
886                 dev_handler->exec_atomic = 0;
887 #endif
888         }
889
890         if (dev_handler->dev_done == NULL)
891                 dev_handler->dev_done_atomic = 1;
892
893 out:
894         TRACE_EXIT_RES(res);
895         return res;
896 }
897
898 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
899         const char *dev_name)
900 {
901         int res, rc;
902         struct scst_device *dev = NULL;
903
904         TRACE_ENTRY();
905
906         if (dev_handler == NULL) {
907                 PRINT_ERROR("%s: valid device handler must be supplied",
908                             __func__);
909                 res = -EINVAL;
910                 goto out;
911         }
912
913         if (dev_name == NULL) {
914                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
915                 res = -EINVAL;
916                 goto out;
917         }
918
919         res = scst_dev_handler_check(dev_handler);
920         if (res != 0)
921                 goto out;
922
923         res = scst_suspend_activity(true);
924         if (res != 0)
925                 goto out;
926
927         if (mutex_lock_interruptible(&scst_mutex) != 0) {
928                 res = -EINTR;
929                 goto out_resume;
930         }
931
932         res = scst_alloc_device(GFP_KERNEL, &dev);
933         if (res != 0)
934                 goto out_up;
935
936         dev->type = dev_handler->type;
937         dev->scsi_dev = NULL;
938         dev->virt_name = kstrdup(dev_name, GFP_KERNEL);
939         if (dev->virt_name == NULL) {
940                 PRINT_ERROR("Unable to allocate virt_name for dev %s",
941                         dev_name);
942                 res = -ENOMEM;
943                 goto out_release;
944         }
945         dev->virt_id = scst_virt_dev_last_id++;
946
947         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
948
949         res = dev->virt_id;
950
951         rc = scst_create_device_sysfs(dev);
952         if (rc != 0) {
953                 res = rc;
954                 goto out_free_del;
955         }
956
957         rc = scst_assign_dev_handler(dev, dev_handler);
958         if (rc != 0) {
959                 res = rc;
960                 goto out_free_del;
961         }
962
963 out_up:
964         mutex_unlock(&scst_mutex);
965
966 out_resume:
967         scst_resume_activity();
968
969 out:
970         if (res > 0)
971                 PRINT_INFO("Attached to virtual device %s (id %d)",
972                         dev_name, dev->virt_id);
973         else
974                 PRINT_INFO("Failed to attach to virtual device %s", dev_name);
975
976         TRACE_EXIT_RES(res);
977         return res;
978
979 out_free_del:
980         list_del(&dev->dev_list_entry);
981
982 out_release:
983         mutex_unlock(&scst_mutex);
984         scst_resume_activity();
985         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
986         goto out;
987 }
988 EXPORT_SYMBOL(scst_register_virtual_device);
989
990 void scst_unregister_virtual_device(int id)
991 {
992         struct scst_device *d, *dev = NULL;
993         struct scst_acg_dev *acg_dev, *aa;
994
995         TRACE_ENTRY();
996
997         scst_suspend_activity(false);
998         mutex_lock(&scst_mutex);
999
1000         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
1001                 if (d->virt_id == id) {
1002                         dev = d;
1003                         TRACE_DBG("Target device %p (id %d) found", dev, id);
1004                         break;
1005                 }
1006         }
1007         if (dev == NULL) {
1008                 PRINT_ERROR("Target virtual device (id %d) not found", id);
1009                 goto out_unblock;
1010         }
1011
1012         list_del(&dev->dev_list_entry);
1013
1014         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
1015                                  dev_acg_dev_list_entry) {
1016                 scst_acg_remove_dev(acg_dev->acg, dev, true);
1017         }
1018
1019         scst_assign_dev_handler(dev, &scst_null_devtype);
1020
1021         PRINT_INFO("Detached from virtual device %s (id %d)",
1022                 dev->virt_name, dev->virt_id);
1023
1024 out_unblock:
1025         mutex_unlock(&scst_mutex);
1026         scst_resume_activity();
1027
1028         scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
1029
1030         TRACE_EXIT();
1031         return;
1032 }
1033 EXPORT_SYMBOL(scst_unregister_virtual_device);
1034
1035 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
1036         const char *version)
1037 {
1038         struct scst_dev_type *dt;
1039         struct scst_device *dev;
1040         int res;
1041         int exist;
1042
1043         TRACE_ENTRY();
1044
1045         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1046                 PRINT_ERROR("Incorrect version of dev handler %s",
1047                         dev_type->name);
1048                 res = -EINVAL;
1049                 goto out_error;
1050         }
1051
1052         res = scst_dev_handler_check(dev_type);
1053         if (res != 0)
1054                 goto out_error;
1055
1056 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED)
1057         if (dev_type->exec == NULL) {
1058 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
1059 #if !defined(CONFIG_SCST_STRICT_SERIALIZING)
1060                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1061                         "supported. Consider applying on your kernel patch "
1062                         "scst_exec_req_fifo-<kernel-version> or define "
1063                         "CONFIG_SCST_STRICT_SERIALIZING", dev_type->name);
1064                 res = -EINVAL;
1065                 goto out_error;
1066 #endif /* !defined(CONFIG_SCST_STRICT_SERIALIZING) */
1067 #else  /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1068                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
1069                         "supported. Consider applying on your kernel patch "
1070                         "scst_exec_req_fifo-<kernel-version>", dev_type->name);
1071                 res = -EINVAL;
1072                 goto out_error;
1073 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) */
1074         }
1075 #endif /* !defined(SCSI_EXEC_REQ_FIFO_DEFINED) */
1076
1077         res = scst_suspend_activity(true);
1078         if (res != 0)
1079                 goto out_error;
1080
1081         if (mutex_lock_interruptible(&scst_mutex) != 0) {
1082                 res = -EINTR;
1083                 goto out_err_res;
1084         }
1085
1086         exist = 0;
1087         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1088                 if (strcmp(dt->name, dev_type->name) == 0) {
1089                         PRINT_ERROR("Device type handler \"%s\" already "
1090                                 "exist", dt->name);
1091                         exist = 1;
1092                         break;
1093                 }
1094         }
1095         if (exist)
1096                 goto out_up;
1097
1098 #ifdef CONFIG_SCST_PROC
1099         if (!dev_type->no_proc) {
1100                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1101                 if (res < 0)
1102                         goto out_up;
1103         }
1104 #endif
1105
1106         res = scst_create_devt_sysfs(dev_type);
1107         if (res < 0)
1108                 goto out_free;
1109
1110         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
1111
1112         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1113                 if (dev->scsi_dev == NULL || dev->handler != &scst_null_devtype)
1114                         continue;
1115                 if (dev->scsi_dev->type == dev_type->type)
1116                         scst_assign_dev_handler(dev, dev_type);
1117         }
1118
1119         mutex_unlock(&scst_mutex);
1120         scst_resume_activity();
1121
1122         if (res == 0) {
1123                 PRINT_INFO("Device handler \"%s\" for type %d registered "
1124                         "successfully", dev_type->name, dev_type->type);
1125         }
1126
1127 out:
1128         TRACE_EXIT_RES(res);
1129         return res;
1130
1131 out_free:
1132 #ifdef CONFIG_SCST_PROC
1133         if (!dev_type->no_proc)
1134                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1135 #endif
1136         scst_devt_sysfs_put(dev_type);
1137
1138 out_up:
1139         mutex_unlock(&scst_mutex);
1140
1141 out_err_res:
1142         scst_resume_activity();
1143
1144 out_error:
1145         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
1146                 dev_type->name, dev_type->type);
1147         goto out;
1148 }
1149 EXPORT_SYMBOL(__scst_register_dev_driver);
1150
1151 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
1152 {
1153         struct scst_device *dev;
1154         struct scst_dev_type *dt;
1155         int found = 0;
1156
1157         TRACE_ENTRY();
1158
1159         scst_suspend_activity(false);
1160         mutex_lock(&scst_mutex);
1161
1162         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1163                 if (strcmp(dt->name, dev_type->name) == 0) {
1164                         found = 1;
1165                         break;
1166                 }
1167         }
1168         if (!found) {
1169                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
1170                         dev_type->name);
1171                 goto out_up;
1172         }
1173
1174         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1175                 if (dev->handler == dev_type) {
1176                         scst_assign_dev_handler(dev, &scst_null_devtype);
1177                         TRACE_DBG("Dev handler removed from device %p", dev);
1178                 }
1179         }
1180
1181         list_del(&dev_type->dev_type_list_entry);
1182
1183         mutex_unlock(&scst_mutex);
1184         scst_resume_activity();
1185
1186 #ifdef CONFIG_SCST_PROC
1187         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1188 #endif
1189
1190         scst_devt_sysfs_put(dev_type);
1191
1192         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
1193                    dev_type->name, dev_type->type);
1194
1195 out:
1196         TRACE_EXIT();
1197         return;
1198
1199 out_up:
1200         mutex_unlock(&scst_mutex);
1201         scst_resume_activity();
1202         goto out;
1203 }
1204 EXPORT_SYMBOL(scst_unregister_dev_driver);
1205
1206 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1207         const char *version)
1208 {
1209         int res;
1210
1211         TRACE_ENTRY();
1212
1213         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1214                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1215                         dev_type->name);
1216                 res = -EINVAL;
1217                 goto out_err;
1218         }
1219
1220         res = scst_dev_handler_check(dev_type);
1221         if (res != 0)
1222                 goto out_err;
1223
1224 #ifdef CONFIG_SCST_PROC
1225         if (!dev_type->no_proc) {
1226                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1227                 if (res < 0)
1228                         goto out_err;
1229         }
1230 #endif
1231
1232         res = scst_create_devt_sysfs(dev_type);
1233         if (res < 0)
1234                 goto out_free;
1235
1236         if (dev_type->type != -1) {
1237                 PRINT_INFO("Virtual device handler %s for type %d "
1238                         "registered successfully", dev_type->name,
1239                         dev_type->type);
1240         } else {
1241                 PRINT_INFO("Virtual device handler \"%s\" registered "
1242                         "successfully", dev_type->name);
1243         }
1244
1245 out:
1246         TRACE_EXIT_RES(res);
1247         return res;
1248
1249 out_free:
1250 #ifdef CONFIG_SCST_PROC
1251         if (!dev_type->no_proc)
1252                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1253 #endif
1254
1255         scst_devt_sysfs_put(dev_type);
1256
1257 out_err:
1258         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1259                 dev_type->name);
1260         goto out;
1261 }
1262 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1263
1264 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1265 {
1266         TRACE_ENTRY();
1267
1268 #ifdef CONFIG_SCST_PROC
1269         if (!dev_type->no_proc)
1270                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1271 #endif
1272
1273         scst_devt_sysfs_put(dev_type);
1274
1275         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1276
1277         TRACE_EXIT();
1278         return;
1279 }
1280 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1281
1282 /* Called under scst_mutex */
1283 int scst_add_dev_threads(struct scst_device *dev, int num)
1284 {
1285         int i, res = 0;
1286         int n = 0;
1287         struct scst_cmd_thread_t *thr;
1288         char nm[12];
1289
1290         TRACE_ENTRY();
1291
1292         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1293                 n++;
1294         }
1295
1296         for (i = 0; i < num; i++) {
1297                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1298                 if (!thr) {
1299                         res = -ENOMEM;
1300                         PRINT_ERROR("Failed to allocate thr %d", res);
1301                         goto out_del;
1302                 }
1303                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1304                 nm[ARRAY_SIZE(nm)-1] = '\0';
1305                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1306                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1307                 if (IS_ERR(thr->cmd_thread)) {
1308                         res = PTR_ERR(thr->cmd_thread);
1309                         PRINT_ERROR("kthread_create() failed: %d", res);
1310                         kfree(thr);
1311                         goto out_del;
1312                 }
1313
1314                 list_add(&thr->thread_list_entry, &dev->threads_list);
1315
1316                 /*
1317                  * ToDo: better to use tgt_dev_io_context instead, but we
1318                  * are not ready for that yet.
1319                  */
1320                 __exit_io_context(thr->cmd_thread->io_context);
1321                 thr->cmd_thread->io_context = ioc_task_link(dev->dev_io_ctx);
1322                 TRACE_DBG("Setting dev io ctx %p on thr %d", dev->dev_io_ctx,
1323                         thr->cmd_thread->pid);
1324
1325                 wake_up_process(thr->cmd_thread);
1326         }
1327
1328 out:
1329         TRACE_EXIT_RES(res);
1330         return res;
1331
1332 out_del:
1333         scst_del_dev_threads(dev, i);
1334         goto out;
1335 }
1336
1337 /* Called under scst_mutex and suspended activity */
1338 static int scst_create_dev_threads(struct scst_device *dev)
1339 {
1340         int res = 0;
1341         int threads_num;
1342
1343         TRACE_ENTRY();
1344
1345         if (dev->handler->threads_num <= 0)
1346                 goto out;
1347
1348         threads_num = dev->handler->threads_num;
1349
1350         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1351         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1352         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1353
1354         res = scst_add_dev_threads(dev, threads_num);
1355         if (res != 0)
1356                 goto out;
1357
1358         mutex_lock(&scst_suspend_mutex);
1359         list_add_tail(&dev->cmd_lists.lists_list_entry,
1360                 &scst_cmd_lists_list);
1361         mutex_unlock(&scst_suspend_mutex);
1362
1363         dev->p_cmd_lists = &dev->cmd_lists;
1364
1365 out:
1366         TRACE_EXIT_RES(res);
1367         return res;
1368 }
1369
1370 /* Called under scst_mutex */
1371 void scst_del_dev_threads(struct scst_device *dev, int num)
1372 {
1373         struct scst_cmd_thread_t *ct, *tmp;
1374         int i = 0;
1375
1376         TRACE_ENTRY();
1377
1378         if (num == 0)
1379                 goto out;
1380
1381         list_for_each_entry_safe_reverse(ct, tmp, &dev->threads_list,
1382                                 thread_list_entry) {
1383                 int rc;
1384                 struct scst_tgt_dev *tgt_dev;
1385
1386                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1387                                 dev_tgt_dev_list_entry) {
1388                         struct scst_thr_data_hdr *td;
1389                         td = __scst_find_thr_data(tgt_dev, ct->cmd_thread);
1390                         if (td != NULL) {
1391                                 scst_thr_data_put(td);
1392                                 break;
1393                         }
1394                 }
1395
1396                 rc = kthread_stop(ct->cmd_thread);
1397                 if (rc < 0)
1398                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1399
1400                 list_del(&ct->thread_list_entry);
1401                 kfree(ct);
1402
1403                 if ((num > 0) && (++i >= num))
1404                         break;
1405         }
1406
1407 out:
1408         TRACE_EXIT();
1409         return;
1410 }
1411
1412 /* Called under scst_mutex and suspended activity */
1413 static void scst_stop_dev_threads(struct scst_device *dev)
1414 {
1415         TRACE_ENTRY();
1416
1417         if (list_empty(&dev->threads_list))
1418                 goto out;
1419
1420         scst_del_dev_threads(dev, -1);
1421
1422         if (dev->p_cmd_lists == &dev->cmd_lists) {
1423                 mutex_lock(&scst_suspend_mutex);
1424                 list_del(&dev->cmd_lists.lists_list_entry);
1425                 mutex_unlock(&scst_suspend_mutex);
1426         }
1427
1428 out:
1429         TRACE_EXIT();
1430         return;
1431 }
1432
1433 /* The activity supposed to be suspended and scst_mutex held */
1434 int scst_assign_dev_handler(struct scst_device *dev,
1435         struct scst_dev_type *handler)
1436 {
1437         int res = 0;
1438         struct scst_tgt_dev *tgt_dev;
1439         LIST_HEAD(attached_tgt_devs);
1440
1441         TRACE_ENTRY();
1442
1443         sBUG_ON(handler == NULL);
1444
1445         if (dev->handler == handler)
1446                 goto out;
1447
1448         if (dev->handler == NULL)
1449                 goto assign;
1450
1451         if (dev->handler->detach_tgt) {
1452                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1453                                 dev_tgt_dev_list_entry) {
1454                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1455                                 tgt_dev);
1456                         dev->handler->detach_tgt(tgt_dev);
1457                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1458                 }
1459         }
1460
1461         if (dev->handler->detach) {
1462                 TRACE_DBG("%s", "Calling dev handler's detach()");
1463                 dev->handler->detach(dev);
1464                 TRACE_DBG("%s", "Old handler's detach() returned");
1465         }
1466
1467         scst_stop_dev_threads(dev);
1468
1469         scst_devt_dev_sysfs_put(dev);
1470
1471 assign:
1472         dev->handler = handler;
1473
1474         if (handler == NULL)
1475                 goto out;
1476
1477         res = scst_create_devt_dev_sysfs(dev);
1478         if (res != 0)
1479                 goto out_null;
1480
1481         res = scst_create_dev_threads(dev);
1482         if (res != 0)
1483                 goto out_remove_sysfs;
1484
1485         if (handler->attach) {
1486                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1487                 res = handler->attach(dev);
1488                 TRACE_DBG("New dev handler's attach() returned %d", res);
1489                 if (res != 0) {
1490                         PRINT_ERROR("New device handler's %s attach() "
1491                                 "failed: %d", handler->name, res);
1492                         goto out_thr_null;
1493                 }
1494         }
1495
1496         if (handler->attach_tgt) {
1497                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1498                                 dev_tgt_dev_list_entry) {
1499                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1500                                 tgt_dev);
1501                         res = handler->attach_tgt(tgt_dev);
1502                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1503                         if (res != 0) {
1504                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1505                                     "failed: %d", handler->name, res);
1506                                 goto out_err_detach_tgt;
1507                         }
1508                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1509                                 &attached_tgt_devs);
1510                 }
1511         }
1512
1513 out:
1514         TRACE_EXIT_RES(res);
1515         return res;
1516
1517 out_err_detach_tgt:
1518         if (handler && handler->detach_tgt) {
1519                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1520                                  extra_tgt_dev_list_entry) {
1521                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1522                                 tgt_dev);
1523                         handler->detach_tgt(tgt_dev);
1524                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1525                 }
1526         }
1527         if (handler && handler->detach) {
1528                 TRACE_DBG("%s", "Calling handler's detach()");
1529                 handler->detach(dev);
1530                 TRACE_DBG("%s", "Handler's detach() returned");
1531         }
1532
1533 out_thr_null:
1534         scst_stop_dev_threads(dev);
1535
1536 out_remove_sysfs:
1537         scst_devt_dev_sysfs_put(dev);
1538
1539 out_null:
1540         dev->handler = &scst_null_devtype;
1541         goto out;
1542 }
1543
1544 int scst_global_threads_count(void)
1545 {
1546         int i;
1547
1548         /*
1549          * Just to lower the race window, when user can get just changed value
1550          */
1551         mutex_lock(&scst_global_threads_mutex);
1552         i = scst_nr_global_threads;
1553         mutex_unlock(&scst_global_threads_mutex);
1554         return i;
1555 }
1556
1557 static void scst_threads_info_init(void)
1558 {
1559         mutex_init(&scst_global_threads_mutex);
1560         INIT_LIST_HEAD(&scst_global_threads_list);
1561 }
1562
1563 /* scst_global_threads_mutex supposed to be held */
1564 void __scst_del_global_threads(int num)
1565 {
1566         struct scst_cmd_thread_t *ct, *tmp;
1567
1568         TRACE_ENTRY();
1569
1570         if (num == 0)
1571                 goto out;
1572
1573         list_for_each_entry_safe(ct, tmp, &scst_global_threads_list,
1574                                 thread_list_entry) {
1575                 int res;
1576
1577                 res = kthread_stop(ct->cmd_thread);
1578                 if (res < 0)
1579                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1580                 list_del(&ct->thread_list_entry);
1581                 kfree(ct);
1582                 scst_nr_global_threads--;
1583                 --num;
1584                 if (num == 0)
1585                         break;
1586         }
1587
1588 out:
1589         TRACE_EXIT();
1590         return;
1591 }
1592
1593 /* scst_global_threads_mutex supposed to be held */
1594 int __scst_add_global_threads(int num)
1595 {
1596         int res = 0, i;
1597         static int scst_thread_num;
1598
1599         TRACE_ENTRY();
1600
1601         for (i = 0; i < num; i++) {
1602                 struct scst_cmd_thread_t *thr;
1603
1604                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1605                 if (!thr) {
1606                         res = -ENOMEM;
1607                         PRINT_ERROR("fail to allocate thr %d", res);
1608                         goto out_error;
1609                 }
1610                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1611                         &scst_main_cmd_lists, "scsi_tgt%d",
1612                         scst_thread_num++);
1613                 if (IS_ERR(thr->cmd_thread)) {
1614                         res = PTR_ERR(thr->cmd_thread);
1615                         PRINT_ERROR("kthread_create() failed: %d", res);
1616                         kfree(thr);
1617                         goto out_error;
1618                 }
1619
1620                 list_add(&thr->thread_list_entry, &scst_global_threads_list);
1621                 scst_nr_global_threads++;
1622
1623                 wake_up_process(thr->cmd_thread);
1624         }
1625         res = 0;
1626
1627 out:
1628         TRACE_EXIT_RES(res);
1629         return res;
1630
1631 out_error:
1632         __scst_del_global_threads(i);
1633         goto out;
1634 }
1635
1636 int scst_add_global_threads(int num)
1637 {
1638         int res;
1639
1640         TRACE_ENTRY();
1641
1642         mutex_lock(&scst_global_threads_mutex);
1643         res = __scst_add_global_threads(num);
1644         mutex_unlock(&scst_global_threads_mutex);
1645
1646         TRACE_EXIT_RES(res);
1647         return res;
1648 }
1649 EXPORT_SYMBOL(scst_add_global_threads);
1650
1651 void scst_del_global_threads(int num)
1652 {
1653         TRACE_ENTRY();
1654
1655         mutex_lock(&scst_global_threads_mutex);
1656         __scst_del_global_threads(num);
1657         mutex_unlock(&scst_global_threads_mutex);
1658
1659         TRACE_EXIT();
1660         return;
1661 }
1662 EXPORT_SYMBOL(scst_del_global_threads);
1663
1664 static void scst_stop_all_threads(void)
1665 {
1666         TRACE_ENTRY();
1667
1668         mutex_lock(&scst_global_threads_mutex);
1669         __scst_del_global_threads(-1);
1670         if (scst_mgmt_cmd_thread)
1671                 kthread_stop(scst_mgmt_cmd_thread);
1672         if (scst_mgmt_thread)
1673                 kthread_stop(scst_mgmt_thread);
1674         if (scst_init_cmd_thread)
1675                 kthread_stop(scst_init_cmd_thread);
1676         mutex_unlock(&scst_global_threads_mutex);
1677
1678         TRACE_EXIT();
1679         return;
1680 }
1681
1682 static int scst_start_all_threads(int num)
1683 {
1684         int res;
1685
1686         TRACE_ENTRY();
1687
1688         mutex_lock(&scst_global_threads_mutex);
1689         res = __scst_add_global_threads(num);
1690         if (res < 0)
1691                 goto out;
1692
1693         scst_init_cmd_thread = kthread_run(scst_init_thread,
1694                 NULL, "scsi_tgt_init");
1695         if (IS_ERR(scst_init_cmd_thread)) {
1696                 res = PTR_ERR(scst_init_cmd_thread);
1697                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1698                 scst_init_cmd_thread = NULL;
1699                 goto out;
1700         }
1701
1702         scst_mgmt_cmd_thread = kthread_run(scst_tm_thread,
1703                 NULL, "scsi_tm");
1704         if (IS_ERR(scst_mgmt_cmd_thread)) {
1705                 res = PTR_ERR(scst_mgmt_cmd_thread);
1706                 PRINT_ERROR("kthread_create() for TM failed: %d", res);
1707                 scst_mgmt_cmd_thread = NULL;
1708                 goto out;
1709         }
1710
1711         scst_mgmt_thread = kthread_run(scst_global_mgmt_thread,
1712                 NULL, "scsi_tgt_mgmt");
1713         if (IS_ERR(scst_mgmt_thread)) {
1714                 res = PTR_ERR(scst_mgmt_thread);
1715                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1716                 scst_mgmt_thread = NULL;
1717                 goto out;
1718         }
1719
1720 out:
1721         mutex_unlock(&scst_global_threads_mutex);
1722         TRACE_EXIT_RES(res);
1723         return res;
1724 }
1725
1726 void scst_get(void)
1727 {
1728         __scst_get(0);
1729 }
1730 EXPORT_SYMBOL(scst_get);
1731
1732 void scst_put(void)
1733 {
1734         __scst_put();
1735 }
1736 EXPORT_SYMBOL(scst_put);
1737
1738 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1739 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1740 #else
1741 static int scst_add(struct device *cdev, struct class_interface *intf)
1742 #endif
1743 {
1744         struct scsi_device *scsidp;
1745         int res = 0;
1746
1747         TRACE_ENTRY();
1748
1749 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1750         scsidp = to_scsi_device(cdev->dev);
1751 #else
1752         scsidp = to_scsi_device(cdev->parent);
1753 #endif
1754
1755         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1756                 res = scst_register_device(scsidp);
1757
1758         TRACE_EXIT();
1759         return res;
1760 }
1761
1762 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1763 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1764 #else
1765 static void scst_remove(struct device *cdev, struct class_interface *intf)
1766 #endif
1767 {
1768         struct scsi_device *scsidp;
1769
1770         TRACE_ENTRY();
1771
1772 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1773         scsidp = to_scsi_device(cdev->dev);
1774 #else
1775         scsidp = to_scsi_device(cdev->parent);
1776 #endif
1777
1778         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1779                 scst_unregister_device(scsidp);
1780
1781         TRACE_EXIT();
1782         return;
1783 }
1784
1785 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1786 static struct class_interface scst_interface = {
1787         .add = scst_add,
1788         .remove = scst_remove,
1789 };
1790 #else
1791 static struct class_interface scst_interface = {
1792         .add_dev = scst_add,
1793         .remove_dev = scst_remove,
1794 };
1795 #endif
1796
1797 static void __init scst_print_config(void)
1798 {
1799         char buf[128];
1800         int i, j;
1801
1802         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1803         j = i;
1804
1805 #ifdef CONFIG_SCST_STRICT_SERIALIZING
1806         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1807 #endif
1808
1809 #ifdef CONFIG_SCST_EXTRACHECKS
1810         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1811                 (j == i) ? "" : ", ");
1812 #endif
1813
1814 #ifdef CONFIG_SCST_TRACING
1815         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1816                 (j == i) ? "" : ", ");
1817 #endif
1818
1819 #ifdef CONFIG_SCST_DEBUG
1820         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1821                 (j == i) ? "" : ", ");
1822 #endif
1823
1824 #ifdef CONFIG_SCST_DEBUG_TM
1825         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1826                 (j == i) ? "" : ", ");
1827 #endif
1828
1829 #ifdef CONFIG_SCST_DEBUG_RETRY
1830         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1831                 (j == i) ? "" : ", ");
1832 #endif
1833
1834 #ifdef CONFIG_SCST_DEBUG_OOM
1835         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1836                 (j == i) ? "" : ", ");
1837 #endif
1838
1839 #ifdef CONFIG_SCST_DEBUG_SN
1840         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1841                 (j == i) ? "" : ", ");
1842 #endif
1843
1844 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
1845         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1846                 (j == i) ? "" : ", ");
1847 #endif
1848
1849 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1850         i += snprintf(&buf[i], sizeof(buf) - i,
1851                 "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1852                 (j == i) ? "" : ", ");
1853 #endif
1854
1855 #ifdef CONFIG_SCST_STRICT_SECURITY
1856         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1857                 (j == i) ? "" : ", ");
1858 #endif
1859
1860         if (j != i)
1861                 PRINT_INFO("%s", buf);
1862 }
1863
1864 static int __init init_scst(void)
1865 {
1866         int res, i;
1867         int scst_num_cpus;
1868
1869         TRACE_ENTRY();
1870
1871 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1872         {
1873                 struct scsi_request *req;
1874                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1875                         sizeof(req->sr_sense_buffer));
1876         }
1877 #else
1878         {
1879                 struct scsi_sense_hdr *shdr;
1880                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1881         }
1882 #endif
1883         {
1884                 struct scst_tgt_dev *t;
1885                 struct scst_cmd *c;
1886                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1887                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1888         }
1889
1890 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
1891 #if !defined(SCST_IO_CONTEXT)
1892         PRINT_WARNING("%s", "Patch io_context was not applied on "
1893                 "your kernel. SCST will be working with not the best "
1894                 "performance.");
1895 #endif
1896 #else
1897         PRINT_WARNING("%s", "There is no patch io_context for your kernel "
1898                 "version. For performance reasons it is strongly recommended "
1899                 "to upgrade your kernel to version >= 2.6.27.x.");
1900 #endif
1901
1902         mutex_init(&scst_mutex);
1903         INIT_LIST_HEAD(&scst_template_list);
1904         INIT_LIST_HEAD(&scst_dev_list);
1905         INIT_LIST_HEAD(&scst_dev_type_list);
1906         spin_lock_init(&scst_main_lock);
1907         INIT_LIST_HEAD(&scst_acg_list);
1908         spin_lock_init(&scst_init_lock);
1909         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1910         INIT_LIST_HEAD(&scst_init_cmd_list);
1911 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1912         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1913 #endif
1914         atomic_set(&scst_cmd_count, 0);
1915         spin_lock_init(&scst_mcmd_lock);
1916         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1917         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1918         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1919         init_waitqueue_head(&scst_mgmt_waitQ);
1920         spin_lock_init(&scst_mgmt_lock);
1921         INIT_LIST_HEAD(&scst_sess_init_list);
1922         INIT_LIST_HEAD(&scst_sess_shut_list);
1923         init_waitqueue_head(&scst_dev_cmd_waitQ);
1924         mutex_init(&scst_suspend_mutex);
1925         INIT_LIST_HEAD(&scst_cmd_lists_list);
1926         scst_virt_dev_last_id = 1;
1927         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1928         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1929         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1930         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1931                 &scst_cmd_lists_list);
1932
1933         res = scst_lib_init();
1934         if (res != 0)
1935                 goto out;
1936
1937         scst_num_cpus = num_online_cpus();
1938
1939         /* ToDo: register_cpu_notifier() */
1940
1941         if (scst_threads == 0)
1942                 scst_threads = scst_num_cpus;
1943
1944         if (scst_threads < 1) {
1945                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1946                 scst_threads = scst_num_cpus;
1947         }
1948
1949         scst_threads_info_init();
1950
1951 #define INIT_CACHEP(p, s, o) do {                                       \
1952                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1953                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1954                           sizeof(struct s));                            \
1955                 if (p == NULL) {                                        \
1956                         res = -ENOMEM;                                  \
1957                         goto o;                                         \
1958                 }                                                       \
1959         } while (0)
1960
1961         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit);
1962         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1963                         out_destroy_mgmt_cache);
1964         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1965                         out_destroy_mgmt_stub_cache);
1966         {
1967                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1968                 INIT_CACHEP(scst_sense_cachep, scst_sense,
1969                             out_destroy_ua_cache);
1970         }
1971         INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
1972         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
1973         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1974         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1975         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1976
1977         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1978                 mempool_free_slab, scst_mgmt_cachep);
1979         if (scst_mgmt_mempool == NULL) {
1980                 res = -ENOMEM;
1981                 goto out_destroy_acg_cache;
1982         }
1983
1984         /*
1985          * All mgmt stubs, UAs and sense buffers are bursty and loosing them
1986          * may have fatal consequences, so let's have big pools for them.
1987          */
1988
1989         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1990                 mempool_free_slab, scst_mgmt_stub_cachep);
1991         if (scst_mgmt_stub_mempool == NULL) {
1992                 res = -ENOMEM;
1993                 goto out_destroy_mgmt_mempool;
1994         }
1995
1996         scst_ua_mempool = mempool_create(512, mempool_alloc_slab,
1997                 mempool_free_slab, scst_ua_cachep);
1998         if (scst_ua_mempool == NULL) {
1999                 res = -ENOMEM;
2000                 goto out_destroy_mgmt_stub_mempool;
2001         }
2002
2003         scst_sense_mempool = mempool_create(1024, mempool_alloc_slab,
2004                 mempool_free_slab, scst_sense_cachep);
2005         if (scst_sense_mempool == NULL) {
2006                 res = -ENOMEM;
2007                 goto out_destroy_ua_mempool;
2008         }
2009
2010         scst_aen_mempool = mempool_create(100, mempool_alloc_slab,
2011                 mempool_free_slab, scst_aen_cachep);
2012         if (scst_aen_mempool == NULL) {
2013                 res = -ENOMEM;
2014                 goto out_destroy_sense_mempool;
2015         }
2016
2017         res = scst_sysfs_init();
2018         if (res != 0)
2019                 goto out_destroy_aen_mempool;
2020
2021         if (scst_max_cmd_mem == 0) {
2022                 struct sysinfo si;
2023                 si_meminfo(&si);
2024 #if BITS_PER_LONG == 32
2025                 scst_max_cmd_mem = min(
2026                         (((uint64_t)(si.totalram - si.totalhigh) << PAGE_SHIFT)
2027                                 >> 20) >> 2, (uint64_t)1 << 30);
2028 #else
2029                 scst_max_cmd_mem = (((si.totalram - si.totalhigh) << PAGE_SHIFT)
2030                                         >> 20) >> 2;
2031 #endif
2032         }
2033
2034         if (scst_max_dev_cmd_mem != 0) {
2035                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
2036                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
2037                                 "scst_max_cmd_mem (%d)",
2038                                 scst_max_dev_cmd_mem,
2039                                 scst_max_cmd_mem);
2040                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
2041                 }
2042         } else
2043                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
2044
2045         res = scst_sgv_pools_init(
2046                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
2047         if (res != 0)
2048                 goto out_sysfs_cleanup;
2049
2050 #ifdef CONFIG_SCST_PROC
2051         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
2052         if (scst_default_acg == NULL) {
2053                 res = -ENOMEM;
2054                 goto out_destroy_sgv_pool;
2055         }
2056 #endif
2057
2058         res = scsi_register_interface(&scst_interface);
2059         if (res != 0)
2060 #ifdef CONFIG_SCST_PROC
2061                 goto out_free_acg;
2062 #else
2063                 goto out_destroy_sgv_pool;
2064 #endif
2065
2066         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
2067                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
2068                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
2069                 tasklet_init(&scst_tasklets[i].tasklet,
2070                              (void *)scst_cmd_tasklet,
2071                              (unsigned long)&scst_tasklets[i]);
2072         }
2073
2074         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
2075                 scst_threads);
2076
2077         res = scst_start_all_threads(scst_threads);
2078         if (res < 0)
2079                 goto out_thread_free;
2080
2081 #ifdef CONFIG_SCST_PROC
2082         res = scst_proc_init_module();
2083         if (res != 0)
2084                 goto out_thread_free;
2085 #endif
2086
2087         PRINT_INFO("SCST version %s loaded successfully (max mem for "
2088                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
2089                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
2090
2091         scst_print_config();
2092
2093 out:
2094         TRACE_EXIT_RES(res);
2095         return res;
2096
2097 out_thread_free:
2098         scst_stop_all_threads();
2099
2100         scsi_unregister_interface(&scst_interface);
2101
2102 #ifdef CONFIG_SCST_PROC
2103 out_free_acg:
2104         scst_destroy_acg(scst_default_acg);
2105 #endif
2106
2107 out_destroy_sgv_pool:
2108         scst_sgv_pools_deinit();
2109
2110 out_sysfs_cleanup:
2111         scst_sysfs_cleanup();
2112
2113 out_destroy_aen_mempool:
2114         mempool_destroy(scst_aen_mempool);
2115
2116 out_destroy_sense_mempool:
2117         mempool_destroy(scst_sense_mempool);
2118
2119 out_destroy_ua_mempool:
2120         mempool_destroy(scst_ua_mempool);
2121
2122 out_destroy_mgmt_stub_mempool:
2123         mempool_destroy(scst_mgmt_stub_mempool);
2124
2125 out_destroy_mgmt_mempool:
2126         mempool_destroy(scst_mgmt_mempool);
2127
2128 out_destroy_acg_cache:
2129         kmem_cache_destroy(scst_acgd_cachep);
2130
2131 out_destroy_tgt_cache:
2132         kmem_cache_destroy(scst_tgtd_cachep);
2133
2134 out_destroy_sess_cache:
2135         kmem_cache_destroy(scst_sess_cachep);
2136
2137 out_destroy_cmd_cache:
2138         kmem_cache_destroy(scst_cmd_cachep);
2139
2140 out_destroy_aen_cache:
2141         kmem_cache_destroy(scst_aen_cachep);
2142
2143 out_destroy_sense_cache:
2144         kmem_cache_destroy(scst_sense_cachep);
2145
2146 out_destroy_ua_cache:
2147         kmem_cache_destroy(scst_ua_cachep);
2148
2149 out_destroy_mgmt_stub_cache:
2150         kmem_cache_destroy(scst_mgmt_stub_cachep);
2151
2152 out_destroy_mgmt_cache:
2153         kmem_cache_destroy(scst_mgmt_cachep);
2154
2155 out_lib_exit:
2156         scst_lib_exit();
2157         goto out;
2158 }
2159
2160 static void __exit exit_scst(void)
2161 {
2162         TRACE_ENTRY();
2163
2164         /* ToDo: unregister_cpu_notifier() */
2165
2166 #ifdef CONFIG_SCST_PROC
2167         scst_proc_cleanup_module();
2168 #endif
2169         scst_sysfs_cleanup();
2170
2171         scst_stop_all_threads();
2172
2173         scsi_unregister_interface(&scst_interface);
2174 #ifdef CONFIG_SCST_PROC
2175         scst_destroy_acg(scst_default_acg);
2176 #endif
2177
2178         scst_sgv_pools_deinit();
2179
2180 #define DEINIT_CACHEP(p) do {           \
2181                 kmem_cache_destroy(p);  \
2182                 p = NULL;               \
2183         } while (0)
2184
2185         mempool_destroy(scst_mgmt_mempool);
2186         mempool_destroy(scst_mgmt_stub_mempool);
2187         mempool_destroy(scst_ua_mempool);
2188         mempool_destroy(scst_sense_mempool);
2189         mempool_destroy(scst_aen_mempool);
2190
2191         DEINIT_CACHEP(scst_mgmt_cachep);
2192         DEINIT_CACHEP(scst_mgmt_stub_cachep);
2193         DEINIT_CACHEP(scst_ua_cachep);
2194         DEINIT_CACHEP(scst_sense_cachep);
2195         DEINIT_CACHEP(scst_aen_cachep);
2196         DEINIT_CACHEP(scst_cmd_cachep);
2197         DEINIT_CACHEP(scst_sess_cachep);
2198         DEINIT_CACHEP(scst_tgtd_cachep);
2199         DEINIT_CACHEP(scst_acgd_cachep);
2200
2201         scst_lib_exit();
2202
2203         PRINT_INFO("%s", "SCST unloaded");
2204
2205         TRACE_EXIT();
2206         return;
2207 }
2208
2209 module_init(init_scst);
2210 module_exit(exit_scst);
2211
2212 MODULE_AUTHOR("Vladislav Bolkhovitin");
2213 MODULE_LICENSE("GPL");
2214 MODULE_DESCRIPTION("SCSI target core");
2215 MODULE_VERSION(SCST_VERSION_STRING);