Minor cleanups and docs updates
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/module.h>
20
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/list.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/sched.h>
28 #include <linux/unistd.h>
29 #include <linux/string.h>
30 #include <linux/kthread.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34 #include "scst_mem.h"
35
36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
37 #warning "HIGHMEM kernel configurations are fully supported, but not \
38         recommended for performance reasons. Consider change VMSPLIT \
39         option or use 64-bit configuration instead. See README file for \
40         details."
41 #endif
42
43 #ifdef SCST_HIGHMEM
44 #error "SCST_HIGHMEM configuration isn't supported and broken, because there \
45         is no real point to support it, at least it definitely doesn't worth \
46         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
47         or in 64-bit configuration instead. See README file for details."
48 #endif
49
50 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
51 #warning "Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
52         your kernel and STRICT_SERIALIZING isn't defined. Pass-through dev \
53         handlers will not be supported."
54 #endif
55
56 /**
57  ** SCST global variables. They are all uninitialized to have their layout in
58  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
59  ** variable separately from nonzero-initialized ones.
60  **/
61
62 /*
63  * All targets, devices and dev_types management is done under this mutex.
64  *
65  * It must NOT be used in any works (schedule_work(), etc.), because
66  * otherwise a deadlock (double lock, actually) is possible, e.g., with
67  * scst_user detach_tgt(), which is called under scst_mutex and calls
68  * flush_scheduled_work().
69  */
70 struct mutex scst_mutex;
71
72 struct list_head scst_template_list;
73 struct list_head scst_dev_list;
74 struct list_head scst_dev_type_list;
75
76 spinlock_t scst_main_lock;
77
78 struct kmem_cache *scst_mgmt_cachep;
79 mempool_t *scst_mgmt_mempool;
80 struct kmem_cache *scst_mgmt_stub_cachep;
81 mempool_t *scst_mgmt_stub_mempool;
82 struct kmem_cache *scst_ua_cachep;
83 mempool_t *scst_ua_mempool;
84 struct kmem_cache *scst_sense_cachep;
85 mempool_t *scst_sense_mempool;
86 struct kmem_cache *scst_tgtd_cachep;
87 struct kmem_cache *scst_sess_cachep;
88 struct kmem_cache *scst_acgd_cachep;
89
90 struct list_head scst_acg_list;
91 struct scst_acg *scst_default_acg;
92
93 spinlock_t scst_init_lock;
94 wait_queue_head_t scst_init_cmd_list_waitQ;
95 struct list_head scst_init_cmd_list;
96 unsigned int scst_init_poll_cnt;
97
98 struct kmem_cache *scst_cmd_cachep;
99
100 #if defined(DEBUG) || defined(TRACING)
101 unsigned long scst_trace_flag;
102 #endif
103
104 unsigned long scst_flags;
105 atomic_t scst_cmd_count;
106
107 struct scst_cmd_lists scst_main_cmd_lists;
108
109 struct scst_tasklet scst_tasklets[NR_CPUS];
110
111 spinlock_t scst_mcmd_lock;
112 struct list_head scst_active_mgmt_cmd_list;
113 struct list_head scst_delayed_mgmt_cmd_list;
114 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
115
116 wait_queue_head_t scst_mgmt_waitQ;
117 spinlock_t scst_mgmt_lock;
118 struct list_head scst_sess_init_list;
119 struct list_head scst_sess_shut_list;
120
121 wait_queue_head_t scst_dev_cmd_waitQ;
122
123 struct mutex scst_suspend_mutex;
124 struct list_head scst_cmd_lists_list;
125
126 static int scst_threads;
127 struct scst_threads_info_t scst_threads_info;
128
129 static int suspend_count;
130
131 static int scst_virt_dev_last_id; /* protected by scst_mutex */
132
133 /*
134  * This buffer and lock are intended to avoid memory allocation, which
135  * could fail in improper places.
136  */
137 spinlock_t scst_temp_UA_lock;
138 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
139
140 unsigned int scst_max_cmd_mem;
141 unsigned int scst_max_dev_cmd_mem;
142
143 module_param_named(scst_threads, scst_threads, int, 0);
144 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
145
146 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, 0);
147 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
148         "all SCSI commands of all devices at any given time in MB");
149
150 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, 0);
151 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
152         "by all SCSI commands of a device at any given time in MB");
153
154 struct scst_dev_type scst_null_devtype = {
155         .name = "none",
156 };
157
158 static void __scst_resume_activity(void);
159
160 int __scst_register_target_template(struct scst_tgt_template *vtt,
161         const char *version)
162 {
163         int res = 0;
164         struct scst_tgt_template *t;
165         static DEFINE_MUTEX(m);
166
167         TRACE_ENTRY();
168
169         INIT_LIST_HEAD(&vtt->tgt_list);
170
171         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
172                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
173                 res = -EINVAL;
174                 goto out_err;
175         }
176
177         if (!vtt->detect) {
178                 PRINT_ERROR("Target driver %s doesn't have a "
179                         "detect() method.", vtt->name);
180                 res = -EINVAL;
181                 goto out_err;
182         }
183
184         if (!vtt->release) {
185                 PRINT_ERROR("Target driver %s doesn't have a "
186                         "release() method.", vtt->name);
187                 res = -EINVAL;
188                 goto out_err;
189         }
190
191         if (!vtt->xmit_response) {
192                 PRINT_ERROR("Target driver %s doesn't have a "
193                         "xmit_response() method.", vtt->name);
194                 res = -EINVAL;
195                 goto out_err;
196         }
197
198         if (vtt->threads_num < 0) {
199                 PRINT_ERROR("Wrong threads_num value %d for "
200                         "target \"%s\"", vtt->threads_num,
201                         vtt->name);
202                 res = -EINVAL;
203                 goto out_err;
204         }
205
206         if (!vtt->no_proc_entry) {
207                 res = scst_build_proc_target_dir_entries(vtt);
208                 if (res < 0)
209                         goto out_err;
210         }
211
212         if (mutex_lock_interruptible(&m) != 0)
213                 goto out_err;
214
215         if (mutex_lock_interruptible(&scst_mutex) != 0)
216                 goto out_m_up;
217         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
218                 if (strcmp(t->name, vtt->name) == 0) {
219                         PRINT_ERROR("Target driver %s already registered",
220                                 vtt->name);
221                         mutex_unlock(&scst_mutex);
222                         goto out_cleanup;
223                 }
224         }
225         mutex_unlock(&scst_mutex);
226
227         TRACE_DBG("%s", "Calling target driver's detect()");
228         res = vtt->detect(vtt);
229         TRACE_DBG("Target driver's detect() returned %d", res);
230         if (res < 0) {
231                 PRINT_ERROR("%s", "The detect() routine failed");
232                 res = -EINVAL;
233                 goto out_cleanup;
234         }
235
236         mutex_lock(&scst_mutex);
237         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
238         mutex_unlock(&scst_mutex);
239
240         res = 0;
241
242         PRINT_INFO("Target template %s registered successfully", vtt->name);
243
244         mutex_unlock(&m);
245
246 out:
247         TRACE_EXIT_RES(res);
248         return res;
249
250 out_cleanup:
251         scst_cleanup_proc_target_dir_entries(vtt);
252
253 out_m_up:
254         mutex_unlock(&m);
255
256 out_err:
257         PRINT_ERROR("Failed to register target template %s", vtt->name);
258         goto out;
259 }
260 EXPORT_SYMBOL(__scst_register_target_template);
261
262 void scst_unregister_target_template(struct scst_tgt_template *vtt)
263 {
264         struct scst_tgt *tgt;
265         struct scst_tgt_template *t;
266         int found = 0;
267
268         TRACE_ENTRY();
269
270         mutex_lock(&scst_mutex);
271
272         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
273                 if (strcmp(t->name, vtt->name) == 0) {
274                         found = 1;
275                         break;
276                 }
277         }
278         if (!found) {
279                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
280                 goto out_up;
281         }
282
283 restart:
284         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
285                 mutex_unlock(&scst_mutex);
286                 scst_unregister(tgt);
287                 mutex_lock(&scst_mutex);
288                 goto restart;
289         }
290         list_del(&vtt->scst_template_list_entry);
291
292         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
293
294 out_up:
295         mutex_unlock(&scst_mutex);
296
297         scst_cleanup_proc_target_dir_entries(vtt);
298
299         TRACE_EXIT();
300         return;
301 }
302 EXPORT_SYMBOL(scst_unregister_target_template);
303
304 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
305         const char *target_name)
306 {
307         struct scst_tgt *tgt;
308         int rc = 0;
309
310         TRACE_ENTRY();
311
312         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
313         if (tgt == NULL) {
314                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of tgt failed");
315                 rc = -ENOMEM;
316                 goto out_err;
317         }
318
319         INIT_LIST_HEAD(&tgt->sess_list);
320         init_waitqueue_head(&tgt->unreg_waitQ);
321         tgt->tgtt = vtt;
322         tgt->sg_tablesize = vtt->sg_tablesize;
323         spin_lock_init(&tgt->tgt_lock);
324         INIT_LIST_HEAD(&tgt->retry_cmd_list);
325         atomic_set(&tgt->finished_cmds, 0);
326         init_timer(&tgt->retry_timer);
327         tgt->retry_timer.data = (unsigned long)tgt;
328         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
329
330         rc = scst_suspend_activity(true);
331         if (rc != 0)
332                 goto out_free_tgt_err;
333
334         if (mutex_lock_interruptible(&scst_mutex) != 0) {
335                 rc = -EINTR;
336                 goto out_resume_free;
337         }
338
339         if (target_name != NULL) {
340                 int len = strlen(target_name) + 1 +
341                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
342
343                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
344                 if (tgt->default_group_name == NULL) {
345                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
346                                 "group name failed");
347                         rc = -ENOMEM;
348                         goto out_unlock_resume;
349                 }
350                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
351                         target_name);
352         }
353
354         rc = scst_build_proc_target_entries(tgt);
355         if (rc < 0)
356                 goto out_free_name;
357         else
358                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
359
360         mutex_unlock(&scst_mutex);
361         scst_resume_activity();
362
363         PRINT_INFO("Target %s (%p) for template %s registered successfully",
364                 target_name, tgt, vtt->name);
365
366 out:
367         TRACE_EXIT();
368         return tgt;
369
370 out_free_name:
371         kfree(tgt->default_group_name);
372
373 out_unlock_resume:
374         mutex_unlock(&scst_mutex);
375
376 out_resume_free:
377         scst_resume_activity();
378
379 out_free_tgt_err:
380         kfree(tgt);
381         tgt = NULL;
382
383 out_err:
384         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
385                 target_name, vtt->name, rc);
386         goto out;
387 }
388 EXPORT_SYMBOL(scst_register);
389
390 static inline int test_sess_list(struct scst_tgt *tgt)
391 {
392         int res;
393         mutex_lock(&scst_mutex);
394         res = list_empty(&tgt->sess_list);
395         mutex_unlock(&scst_mutex);
396         return res;
397 }
398
399 void scst_unregister(struct scst_tgt *tgt)
400 {
401         struct scst_session *sess;
402         struct scst_tgt_template *vtt = tgt->tgtt;
403
404         TRACE_ENTRY();
405
406         TRACE_DBG("%s", "Calling target driver's release()");
407         tgt->tgtt->release(tgt);
408         TRACE_DBG("%s", "Target driver's release() returned");
409
410         mutex_lock(&scst_mutex);
411         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
412                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
413         }
414         mutex_unlock(&scst_mutex);
415
416         TRACE_DBG("%s", "Waiting for sessions shutdown");
417         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
418         TRACE_DBG("%s", "wait_event() returned");
419
420         scst_suspend_activity(false);
421         mutex_lock(&scst_mutex);
422
423         list_del(&tgt->tgt_list_entry);
424
425         scst_cleanup_proc_target_entries(tgt);
426
427         kfree(tgt->default_group_name);
428
429         mutex_unlock(&scst_mutex);
430         scst_resume_activity();
431
432         del_timer_sync(&tgt->retry_timer);
433
434         PRINT_INFO("Target %p for template %s unregistered successfully",
435                 tgt, vtt->name);
436
437         kfree(tgt);
438
439         TRACE_EXIT();
440         return;
441 }
442 EXPORT_SYMBOL(scst_unregister);
443
444 static int scst_susp_wait(bool interruptible)
445 {
446         int res = 0;
447
448         TRACE_ENTRY();
449
450         if (interruptible) {
451                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
452                         (atomic_read(&scst_cmd_count) == 0),
453                         SCST_SUSPENDING_TIMEOUT);
454                 if (res <= 0) {
455                         __scst_resume_activity();
456                         if (res == 0)
457                                 res = -EBUSY;
458                 } else
459                         res = 0;
460         } else
461                 wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
462
463         TRACE_MGMT_DBG("wait_event() returned %d", res);
464
465         TRACE_EXIT_RES(res);
466         return res;
467 }
468
469 int scst_suspend_activity(bool interruptible)
470 {
471         int res = 0;
472
473         TRACE_ENTRY();
474
475         if (interruptible) {
476                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
477                         res = -EINTR;
478                         goto out;
479                 }
480         } else
481                 mutex_lock(&scst_suspend_mutex);
482
483         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
484         suspend_count++;
485         if (suspend_count > 1)
486                 goto out_up;
487
488         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
489         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
490         smp_mb__after_set_bit();
491
492         /*
493          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
494          * information about scst_user behavior.
495          *
496          * ToDo: make the global suspending unneeded (Switch to per-device
497          * reference counting? That would mean to switch off from lockless
498          * implementation of scst_translate_lun().. )
499          */
500
501         if (atomic_read(&scst_cmd_count) != 0) {
502                 PRINT_INFO("Waiting for %d active commands to complete... This "
503                         "might take few minutes for disks or few hours for "
504                         "tapes, if you use long executed commands, like "
505                         "REWIND or FORMAT. In case, if you have a hung user "
506                         "space device (i.e. made using scst_user module) not "
507                         "responding to any commands, if might take virtually "
508                         "forever until the corresponding user space "
509                         "program recovers and starts responding or gets "
510                         "killed.", atomic_read(&scst_cmd_count));
511         }
512
513         res = scst_susp_wait(interruptible);
514         if (res != 0)
515                 goto out_clear;
516
517         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
518         smp_mb__after_clear_bit();
519
520         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
521                 atomic_read(&scst_cmd_count));
522
523         res = scst_susp_wait(interruptible);
524         if (res != 0)
525                 goto out_clear;
526
527         PRINT_INFO("%s", "All active commands completed");
528
529 out_up:
530         mutex_unlock(&scst_suspend_mutex);
531
532 out:
533         TRACE_EXIT_RES(res);
534         return res;
535
536 out_clear:
537         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
538         smp_mb__after_clear_bit();
539         goto out_up;
540 }
541 EXPORT_SYMBOL(scst_suspend_activity);
542
543 static void __scst_resume_activity(void)
544 {
545         struct scst_cmd_lists *l;
546
547         TRACE_ENTRY();
548
549         suspend_count--;
550         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
551         if (suspend_count > 0)
552                 goto out;
553
554         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
555         smp_mb__after_clear_bit();
556
557         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
558                 wake_up_all(&l->cmd_list_waitQ);
559         }
560         wake_up_all(&scst_init_cmd_list_waitQ);
561
562         spin_lock_irq(&scst_mcmd_lock);
563         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
564                 struct scst_mgmt_cmd *m;
565                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
566                                 mgmt_cmd_list_entry);
567                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
568                         "mgmt cmd list", m);
569                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
570         }
571         spin_unlock_irq(&scst_mcmd_lock);
572         wake_up_all(&scst_mgmt_cmd_list_waitQ);
573
574 out:
575         TRACE_EXIT();
576         return;
577 }
578
579 void scst_resume_activity(void)
580 {
581         TRACE_ENTRY();
582
583         mutex_lock(&scst_suspend_mutex);
584         __scst_resume_activity();
585         mutex_unlock(&scst_suspend_mutex);
586
587         TRACE_EXIT();
588         return;
589 }
590 EXPORT_SYMBOL(scst_resume_activity);
591
592 static int scst_register_device(struct scsi_device *scsidp)
593 {
594         int res = 0;
595         struct scst_device *dev;
596         struct scst_dev_type *dt;
597
598         TRACE_ENTRY();
599
600         res = scst_suspend_activity(true);
601         if (res != 0)
602                 goto out_err;
603
604         if (mutex_lock_interruptible(&scst_mutex) != 0) {
605                 res = -EINTR;
606                 goto out_resume;
607         }
608
609         res = scst_alloc_device(GFP_KERNEL, &dev);
610         if (res != 0)
611                 goto out_up;
612
613         dev->type = scsidp->type;
614
615         dev->rq_disk = alloc_disk(1);
616         if (dev->rq_disk == NULL) {
617                 res = -ENOMEM;
618                 goto out_free_dev;
619         }
620         dev->rq_disk->major = SCST_MAJOR;
621
622         dev->scsi_dev = scsidp;
623
624         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
625
626         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
627                 if (dt->type == scsidp->type) {
628                         res = scst_assign_dev_handler(dev, dt);
629                         if (res != 0)
630                                 goto out_free;
631                         break;
632                 }
633         }
634
635 out_up:
636         mutex_unlock(&scst_mutex);
637
638 out_resume:
639         scst_resume_activity();
640
641 out_err:
642         if (res == 0) {
643                 PRINT_INFO("Attached SCSI target mid-level at "
644                     "scsi%d, channel %d, id %d, lun %d, type %d",
645                     scsidp->host->host_no, scsidp->channel, scsidp->id,
646                     scsidp->lun, scsidp->type);
647         } else {
648                 PRINT_ERROR("Failed to attach SCSI target mid-level "
649                     "at scsi%d, channel %d, id %d, lun %d, type %d",
650                     scsidp->host->host_no, scsidp->channel, scsidp->id,
651                     scsidp->lun, scsidp->type);
652         }
653
654         TRACE_EXIT_RES(res);
655         return res;
656
657 out_free:
658         list_del(&dev->dev_list_entry);
659         put_disk(dev->rq_disk);
660
661 out_free_dev:
662         scst_free_device(dev);
663         goto out_up;
664 }
665
666 static void scst_unregister_device(struct scsi_device *scsidp)
667 {
668         struct scst_device *d, *dev = NULL;
669         struct scst_acg_dev *acg_dev, *aa;
670
671         TRACE_ENTRY();
672
673         scst_suspend_activity(false);
674         mutex_lock(&scst_mutex);
675
676         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
677                 if (d->scsi_dev == scsidp) {
678                         dev = d;
679                         TRACE_DBG("Target device %p found", dev);
680                         break;
681                 }
682         }
683         if (dev == NULL) {
684                 PRINT_ERROR("%s", "Target device not found");
685                 goto out_unblock;
686         }
687
688         list_del(&dev->dev_list_entry);
689
690         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
691                                  dev_acg_dev_list_entry)
692         {
693                 scst_acg_remove_dev(acg_dev->acg, dev);
694         }
695
696         scst_assign_dev_handler(dev, &scst_null_devtype);
697
698         put_disk(dev->rq_disk);
699         scst_free_device(dev);
700
701         PRINT_INFO("Detached SCSI target mid-level from scsi%d, channel %d, "
702                 "id %d, lun %d, type %d", scsidp->host->host_no,
703                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
704
705 out_unblock:
706         mutex_unlock(&scst_mutex);
707         scst_resume_activity();
708
709         TRACE_EXIT();
710         return;
711 }
712
713 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
714 {
715         int res = 0;
716
717         if (dev_handler->parse == NULL) {
718                 PRINT_ERROR("scst dev_type driver %s doesn't have a "
719                         "parse() method.", dev_handler->name);
720                 res = -EINVAL;
721                 goto out;
722         }
723
724         if (dev_handler->exec == NULL) {
725 #ifdef ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
726                 dev_handler->exec_atomic = 1;
727 #else
728                 dev_handler->exec_atomic = 0;
729 #endif
730         }
731
732         if (dev_handler->dev_done == NULL)
733                 dev_handler->dev_done_atomic = 1;
734
735 out:
736         TRACE_EXIT_RES(res);
737         return res;
738 }
739
740 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
741         const char *dev_name)
742 {
743         int res, rc;
744         struct scst_device *dev = NULL;
745
746         TRACE_ENTRY();
747
748         if (dev_handler == NULL) {
749                 PRINT_ERROR("%s: valid device handler must be supplied",
750                             __func__);
751                 res = -EINVAL;
752                 goto out;
753         }
754
755         if (dev_name == NULL) {
756                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
757                 res = -EINVAL;
758                 goto out;
759         }
760
761         res = scst_dev_handler_check(dev_handler);
762         if (res != 0)
763                 goto out;
764
765         res = scst_suspend_activity(true);
766         if (res != 0)
767                 goto out;
768
769         if (mutex_lock_interruptible(&scst_mutex) != 0) {
770                 res = -EINTR;
771                 goto out_resume;
772         }
773
774         res = scst_alloc_device(GFP_KERNEL, &dev);
775         if (res != 0)
776                 goto out_up;
777
778         dev->type = dev_handler->type;
779         dev->scsi_dev = NULL;
780         dev->virt_name = dev_name;
781         dev->virt_id = scst_virt_dev_last_id++;
782
783         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
784
785         res = dev->virt_id;
786
787         rc = scst_assign_dev_handler(dev, dev_handler);
788         if (rc != 0) {
789                 res = rc;
790                 goto out_free_del;
791         }
792
793 out_up:
794         mutex_unlock(&scst_mutex);
795
796 out_resume:
797         scst_resume_activity();
798
799 out:
800         if (res > 0) {
801                 PRINT_INFO("Attached SCSI target mid-level to virtual "
802                     "device %s (id %d)", dev_name, dev->virt_id);
803         } else {
804                 PRINT_INFO("Failed to attach SCSI target mid-level to "
805                     "virtual device %s", dev_name);
806         }
807
808         TRACE_EXIT_RES(res);
809         return res;
810
811 out_free_del:
812         list_del(&dev->dev_list_entry);
813         scst_free_device(dev);
814         goto out_up;
815 }
816 EXPORT_SYMBOL(scst_register_virtual_device);
817
818 void scst_unregister_virtual_device(int id)
819 {
820         struct scst_device *d, *dev = NULL;
821         struct scst_acg_dev *acg_dev, *aa;
822
823         TRACE_ENTRY();
824
825         scst_suspend_activity(false);
826         mutex_lock(&scst_mutex);
827
828         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
829                 if (d->virt_id == id) {
830                         dev = d;
831                         TRACE_DBG("Target device %p (id %d) found", dev, id);
832                         break;
833                 }
834         }
835         if (dev == NULL) {
836                 PRINT_ERROR("Target virtual device (id %d) not found", id);
837                 goto out_unblock;
838         }
839
840         list_del(&dev->dev_list_entry);
841
842         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
843                                  dev_acg_dev_list_entry)
844         {
845                 scst_acg_remove_dev(acg_dev->acg, dev);
846         }
847
848         scst_assign_dev_handler(dev, &scst_null_devtype);
849
850         PRINT_INFO("Detached SCSI target mid-level from virtual device %s "
851                 "(id %d)", dev->virt_name, dev->virt_id);
852
853         scst_free_device(dev);
854
855 out_unblock:
856         mutex_unlock(&scst_mutex);
857         scst_resume_activity();
858
859         TRACE_EXIT();
860         return;
861 }
862 EXPORT_SYMBOL(scst_unregister_virtual_device);
863
864 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
865         const char *version)
866 {
867         struct scst_dev_type *dt;
868         struct scst_device *dev;
869         int res;
870         int exist;
871
872         TRACE_ENTRY();
873
874         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
875                 PRINT_ERROR("Incorrect version of dev handler %s",
876                         dev_type->name);
877                 res = -EINVAL;
878                 goto out_error;
879         }
880
881         res = scst_dev_handler_check(dev_type);
882         if (res != 0)
883                 goto out_error;
884
885 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
886         if (dev_type->exec == NULL) {
887                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
888                         "supported. Consider applying on your kernel patch "
889                         "scst_exec_req_fifo-<kernel-version>.patch or define "
890                         "STRICT_SERIALIZING", dev_type->name);
891                 res = -EINVAL;
892                 goto out;
893         }
894 #endif
895
896         res = scst_suspend_activity(true);
897         if (res != 0)
898                 goto out_error;
899
900         if (mutex_lock_interruptible(&scst_mutex) != 0) {
901                 res = -EINTR;
902                 goto out_err_res;
903         }
904
905         exist = 0;
906         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
907                 if (strcmp(dt->name, dev_type->name) == 0) {
908                         PRINT_ERROR("Device type handler \"%s\" already "
909                                 "exist", dt->name);
910                         exist = 1;
911                         break;
912                 }
913         }
914         if (exist)
915                 goto out_up;
916
917         res = scst_build_proc_dev_handler_dir_entries(dev_type);
918         if (res < 0)
919                 goto out_up;
920
921         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
922
923         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
924                 if ((dev->scsi_dev == NULL) || (dev->handler != &scst_null_devtype))
925                         continue;
926                 if (dev->scsi_dev->type == dev_type->type)
927                         scst_assign_dev_handler(dev, dev_type);
928         }
929
930         mutex_unlock(&scst_mutex);
931         scst_resume_activity();
932
933         if (res == 0) {
934                 PRINT_INFO("Device handler \"%s\" for type %d registered "
935                         "successfully", dev_type->name, dev_type->type);
936         }
937
938 out:
939         TRACE_EXIT_RES(res);
940         return res;
941
942 out_up:
943         mutex_unlock(&scst_mutex);
944
945 out_err_res:
946         scst_resume_activity();
947
948 out_error:
949         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
950                 dev_type->name, dev_type->type);
951         goto out;
952 }
953 EXPORT_SYMBOL(__scst_register_dev_driver);
954
955 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
956 {
957         struct scst_device *dev;
958         struct scst_dev_type *dt;
959         int found = 0;
960
961         TRACE_ENTRY();
962
963         scst_suspend_activity(false);
964         mutex_lock(&scst_mutex);
965
966         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
967                 if (strcmp(dt->name, dev_type->name) == 0) {
968                         found = 1;
969                         break;
970                 }
971         }
972         if (!found) {
973                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
974                         dev_type->name);
975                 goto out_up;
976         }
977
978         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
979                 if (dev->handler == dev_type) {
980                         scst_assign_dev_handler(dev, &scst_null_devtype);
981                         TRACE_DBG("Dev handler removed from device %p", dev);
982                 }
983         }
984
985         list_del(&dev_type->dev_type_list_entry);
986
987         mutex_unlock(&scst_mutex);
988         scst_resume_activity();
989
990         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
991
992         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
993                    dev_type->name, dev_type->type);
994
995 out:
996         TRACE_EXIT();
997         return;
998
999 out_up:
1000         mutex_unlock(&scst_mutex);
1001         scst_resume_activity();
1002         goto out;
1003 }
1004 EXPORT_SYMBOL(scst_unregister_dev_driver);
1005
1006 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1007         const char *version)
1008 {
1009         int res;
1010
1011         TRACE_ENTRY();
1012
1013         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1014                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1015                         dev_type->name);
1016                 res = -EINVAL;
1017                 goto out_err;
1018         }
1019
1020         res = scst_dev_handler_check(dev_type);
1021         if (res != 0)
1022                 goto out_err;
1023
1024         if (!dev_type->no_proc) {
1025                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1026                 if (res < 0)
1027                         goto out_err;
1028         }
1029
1030         if (dev_type->type != -1) {
1031                 PRINT_INFO("Virtual device handler %s for type %d "
1032                         "registered successfully", dev_type->name,
1033                         dev_type->type);
1034         } else {
1035                 PRINT_INFO("Virtual device handler \"%s\" registered "
1036                         "successfully", dev_type->name);
1037         }
1038
1039 out:
1040         TRACE_EXIT_RES(res);
1041         return res;
1042
1043 out_err:
1044         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1045                 dev_type->name);
1046         goto out;
1047 }
1048 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1049
1050 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1051 {
1052         TRACE_ENTRY();
1053
1054         if (!dev_type->no_proc)
1055                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1056
1057         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1058
1059         TRACE_EXIT();
1060         return;
1061 }
1062 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1063
1064 /* Called under scst_mutex */
1065 int scst_add_dev_threads(struct scst_device *dev, int num)
1066 {
1067         int i, res = 0;
1068         int n = 0;
1069         struct scst_cmd_thread_t *thr;
1070         char nm[12];
1071
1072         TRACE_ENTRY();
1073
1074         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1075                 n++;
1076         }
1077
1078         for (i = 0; i < num; i++) {
1079                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1080                 if (!thr) {
1081                         res = -ENOMEM;
1082                         PRINT_ERROR("Failed to allocate thr %d", res);
1083                         goto out;
1084                 }
1085                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1086                 nm[ARRAY_SIZE(nm)-1] = '\0';
1087                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1088                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1089                 if (IS_ERR(thr->cmd_thread)) {
1090                         res = PTR_ERR(thr->cmd_thread);
1091                         PRINT_ERROR("kthread_create() failed: %d", res);
1092                         kfree(thr);
1093                         goto out;
1094                 }
1095                 list_add(&thr->thread_list_entry, &dev->threads_list);
1096         }
1097
1098 out:
1099         TRACE_EXIT_RES(res);
1100         return res;
1101 }
1102
1103 /* Called under scst_mutex and suspended activity */
1104 static int scst_create_dev_threads(struct scst_device *dev)
1105 {
1106         int res = 0;
1107         int threads_num;
1108
1109         TRACE_ENTRY();
1110
1111         if (dev->handler->threads_num <= 0)
1112                 goto out;
1113
1114         threads_num = dev->handler->threads_num;
1115
1116         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1117         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1118         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1119
1120         res = scst_add_dev_threads(dev, threads_num);
1121         if (res != 0)
1122                 goto out;
1123
1124         mutex_lock(&scst_suspend_mutex);
1125         list_add_tail(&dev->cmd_lists.lists_list_entry,
1126                 &scst_cmd_lists_list);
1127         mutex_unlock(&scst_suspend_mutex);
1128
1129         dev->p_cmd_lists = &dev->cmd_lists;
1130
1131 out:
1132         TRACE_EXIT_RES(res);
1133         return res;
1134 }
1135
1136 /* Called under scst_mutex */
1137 void scst_del_dev_threads(struct scst_device *dev, int num)
1138 {
1139         struct scst_cmd_thread_t *ct, *tmp;
1140         int i = 0;
1141
1142         TRACE_ENTRY();
1143
1144         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
1145                                 thread_list_entry) {
1146                 int rc = kthread_stop(ct->cmd_thread);
1147                 if (rc < 0)
1148                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1149                 list_del(&ct->thread_list_entry);
1150                 kfree(ct);
1151                 if ((num > 0) && (++i >= num))
1152                         break;
1153         }
1154
1155         TRACE_EXIT();
1156         return;
1157 }
1158
1159 /* Called under scst_mutex and suspended activity */
1160 static void scst_stop_dev_threads(struct scst_device *dev)
1161 {
1162         TRACE_ENTRY();
1163
1164         if (list_empty(&dev->threads_list))
1165                 goto out;
1166
1167         scst_del_dev_threads(dev, -1);
1168
1169         if (dev->p_cmd_lists == &dev->cmd_lists) {
1170                 mutex_lock(&scst_suspend_mutex);
1171                 list_del(&dev->cmd_lists.lists_list_entry);
1172                 mutex_unlock(&scst_suspend_mutex);
1173         }
1174
1175 out:
1176         TRACE_EXIT();
1177         return;
1178 }
1179
1180 /* The activity supposed to be suspended and scst_mutex held */
1181 int scst_assign_dev_handler(struct scst_device *dev,
1182         struct scst_dev_type *handler)
1183 {
1184         int res = 0;
1185         struct scst_tgt_dev *tgt_dev;
1186         LIST_HEAD(attached_tgt_devs);
1187
1188         TRACE_ENTRY();
1189
1190         sBUG_ON(handler == NULL);
1191
1192         if (dev->handler == handler)
1193                 goto out;
1194
1195         if (dev->handler && dev->handler->detach_tgt) {
1196                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1197                                 dev_tgt_dev_list_entry) {
1198                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1199                                 tgt_dev);
1200                         dev->handler->detach_tgt(tgt_dev);
1201                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1202                 }
1203         }
1204
1205         if (dev->handler && dev->handler->detach) {
1206                 TRACE_DBG("%s", "Calling dev handler's detach()");
1207                 dev->handler->detach(dev);
1208                 TRACE_DBG("%s", "Old handler's detach() returned");
1209         }
1210
1211         scst_stop_dev_threads(dev);
1212
1213         dev->handler = handler;
1214
1215         if (handler) {
1216                 res = scst_create_dev_threads(dev);
1217                 if (res != 0)
1218                         goto out_null;
1219         }
1220
1221         if (handler && handler->attach) {
1222                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1223                 res = handler->attach(dev);
1224                 TRACE_DBG("New dev handler's attach() returned %d", res);
1225                 if (res != 0) {
1226                         PRINT_ERROR("New device handler's %s attach() "
1227                                 "failed: %d", handler->name, res);
1228                 }
1229                 goto out_thr_null;
1230         }
1231
1232         if (handler && handler->attach_tgt) {
1233                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1234                                 dev_tgt_dev_list_entry) {
1235                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1236                                 tgt_dev);
1237                         res = handler->attach_tgt(tgt_dev);
1238                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1239                         if (res != 0) {
1240                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1241                                     "failed: %d", handler->name, res);
1242                                 goto out_err_detach_tgt;
1243                         }
1244                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1245                                 &attached_tgt_devs);
1246                 }
1247         }
1248
1249 out_thr_null:
1250         if (res != 0)
1251                 scst_stop_dev_threads(dev);
1252
1253 out_null:
1254         if (res != 0)
1255                 dev->handler = &scst_null_devtype;
1256
1257 out:
1258         TRACE_EXIT_RES(res);
1259         return res;
1260
1261 out_err_detach_tgt:
1262         if (handler && handler->detach_tgt) {
1263                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1264                                  extra_tgt_dev_list_entry)
1265                 {
1266                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1267                                 tgt_dev);
1268                         handler->detach_tgt(tgt_dev);
1269                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1270                 }
1271         }
1272         if (handler && handler->detach) {
1273                 TRACE_DBG("%s", "Calling handler's detach()");
1274                 handler->detach(dev);
1275                 TRACE_DBG("%s", "Handler's detach() returned");
1276         }
1277         goto out_null;
1278 }
1279
1280 int scst_cmd_threads_count(void)
1281 {
1282         int i;
1283
1284         /* Just to lower the race window, when user can get just changed value */
1285         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1286         i = scst_threads_info.nr_cmd_threads;
1287         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1288         return i;
1289 }
1290
1291 static void scst_threads_info_init(void)
1292 {
1293         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1294         mutex_init(&scst_threads_info.cmd_threads_mutex);
1295         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1296 }
1297
1298 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1299 void __scst_del_cmd_threads(int num)
1300 {
1301         struct scst_cmd_thread_t *ct, *tmp;
1302         int i;
1303
1304         TRACE_ENTRY();
1305
1306         i = scst_threads_info.nr_cmd_threads;
1307         if (num <= 0 || num > i) {
1308                 PRINT_ERROR("can not del %d cmd threads from %d", num, i);
1309                 return;
1310         }
1311
1312         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1313                                 thread_list_entry) {
1314                 int res;
1315
1316                 res = kthread_stop(ct->cmd_thread);
1317                 if (res < 0)
1318                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1319                 list_del(&ct->thread_list_entry);
1320                 kfree(ct);
1321                 scst_threads_info.nr_cmd_threads--;
1322                 --num;
1323                 if (num == 0)
1324                         break;
1325         }
1326
1327         TRACE_EXIT();
1328         return;
1329 }
1330
1331 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1332 int __scst_add_cmd_threads(int num)
1333 {
1334         int res = 0, i;
1335         static int scst_thread_num;
1336
1337         TRACE_ENTRY();
1338
1339         for (i = 0; i < num; i++) {
1340                 struct scst_cmd_thread_t *thr;
1341
1342                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1343                 if (!thr) {
1344                         res = -ENOMEM;
1345                         PRINT_ERROR("fail to allocate thr %d", res);
1346                         goto out_error;
1347                 }
1348                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1349                         &scst_main_cmd_lists, "scsi_tgt%d",
1350                         scst_thread_num++);
1351                 if (IS_ERR(thr->cmd_thread)) {
1352                         res = PTR_ERR(thr->cmd_thread);
1353                         PRINT_ERROR("kthread_create() failed: %d", res);
1354                         kfree(thr);
1355                         goto out_error;
1356                 }
1357                 list_add(&thr->thread_list_entry,
1358                         &scst_threads_info.cmd_threads_list);
1359                 scst_threads_info.nr_cmd_threads++;
1360         }
1361         res = 0;
1362
1363 out:
1364         TRACE_EXIT_RES(res);
1365         return res;
1366
1367 out_error:
1368         if (i > 0)
1369                 __scst_del_cmd_threads(i - 1);
1370         goto out;
1371 }
1372
1373 int scst_add_cmd_threads(int num)
1374 {
1375         int res;
1376
1377         TRACE_ENTRY();
1378
1379         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1380         res = __scst_add_cmd_threads(num);
1381         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1382
1383         TRACE_EXIT_RES(res);
1384         return res;
1385 }
1386 EXPORT_SYMBOL(scst_add_cmd_threads);
1387
1388 void scst_del_cmd_threads(int num)
1389 {
1390         TRACE_ENTRY();
1391
1392         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1393         __scst_del_cmd_threads(num);
1394         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1395
1396         TRACE_EXIT();
1397         return;
1398 }
1399 EXPORT_SYMBOL(scst_del_cmd_threads);
1400
1401 static void scst_stop_all_threads(void)
1402 {
1403         TRACE_ENTRY();
1404
1405         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1406         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1407         if (scst_threads_info.mgmt_cmd_thread)
1408                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1409         if (scst_threads_info.mgmt_thread)
1410                 kthread_stop(scst_threads_info.mgmt_thread);
1411         if (scst_threads_info.init_cmd_thread)
1412                 kthread_stop(scst_threads_info.init_cmd_thread);
1413         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1414
1415         TRACE_EXIT();
1416         return;
1417 }
1418
1419 static int scst_start_all_threads(int num)
1420 {
1421         int res;
1422
1423         TRACE_ENTRY();
1424
1425         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1426         res = __scst_add_cmd_threads(num);
1427         if (res < 0)
1428                 goto out;
1429
1430         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1431                 NULL, "scsi_tgt_init");
1432         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1433                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1434                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1435                 scst_threads_info.init_cmd_thread = NULL;
1436                 goto out;
1437         }
1438
1439         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1440                 NULL, "scsi_tgt_mc");
1441         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1442                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1443                 PRINT_ERROR("kthread_create() for mcmd failed: %d", res);
1444                 scst_threads_info.mgmt_cmd_thread = NULL;
1445                 goto out;
1446         }
1447
1448         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1449                 NULL, "scsi_tgt_mgmt");
1450         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1451                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1452                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1453                 scst_threads_info.mgmt_thread = NULL;
1454                 goto out;
1455         }
1456
1457 out:
1458         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1459         TRACE_EXIT_RES(res);
1460         return res;
1461 }
1462
1463 void scst_get(void)
1464 {
1465         __scst_get(0);
1466 }
1467 EXPORT_SYMBOL(scst_get);
1468
1469 void scst_put(void)
1470 {
1471         __scst_put();
1472 }
1473 EXPORT_SYMBOL(scst_put);
1474
1475 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 15)
1476 static int scst_add(struct class_device *cdev)
1477 #else
1478 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1479 #endif
1480 {
1481         struct scsi_device *scsidp;
1482         int res = 0;
1483
1484         TRACE_ENTRY();
1485
1486         scsidp = to_scsi_device(cdev->dev);
1487         res = scst_register_device(scsidp);
1488
1489         TRACE_EXIT();
1490         return res;
1491 }
1492
1493 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 15)
1494 static void scst_remove(struct class_device *cdev)
1495 #else
1496 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1497 #endif
1498 {
1499         struct scsi_device *scsidp;
1500
1501         TRACE_ENTRY();
1502
1503         scsidp = to_scsi_device(cdev->dev);
1504         scst_unregister_device(scsidp);
1505
1506         TRACE_EXIT();
1507         return;
1508 }
1509
1510 static struct class_interface scst_interface = {
1511         .add = scst_add,
1512         .remove = scst_remove,
1513 };
1514
1515 static void __init scst_print_config(void)
1516 {
1517         char buf[128];
1518         int i, j;
1519
1520         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1521         j = i;
1522
1523 #ifdef STRICT_SERIALIZING
1524         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1525 #endif
1526
1527 #ifdef EXTRACHECKS
1528         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1529                 (j == i) ? "" : ", ");
1530 #endif
1531
1532 #ifdef TRACING
1533         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1534                 (j == i) ? "" : ", ");
1535 #endif
1536
1537 #ifdef DEBUG
1538         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1539                 (j == i) ? "" : ", ");
1540 #endif
1541
1542 #ifdef DEBUG_TM
1543         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1544                 (j == i) ? "" : ", ");
1545 #endif
1546
1547 #ifdef DEBUG_RETRY
1548         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1549                 (j == i) ? "" : ", ");
1550 #endif
1551
1552 #ifdef DEBUG_OOM
1553         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1554                 (j == i) ? "" : ", ");
1555 #endif
1556
1557 #ifdef DEBUG_SN
1558         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1559                 (j == i) ? "" : ", ");
1560 #endif
1561
1562 #ifdef USE_EXPECTED_VALUES
1563         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1564                 (j == i) ? "" : ", ");
1565 #endif
1566
1567 #ifdef ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1568         i += snprintf(&buf[i], sizeof(buf) - i, "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1569                 (j == i) ? "" : ", ");
1570 #endif
1571
1572 #ifdef SCST_STRICT_SECURITY
1573         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1574                 (j == i) ? "" : ", ");
1575 #endif
1576
1577 #ifdef SCST_HIGHMEM
1578         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_HIGHMEM",
1579                 (j == i) ? "" : ", ");
1580 #endif
1581
1582         if (j != i)
1583                 PRINT_INFO("%s", buf);
1584 }
1585
1586 static int __init init_scst(void)
1587 {
1588         int res = 0, i;
1589         int scst_num_cpus;
1590
1591         TRACE_ENTRY();
1592
1593 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1594         {
1595                 struct scsi_request *req;
1596                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1597                         sizeof(req->sr_sense_buffer));
1598         }
1599 #else
1600         {
1601                 struct scsi_sense_hdr *shdr;
1602                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1603         }
1604 #endif
1605         {
1606                 struct scst_tgt_dev *t;
1607                 struct scst_cmd *c;
1608                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1609                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1610         }
1611
1612         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1613         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1614         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1615         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1616
1617         mutex_init(&scst_mutex);
1618         INIT_LIST_HEAD(&scst_template_list);
1619         INIT_LIST_HEAD(&scst_dev_list);
1620         INIT_LIST_HEAD(&scst_dev_type_list);
1621         spin_lock_init(&scst_main_lock);
1622         INIT_LIST_HEAD(&scst_acg_list);
1623         spin_lock_init(&scst_init_lock);
1624         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1625         INIT_LIST_HEAD(&scst_init_cmd_list);
1626 #if defined(DEBUG) || defined(TRACING)
1627         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1628 #endif
1629         atomic_set(&scst_cmd_count, 0);
1630         spin_lock_init(&scst_mcmd_lock);
1631         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1632         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1633         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1634         init_waitqueue_head(&scst_mgmt_waitQ);
1635         spin_lock_init(&scst_mgmt_lock);
1636         INIT_LIST_HEAD(&scst_sess_init_list);
1637         INIT_LIST_HEAD(&scst_sess_shut_list);
1638         init_waitqueue_head(&scst_dev_cmd_waitQ);
1639         mutex_init(&scst_suspend_mutex);
1640         INIT_LIST_HEAD(&scst_cmd_lists_list);
1641         scst_virt_dev_last_id = 1;
1642         spin_lock_init(&scst_temp_UA_lock);
1643
1644         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1645         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1646         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1647         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1648                 &scst_cmd_lists_list);
1649
1650         scst_num_cpus = num_online_cpus();
1651
1652         /* ToDo: register_cpu_notifier() */
1653
1654         if (scst_threads == 0)
1655                 scst_threads = scst_num_cpus;
1656
1657         if (scst_threads < 1) {
1658                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1659                 scst_threads = scst_num_cpus;
1660         }
1661
1662         scst_threads_info_init();
1663
1664 #define INIT_CACHEP(p, s, o) do {                                       \
1665                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1666                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1667                           sizeof(struct s));                            \
1668                 if (p == NULL) {                                        \
1669                         res = -ENOMEM;                                  \
1670                         goto o;                                         \
1671                 }                                                       \
1672         } while (0)
1673
1674         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out);
1675         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1676                         out_destroy_mgmt_cache);
1677         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1678                         out_destroy_mgmt_stub_cache);
1679         {
1680                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1681                 INIT_CACHEP(scst_sense_cachep, scst_sense, out_destroy_ua_cache);
1682         }
1683         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_sense_cache);
1684         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1685         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1686         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1687
1688         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1689                 mempool_free_slab, scst_mgmt_cachep);
1690         if (scst_mgmt_mempool == NULL) {
1691                 res = -ENOMEM;
1692                 goto out_destroy_acg_cache;
1693         }
1694
1695         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1696                 mempool_free_slab, scst_mgmt_stub_cachep);
1697         if (scst_mgmt_stub_mempool == NULL) {
1698                 res = -ENOMEM;
1699                 goto out_destroy_mgmt_mempool;
1700         }
1701
1702         scst_ua_mempool = mempool_create(64, mempool_alloc_slab,
1703                 mempool_free_slab, scst_ua_cachep);
1704         if (scst_ua_mempool == NULL) {
1705                 res = -ENOMEM;
1706                 goto out_destroy_mgmt_stub_mempool;
1707         }
1708
1709         /* Loosing sense may have fatal consequences, so let's have a big pool */
1710         scst_sense_mempool = mempool_create(128, mempool_alloc_slab,
1711                 mempool_free_slab, scst_sense_cachep);
1712         if (scst_sense_mempool == NULL) {
1713                 res = -ENOMEM;
1714                 goto out_destroy_ua_mempool;
1715         }
1716
1717         if (scst_max_cmd_mem == 0) {
1718                 struct sysinfo si;
1719                 si_meminfo(&si);
1720 #if BITS_PER_LONG == 32
1721                 scst_max_cmd_mem = min(
1722                         (((uint64_t)si.totalram << PAGE_SHIFT) >> 20) >> 2,
1723                         (uint64_t)1 << 30);
1724 #else
1725                 scst_max_cmd_mem = ((si.totalram << PAGE_SHIFT) >> 20) >> 2;
1726 #endif
1727         }
1728
1729         if (scst_max_dev_cmd_mem != 0) {
1730                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
1731                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
1732                                 "scst_max_cmd_mem (%d)",
1733                                 scst_max_dev_cmd_mem,
1734                                 scst_max_cmd_mem);
1735                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
1736                 }
1737         } else
1738                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
1739
1740         res = scst_sgv_pools_init(
1741                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
1742         if (res != 0)
1743                 goto out_destroy_sense_mempool;
1744
1745         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1746         if (scst_default_acg == NULL) {
1747                 res = -ENOMEM;
1748                 goto out_destroy_sgv_pool;
1749         }
1750
1751         res = scsi_register_interface(&scst_interface);
1752         if (res != 0)
1753                 goto out_free_acg;
1754
1755         scst_scsi_op_list_init();
1756
1757         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1758                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1759                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1760                 tasklet_init(&scst_tasklets[i].tasklet, (void *)scst_cmd_tasklet,
1761                         (unsigned long)&scst_tasklets[i]);
1762         }
1763
1764         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1765                 scst_threads);
1766
1767         res = scst_start_all_threads(scst_threads);
1768         if (res < 0)
1769                 goto out_thread_free;
1770
1771         res = scst_proc_init_module();
1772         if (res != 0)
1773                 goto out_thread_free;
1774
1775
1776         PRINT_INFO("SCST version %s loaded successfully (max mem for "
1777                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
1778                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
1779
1780         scst_print_config();
1781
1782 out:
1783         TRACE_EXIT_RES(res);
1784         return res;
1785
1786 out_thread_free:
1787         scst_stop_all_threads();
1788
1789         scsi_unregister_interface(&scst_interface);
1790
1791 out_free_acg:
1792         scst_destroy_acg(scst_default_acg);
1793
1794 out_destroy_sgv_pool:
1795         scst_sgv_pools_deinit();
1796
1797 out_destroy_sense_mempool:
1798         mempool_destroy(scst_sense_mempool);
1799
1800 out_destroy_ua_mempool:
1801         mempool_destroy(scst_ua_mempool);
1802
1803 out_destroy_mgmt_stub_mempool:
1804         mempool_destroy(scst_mgmt_stub_mempool);
1805
1806 out_destroy_mgmt_mempool:
1807         mempool_destroy(scst_mgmt_mempool);
1808
1809 out_destroy_acg_cache:
1810         kmem_cache_destroy(scst_acgd_cachep);
1811
1812 out_destroy_tgt_cache:
1813         kmem_cache_destroy(scst_tgtd_cachep);
1814
1815 out_destroy_sess_cache:
1816         kmem_cache_destroy(scst_sess_cachep);
1817
1818 out_destroy_cmd_cache:
1819         kmem_cache_destroy(scst_cmd_cachep);
1820
1821 out_destroy_sense_cache:
1822         kmem_cache_destroy(scst_sense_cachep);
1823
1824 out_destroy_ua_cache:
1825         kmem_cache_destroy(scst_ua_cachep);
1826
1827 out_destroy_mgmt_stub_cache:
1828         kmem_cache_destroy(scst_mgmt_stub_cachep);
1829
1830 out_destroy_mgmt_cache:
1831         kmem_cache_destroy(scst_mgmt_cachep);
1832         goto out;
1833 }
1834
1835 static void __exit exit_scst(void)
1836 {
1837 #ifdef CONFIG_LOCKDEP
1838         static /* To hide the lockdep's warning about non-static key */
1839 #endif
1840         DECLARE_MUTEX_LOCKED(shm);
1841
1842         TRACE_ENTRY();
1843
1844         /* ToDo: unregister_cpu_notifier() */
1845
1846         scst_proc_cleanup_module();
1847
1848         scst_stop_all_threads();
1849
1850         scsi_unregister_interface(&scst_interface);
1851         scst_destroy_acg(scst_default_acg);
1852
1853         scst_sgv_pools_deinit();
1854
1855 #define DEINIT_CACHEP(p) do {           \
1856                 kmem_cache_destroy(p);  \
1857                 p = NULL;               \
1858         } while (0)
1859
1860         mempool_destroy(scst_mgmt_mempool);
1861         mempool_destroy(scst_mgmt_stub_mempool);
1862         mempool_destroy(scst_ua_mempool);
1863         mempool_destroy(scst_sense_mempool);
1864
1865         DEINIT_CACHEP(scst_mgmt_cachep);
1866         DEINIT_CACHEP(scst_mgmt_stub_cachep);
1867         DEINIT_CACHEP(scst_ua_cachep);
1868         DEINIT_CACHEP(scst_sense_cachep);
1869         DEINIT_CACHEP(scst_cmd_cachep);
1870         DEINIT_CACHEP(scst_sess_cachep);
1871         DEINIT_CACHEP(scst_tgtd_cachep);
1872         DEINIT_CACHEP(scst_acgd_cachep);
1873
1874         PRINT_INFO("%s", "SCST unloaded");
1875
1876         TRACE_EXIT();
1877         return;
1878 }
1879
1880
1881 module_init(init_scst);
1882 module_exit(exit_scst);
1883
1884 MODULE_AUTHOR("Vladislav Bolkhovitin");
1885 MODULE_LICENSE("GPL");
1886 MODULE_DESCRIPTION("SCSI target core");
1887 MODULE_VERSION(SCST_VERSION_STRING);