7da5b0680692ba3b77a4a128091ad0f9f3510488
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/module.h>
20
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/list.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/sched.h>
28 #include <linux/unistd.h>
29 #include <linux/string.h>
30 #include <linux/kthread.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34 #include "scst_mem.h"
35
36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
37 #warning "HIGHMEM kernel configurations are fully supported, but not \
38         recommended for performance reasons. Consider change VMSPLIT \
39         option or use 64-bit configuration instead. See README file for \
40         details."
41 #endif
42
43 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(CONFIG_SCST_STRICT_SERIALIZING)
44 #warning "Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
45         your kernel and CONFIG_SCST_STRICT_SERIALIZING isn't defined. Pass-through dev \
46         handlers will not be supported."
47 #endif
48
49 /**
50  ** SCST global variables. They are all uninitialized to have their layout in
51  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
52  ** variable separately from nonzero-initialized ones.
53  **/
54
55 /*
56  * All targets, devices and dev_types management is done under this mutex.
57  *
58  * It must NOT be used in any works (schedule_work(), etc.), because
59  * otherwise a deadlock (double lock, actually) is possible, e.g., with
60  * scst_user detach_tgt(), which is called under scst_mutex and calls
61  * flush_scheduled_work().
62  */
63 struct mutex scst_mutex;
64
65 struct list_head scst_template_list;
66 struct list_head scst_dev_list;
67 struct list_head scst_dev_type_list;
68
69 spinlock_t scst_main_lock;
70
71 struct kmem_cache *scst_mgmt_cachep;
72 mempool_t *scst_mgmt_mempool;
73 struct kmem_cache *scst_mgmt_stub_cachep;
74 mempool_t *scst_mgmt_stub_mempool;
75 struct kmem_cache *scst_ua_cachep;
76 mempool_t *scst_ua_mempool;
77 struct kmem_cache *scst_sense_cachep;
78 mempool_t *scst_sense_mempool;
79 struct kmem_cache *scst_tgtd_cachep;
80 struct kmem_cache *scst_sess_cachep;
81 struct kmem_cache *scst_acgd_cachep;
82
83 struct list_head scst_acg_list;
84 struct scst_acg *scst_default_acg;
85
86 spinlock_t scst_init_lock;
87 wait_queue_head_t scst_init_cmd_list_waitQ;
88 struct list_head scst_init_cmd_list;
89 unsigned int scst_init_poll_cnt;
90
91 struct kmem_cache *scst_cmd_cachep;
92
93 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
94 unsigned long scst_trace_flag;
95 #endif
96
97 unsigned long scst_flags;
98 atomic_t scst_cmd_count;
99
100 struct scst_cmd_lists scst_main_cmd_lists;
101
102 struct scst_tasklet scst_tasklets[NR_CPUS];
103
104 spinlock_t scst_mcmd_lock;
105 struct list_head scst_active_mgmt_cmd_list;
106 struct list_head scst_delayed_mgmt_cmd_list;
107 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
108
109 wait_queue_head_t scst_mgmt_waitQ;
110 spinlock_t scst_mgmt_lock;
111 struct list_head scst_sess_init_list;
112 struct list_head scst_sess_shut_list;
113
114 wait_queue_head_t scst_dev_cmd_waitQ;
115
116 struct mutex scst_suspend_mutex;
117 struct list_head scst_cmd_lists_list;
118
119 static int scst_threads;
120 struct scst_threads_info_t scst_threads_info;
121
122 static int suspend_count;
123
124 static int scst_virt_dev_last_id; /* protected by scst_mutex */
125
126 /*
127  * This buffer and lock are intended to avoid memory allocation, which
128  * could fail in improper places.
129  */
130 spinlock_t scst_temp_UA_lock;
131 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
132
133 unsigned int scst_max_cmd_mem;
134 unsigned int scst_max_dev_cmd_mem;
135
136 module_param_named(scst_threads, scst_threads, int, 0);
137 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
138
139 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, 0);
140 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
141         "all SCSI commands of all devices at any given time in MB");
142
143 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, 0);
144 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
145         "by all SCSI commands of a device at any given time in MB");
146
147 struct scst_dev_type scst_null_devtype = {
148         .name = "none",
149 };
150
151 static void __scst_resume_activity(void);
152
153 int __scst_register_target_template(struct scst_tgt_template *vtt,
154         const char *version)
155 {
156         int res = 0;
157         struct scst_tgt_template *t;
158         static DEFINE_MUTEX(m);
159
160         TRACE_ENTRY();
161
162         INIT_LIST_HEAD(&vtt->tgt_list);
163
164         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
165                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
166                 res = -EINVAL;
167                 goto out_err;
168         }
169
170         if (!vtt->detect) {
171                 PRINT_ERROR("Target driver %s doesn't have a "
172                         "detect() method.", vtt->name);
173                 res = -EINVAL;
174                 goto out_err;
175         }
176
177         if (!vtt->release) {
178                 PRINT_ERROR("Target driver %s doesn't have a "
179                         "release() method.", vtt->name);
180                 res = -EINVAL;
181                 goto out_err;
182         }
183
184         if (!vtt->xmit_response) {
185                 PRINT_ERROR("Target driver %s doesn't have a "
186                         "xmit_response() method.", vtt->name);
187                 res = -EINVAL;
188                 goto out_err;
189         }
190
191         if (vtt->threads_num < 0) {
192                 PRINT_ERROR("Wrong threads_num value %d for "
193                         "target \"%s\"", vtt->threads_num,
194                         vtt->name);
195                 res = -EINVAL;
196                 goto out_err;
197         }
198
199         if (!vtt->no_proc_entry) {
200                 res = scst_build_proc_target_dir_entries(vtt);
201                 if (res < 0)
202                         goto out_err;
203         }
204
205         if (vtt->rdy_to_xfer == NULL)
206                 vtt->rdy_to_xfer_atomic = 1;
207
208         if (mutex_lock_interruptible(&m) != 0)
209                 goto out_err;
210
211         if (mutex_lock_interruptible(&scst_mutex) != 0)
212                 goto out_m_up;
213         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
214                 if (strcmp(t->name, vtt->name) == 0) {
215                         PRINT_ERROR("Target driver %s already registered",
216                                 vtt->name);
217                         mutex_unlock(&scst_mutex);
218                         goto out_cleanup;
219                 }
220         }
221         mutex_unlock(&scst_mutex);
222
223         TRACE_DBG("%s", "Calling target driver's detect()");
224         res = vtt->detect(vtt);
225         TRACE_DBG("Target driver's detect() returned %d", res);
226         if (res < 0) {
227                 PRINT_ERROR("%s", "The detect() routine failed");
228                 res = -EINVAL;
229                 goto out_cleanup;
230         }
231
232         mutex_lock(&scst_mutex);
233         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
234         mutex_unlock(&scst_mutex);
235
236         res = 0;
237
238         PRINT_INFO("Target template %s registered successfully", vtt->name);
239
240         mutex_unlock(&m);
241
242 out:
243         TRACE_EXIT_RES(res);
244         return res;
245
246 out_cleanup:
247         scst_cleanup_proc_target_dir_entries(vtt);
248
249 out_m_up:
250         mutex_unlock(&m);
251
252 out_err:
253         PRINT_ERROR("Failed to register target template %s", vtt->name);
254         goto out;
255 }
256 EXPORT_SYMBOL(__scst_register_target_template);
257
258 void scst_unregister_target_template(struct scst_tgt_template *vtt)
259 {
260         struct scst_tgt *tgt;
261         struct scst_tgt_template *t;
262         int found = 0;
263
264         TRACE_ENTRY();
265
266         mutex_lock(&scst_mutex);
267
268         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
269                 if (strcmp(t->name, vtt->name) == 0) {
270                         found = 1;
271                         break;
272                 }
273         }
274         if (!found) {
275                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
276                 goto out_up;
277         }
278
279 restart:
280         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
281                 mutex_unlock(&scst_mutex);
282                 scst_unregister(tgt);
283                 mutex_lock(&scst_mutex);
284                 goto restart;
285         }
286         list_del(&vtt->scst_template_list_entry);
287
288         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
289
290 out_up:
291         mutex_unlock(&scst_mutex);
292
293         scst_cleanup_proc_target_dir_entries(vtt);
294
295         TRACE_EXIT();
296         return;
297 }
298 EXPORT_SYMBOL(scst_unregister_target_template);
299
300 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
301         const char *target_name)
302 {
303         struct scst_tgt *tgt;
304         int rc = 0;
305
306         TRACE_ENTRY();
307
308         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
309         if (tgt == NULL) {
310                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of tgt failed");
311                 rc = -ENOMEM;
312                 goto out_err;
313         }
314
315         INIT_LIST_HEAD(&tgt->sess_list);
316         init_waitqueue_head(&tgt->unreg_waitQ);
317         tgt->tgtt = vtt;
318         tgt->sg_tablesize = vtt->sg_tablesize;
319         spin_lock_init(&tgt->tgt_lock);
320         INIT_LIST_HEAD(&tgt->retry_cmd_list);
321         atomic_set(&tgt->finished_cmds, 0);
322         init_timer(&tgt->retry_timer);
323         tgt->retry_timer.data = (unsigned long)tgt;
324         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
325
326         rc = scst_suspend_activity(true);
327         if (rc != 0)
328                 goto out_free_tgt_err;
329
330         if (mutex_lock_interruptible(&scst_mutex) != 0) {
331                 rc = -EINTR;
332                 goto out_resume_free;
333         }
334
335         if (target_name != NULL) {
336                 int len = strlen(target_name) + 1 +
337                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
338
339                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
340                 if (tgt->default_group_name == NULL) {
341                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
342                                 "group name failed");
343                         rc = -ENOMEM;
344                         goto out_unlock_resume;
345                 }
346                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
347                         target_name);
348         }
349
350         rc = scst_build_proc_target_entries(tgt);
351         if (rc < 0)
352                 goto out_free_name;
353         else
354                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
355
356         mutex_unlock(&scst_mutex);
357         scst_resume_activity();
358
359         PRINT_INFO("Target %s (%p) for template %s registered successfully",
360                 target_name, tgt, vtt->name);
361
362 out:
363         TRACE_EXIT();
364         return tgt;
365
366 out_free_name:
367         kfree(tgt->default_group_name);
368
369 out_unlock_resume:
370         mutex_unlock(&scst_mutex);
371
372 out_resume_free:
373         scst_resume_activity();
374
375 out_free_tgt_err:
376         kfree(tgt);
377         tgt = NULL;
378
379 out_err:
380         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
381                 target_name, vtt->name, rc);
382         goto out;
383 }
384 EXPORT_SYMBOL(scst_register);
385
386 static inline int test_sess_list(struct scst_tgt *tgt)
387 {
388         int res;
389         mutex_lock(&scst_mutex);
390         res = list_empty(&tgt->sess_list);
391         mutex_unlock(&scst_mutex);
392         return res;
393 }
394
395 void scst_unregister(struct scst_tgt *tgt)
396 {
397         struct scst_session *sess;
398         struct scst_tgt_template *vtt = tgt->tgtt;
399
400         TRACE_ENTRY();
401
402         TRACE_DBG("%s", "Calling target driver's release()");
403         tgt->tgtt->release(tgt);
404         TRACE_DBG("%s", "Target driver's release() returned");
405
406         mutex_lock(&scst_mutex);
407         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
408                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
409         }
410         mutex_unlock(&scst_mutex);
411
412         TRACE_DBG("%s", "Waiting for sessions shutdown");
413         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
414         TRACE_DBG("%s", "wait_event() returned");
415
416         scst_suspend_activity(false);
417         mutex_lock(&scst_mutex);
418
419         list_del(&tgt->tgt_list_entry);
420
421         scst_cleanup_proc_target_entries(tgt);
422
423         kfree(tgt->default_group_name);
424
425         mutex_unlock(&scst_mutex);
426         scst_resume_activity();
427
428         del_timer_sync(&tgt->retry_timer);
429
430         PRINT_INFO("Target %p for template %s unregistered successfully",
431                 tgt, vtt->name);
432
433         kfree(tgt);
434
435         TRACE_EXIT();
436         return;
437 }
438 EXPORT_SYMBOL(scst_unregister);
439
440 static int scst_susp_wait(bool interruptible)
441 {
442         int res = 0;
443
444         TRACE_ENTRY();
445
446         if (interruptible) {
447                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
448                         (atomic_read(&scst_cmd_count) == 0),
449                         SCST_SUSPENDING_TIMEOUT);
450                 if (res <= 0) {
451                         __scst_resume_activity();
452                         if (res == 0)
453                                 res = -EBUSY;
454                 } else
455                         res = 0;
456         } else
457                 wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
458
459         TRACE_MGMT_DBG("wait_event() returned %d", res);
460
461         TRACE_EXIT_RES(res);
462         return res;
463 }
464
465 int scst_suspend_activity(bool interruptible)
466 {
467         int res = 0;
468         bool rep = false;
469
470         TRACE_ENTRY();
471
472         if (interruptible) {
473                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
474                         res = -EINTR;
475                         goto out;
476                 }
477         } else
478                 mutex_lock(&scst_suspend_mutex);
479
480         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
481         suspend_count++;
482         if (suspend_count > 1)
483                 goto out_up;
484
485         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
486         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
487         smp_mb__after_set_bit();
488
489         /*
490          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
491          * information about scst_user behavior.
492          *
493          * ToDo: make the global suspending unneeded (Switch to per-device
494          * reference counting? That would mean to switch off from lockless
495          * implementation of scst_translate_lun().. )
496          */
497
498         if (atomic_read(&scst_cmd_count) != 0) {
499                 PRINT_INFO("Waiting for %d active commands to complete... This "
500                         "might take few minutes for disks or few hours for "
501                         "tapes, if you use long executed commands, like "
502                         "REWIND or FORMAT. In case, if you have a hung user "
503                         "space device (i.e. made using scst_user module) not "
504                         "responding to any commands, if might take virtually "
505                         "forever until the corresponding user space "
506                         "program recovers and starts responding or gets "
507                         "killed.", atomic_read(&scst_cmd_count));
508                 rep = true;
509         }
510
511         res = scst_susp_wait(interruptible);
512         if (res != 0)
513                 goto out_clear;
514
515         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
516         smp_mb__after_clear_bit();
517
518         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
519                 atomic_read(&scst_cmd_count));
520
521         res = scst_susp_wait(interruptible);
522         if (res != 0)
523                 goto out_clear;
524
525         if (rep)
526                 PRINT_INFO("%s", "All active commands completed");
527
528 out_up:
529         mutex_unlock(&scst_suspend_mutex);
530
531 out:
532         TRACE_EXIT_RES(res);
533         return res;
534
535 out_clear:
536         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
537         smp_mb__after_clear_bit();
538         goto out_up;
539 }
540 EXPORT_SYMBOL(scst_suspend_activity);
541
542 static void __scst_resume_activity(void)
543 {
544         struct scst_cmd_lists *l;
545
546         TRACE_ENTRY();
547
548         suspend_count--;
549         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
550         if (suspend_count > 0)
551                 goto out;
552
553         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
554         smp_mb__after_clear_bit();
555
556         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
557                 wake_up_all(&l->cmd_list_waitQ);
558         }
559         wake_up_all(&scst_init_cmd_list_waitQ);
560
561         spin_lock_irq(&scst_mcmd_lock);
562         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
563                 struct scst_mgmt_cmd *m;
564                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
565                                 mgmt_cmd_list_entry);
566                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
567                         "mgmt cmd list", m);
568                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
569         }
570         spin_unlock_irq(&scst_mcmd_lock);
571         wake_up_all(&scst_mgmt_cmd_list_waitQ);
572
573 out:
574         TRACE_EXIT();
575         return;
576 }
577
578 void scst_resume_activity(void)
579 {
580         TRACE_ENTRY();
581
582         mutex_lock(&scst_suspend_mutex);
583         __scst_resume_activity();
584         mutex_unlock(&scst_suspend_mutex);
585
586         TRACE_EXIT();
587         return;
588 }
589 EXPORT_SYMBOL(scst_resume_activity);
590
591 static int scst_register_device(struct scsi_device *scsidp)
592 {
593         int res = 0;
594         struct scst_device *dev;
595         struct scst_dev_type *dt;
596
597         TRACE_ENTRY();
598
599         res = scst_suspend_activity(true);
600         if (res != 0)
601                 goto out_err;
602
603         if (mutex_lock_interruptible(&scst_mutex) != 0) {
604                 res = -EINTR;
605                 goto out_resume;
606         }
607
608         res = scst_alloc_device(GFP_KERNEL, &dev);
609         if (res != 0)
610                 goto out_up;
611
612         dev->type = scsidp->type;
613
614         dev->rq_disk = alloc_disk(1);
615         if (dev->rq_disk == NULL) {
616                 res = -ENOMEM;
617                 goto out_free_dev;
618         }
619         dev->rq_disk->major = SCST_MAJOR;
620
621         dev->scsi_dev = scsidp;
622
623         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
624
625         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
626                 if (dt->type == scsidp->type) {
627                         res = scst_assign_dev_handler(dev, dt);
628                         if (res != 0)
629                                 goto out_free;
630                         break;
631                 }
632         }
633
634 out_up:
635         mutex_unlock(&scst_mutex);
636
637 out_resume:
638         scst_resume_activity();
639
640 out_err:
641         if (res == 0) {
642                 PRINT_INFO("Attached SCSI target mid-level at "
643                     "scsi%d, channel %d, id %d, lun %d, type %d",
644                     scsidp->host->host_no, scsidp->channel, scsidp->id,
645                     scsidp->lun, scsidp->type);
646         } else {
647                 PRINT_ERROR("Failed to attach SCSI target mid-level "
648                     "at scsi%d, channel %d, id %d, lun %d, type %d",
649                     scsidp->host->host_no, scsidp->channel, scsidp->id,
650                     scsidp->lun, scsidp->type);
651         }
652
653         TRACE_EXIT_RES(res);
654         return res;
655
656 out_free:
657         list_del(&dev->dev_list_entry);
658         put_disk(dev->rq_disk);
659
660 out_free_dev:
661         scst_free_device(dev);
662         goto out_up;
663 }
664
665 static void scst_unregister_device(struct scsi_device *scsidp)
666 {
667         struct scst_device *d, *dev = NULL;
668         struct scst_acg_dev *acg_dev, *aa;
669
670         TRACE_ENTRY();
671
672         scst_suspend_activity(false);
673         mutex_lock(&scst_mutex);
674
675         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
676                 if (d->scsi_dev == scsidp) {
677                         dev = d;
678                         TRACE_DBG("Target device %p found", dev);
679                         break;
680                 }
681         }
682         if (dev == NULL) {
683                 PRINT_ERROR("%s", "Target device not found");
684                 goto out_unblock;
685         }
686
687         list_del(&dev->dev_list_entry);
688
689         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
690                                  dev_acg_dev_list_entry)
691         {
692                 scst_acg_remove_dev(acg_dev->acg, dev);
693         }
694
695         scst_assign_dev_handler(dev, &scst_null_devtype);
696
697         put_disk(dev->rq_disk);
698         scst_free_device(dev);
699
700         PRINT_INFO("Detached SCSI target mid-level from scsi%d, channel %d, "
701                 "id %d, lun %d, type %d", scsidp->host->host_no,
702                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
703
704 out_unblock:
705         mutex_unlock(&scst_mutex);
706         scst_resume_activity();
707
708         TRACE_EXIT();
709         return;
710 }
711
712 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
713 {
714         int res = 0;
715
716         if (dev_handler->parse == NULL) {
717                 PRINT_ERROR("scst dev_type driver %s doesn't have a "
718                         "parse() method.", dev_handler->name);
719                 res = -EINVAL;
720                 goto out;
721         }
722
723         if (dev_handler->exec == NULL) {
724 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
725                 dev_handler->exec_atomic = 1;
726 #else
727                 dev_handler->exec_atomic = 0;
728 #endif
729         }
730
731         if (dev_handler->dev_done == NULL)
732                 dev_handler->dev_done_atomic = 1;
733
734 out:
735         TRACE_EXIT_RES(res);
736         return res;
737 }
738
739 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
740         const char *dev_name)
741 {
742         int res, rc;
743         struct scst_device *dev = NULL;
744
745         TRACE_ENTRY();
746
747         if (dev_handler == NULL) {
748                 PRINT_ERROR("%s: valid device handler must be supplied",
749                             __func__);
750                 res = -EINVAL;
751                 goto out;
752         }
753
754         if (dev_name == NULL) {
755                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
756                 res = -EINVAL;
757                 goto out;
758         }
759
760         res = scst_dev_handler_check(dev_handler);
761         if (res != 0)
762                 goto out;
763
764         res = scst_suspend_activity(true);
765         if (res != 0)
766                 goto out;
767
768         if (mutex_lock_interruptible(&scst_mutex) != 0) {
769                 res = -EINTR;
770                 goto out_resume;
771         }
772
773         res = scst_alloc_device(GFP_KERNEL, &dev);
774         if (res != 0)
775                 goto out_up;
776
777         dev->type = dev_handler->type;
778         dev->scsi_dev = NULL;
779         dev->virt_name = dev_name;
780         dev->virt_id = scst_virt_dev_last_id++;
781
782         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
783
784         res = dev->virt_id;
785
786         rc = scst_assign_dev_handler(dev, dev_handler);
787         if (rc != 0) {
788                 res = rc;
789                 goto out_free_del;
790         }
791
792 out_up:
793         mutex_unlock(&scst_mutex);
794
795 out_resume:
796         scst_resume_activity();
797
798 out:
799         if (res > 0) {
800                 PRINT_INFO("Attached SCSI target mid-level to virtual "
801                     "device %s (id %d)", dev_name, dev->virt_id);
802         } else {
803                 PRINT_INFO("Failed to attach SCSI target mid-level to "
804                     "virtual device %s", dev_name);
805         }
806
807         TRACE_EXIT_RES(res);
808         return res;
809
810 out_free_del:
811         list_del(&dev->dev_list_entry);
812         scst_free_device(dev);
813         goto out_up;
814 }
815 EXPORT_SYMBOL(scst_register_virtual_device);
816
817 void scst_unregister_virtual_device(int id)
818 {
819         struct scst_device *d, *dev = NULL;
820         struct scst_acg_dev *acg_dev, *aa;
821
822         TRACE_ENTRY();
823
824         scst_suspend_activity(false);
825         mutex_lock(&scst_mutex);
826
827         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
828                 if (d->virt_id == id) {
829                         dev = d;
830                         TRACE_DBG("Target device %p (id %d) found", dev, id);
831                         break;
832                 }
833         }
834         if (dev == NULL) {
835                 PRINT_ERROR("Target virtual device (id %d) not found", id);
836                 goto out_unblock;
837         }
838
839         list_del(&dev->dev_list_entry);
840
841         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
842                                  dev_acg_dev_list_entry)
843         {
844                 scst_acg_remove_dev(acg_dev->acg, dev);
845         }
846
847         scst_assign_dev_handler(dev, &scst_null_devtype);
848
849         PRINT_INFO("Detached SCSI target mid-level from virtual device %s "
850                 "(id %d)", dev->virt_name, dev->virt_id);
851
852         scst_free_device(dev);
853
854 out_unblock:
855         mutex_unlock(&scst_mutex);
856         scst_resume_activity();
857
858         TRACE_EXIT();
859         return;
860 }
861 EXPORT_SYMBOL(scst_unregister_virtual_device);
862
863 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
864         const char *version)
865 {
866         struct scst_dev_type *dt;
867         struct scst_device *dev;
868         int res;
869         int exist;
870
871         TRACE_ENTRY();
872
873         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
874                 PRINT_ERROR("Incorrect version of dev handler %s",
875                         dev_type->name);
876                 res = -EINVAL;
877                 goto out_error;
878         }
879
880         res = scst_dev_handler_check(dev_type);
881         if (res != 0)
882                 goto out_error;
883
884 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(CONFIG_SCST_STRICT_SERIALIZING)
885         if (dev_type->exec == NULL) {
886                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
887                         "supported. Consider applying on your kernel patch "
888                         "scst_exec_req_fifo-<kernel-version>.patch or define "
889                         "CONFIG_SCST_STRICT_SERIALIZING", dev_type->name);
890                 res = -EINVAL;
891                 goto out;
892         }
893 #endif
894
895         res = scst_suspend_activity(true);
896         if (res != 0)
897                 goto out_error;
898
899         if (mutex_lock_interruptible(&scst_mutex) != 0) {
900                 res = -EINTR;
901                 goto out_err_res;
902         }
903
904         exist = 0;
905         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
906                 if (strcmp(dt->name, dev_type->name) == 0) {
907                         PRINT_ERROR("Device type handler \"%s\" already "
908                                 "exist", dt->name);
909                         exist = 1;
910                         break;
911                 }
912         }
913         if (exist)
914                 goto out_up;
915
916         res = scst_build_proc_dev_handler_dir_entries(dev_type);
917         if (res < 0)
918                 goto out_up;
919
920         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
921
922         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
923                 if ((dev->scsi_dev == NULL) || (dev->handler != &scst_null_devtype))
924                         continue;
925                 if (dev->scsi_dev->type == dev_type->type)
926                         scst_assign_dev_handler(dev, dev_type);
927         }
928
929         mutex_unlock(&scst_mutex);
930         scst_resume_activity();
931
932         if (res == 0) {
933                 PRINT_INFO("Device handler \"%s\" for type %d registered "
934                         "successfully", dev_type->name, dev_type->type);
935         }
936
937 out:
938         TRACE_EXIT_RES(res);
939         return res;
940
941 out_up:
942         mutex_unlock(&scst_mutex);
943
944 out_err_res:
945         scst_resume_activity();
946
947 out_error:
948         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
949                 dev_type->name, dev_type->type);
950         goto out;
951 }
952 EXPORT_SYMBOL(__scst_register_dev_driver);
953
954 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
955 {
956         struct scst_device *dev;
957         struct scst_dev_type *dt;
958         int found = 0;
959
960         TRACE_ENTRY();
961
962         scst_suspend_activity(false);
963         mutex_lock(&scst_mutex);
964
965         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
966                 if (strcmp(dt->name, dev_type->name) == 0) {
967                         found = 1;
968                         break;
969                 }
970         }
971         if (!found) {
972                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
973                         dev_type->name);
974                 goto out_up;
975         }
976
977         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
978                 if (dev->handler == dev_type) {
979                         scst_assign_dev_handler(dev, &scst_null_devtype);
980                         TRACE_DBG("Dev handler removed from device %p", dev);
981                 }
982         }
983
984         list_del(&dev_type->dev_type_list_entry);
985
986         mutex_unlock(&scst_mutex);
987         scst_resume_activity();
988
989         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
990
991         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
992                    dev_type->name, dev_type->type);
993
994 out:
995         TRACE_EXIT();
996         return;
997
998 out_up:
999         mutex_unlock(&scst_mutex);
1000         scst_resume_activity();
1001         goto out;
1002 }
1003 EXPORT_SYMBOL(scst_unregister_dev_driver);
1004
1005 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1006         const char *version)
1007 {
1008         int res;
1009
1010         TRACE_ENTRY();
1011
1012         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1013                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1014                         dev_type->name);
1015                 res = -EINVAL;
1016                 goto out_err;
1017         }
1018
1019         res = scst_dev_handler_check(dev_type);
1020         if (res != 0)
1021                 goto out_err;
1022
1023         if (!dev_type->no_proc) {
1024                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1025                 if (res < 0)
1026                         goto out_err;
1027         }
1028
1029         if (dev_type->type != -1) {
1030                 PRINT_INFO("Virtual device handler %s for type %d "
1031                         "registered successfully", dev_type->name,
1032                         dev_type->type);
1033         } else {
1034                 PRINT_INFO("Virtual device handler \"%s\" registered "
1035                         "successfully", dev_type->name);
1036         }
1037
1038 out:
1039         TRACE_EXIT_RES(res);
1040         return res;
1041
1042 out_err:
1043         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1044                 dev_type->name);
1045         goto out;
1046 }
1047 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1048
1049 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1050 {
1051         TRACE_ENTRY();
1052
1053         if (!dev_type->no_proc)
1054                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1055
1056         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1057
1058         TRACE_EXIT();
1059         return;
1060 }
1061 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1062
1063 /* Called under scst_mutex */
1064 int scst_add_dev_threads(struct scst_device *dev, int num)
1065 {
1066         int i, res = 0;
1067         int n = 0;
1068         struct scst_cmd_thread_t *thr;
1069         char nm[12];
1070
1071         TRACE_ENTRY();
1072
1073         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1074                 n++;
1075         }
1076
1077         for (i = 0; i < num; i++) {
1078                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1079                 if (!thr) {
1080                         res = -ENOMEM;
1081                         PRINT_ERROR("Failed to allocate thr %d", res);
1082                         goto out;
1083                 }
1084                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1085                 nm[ARRAY_SIZE(nm)-1] = '\0';
1086                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1087                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1088                 if (IS_ERR(thr->cmd_thread)) {
1089                         res = PTR_ERR(thr->cmd_thread);
1090                         PRINT_ERROR("kthread_create() failed: %d", res);
1091                         kfree(thr);
1092                         goto out;
1093                 }
1094                 list_add(&thr->thread_list_entry, &dev->threads_list);
1095         }
1096
1097 out:
1098         TRACE_EXIT_RES(res);
1099         return res;
1100 }
1101
1102 /* Called under scst_mutex and suspended activity */
1103 static int scst_create_dev_threads(struct scst_device *dev)
1104 {
1105         int res = 0;
1106         int threads_num;
1107
1108         TRACE_ENTRY();
1109
1110         if (dev->handler->threads_num <= 0)
1111                 goto out;
1112
1113         threads_num = dev->handler->threads_num;
1114
1115         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1116         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1117         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1118
1119         res = scst_add_dev_threads(dev, threads_num);
1120         if (res != 0)
1121                 goto out;
1122
1123         mutex_lock(&scst_suspend_mutex);
1124         list_add_tail(&dev->cmd_lists.lists_list_entry,
1125                 &scst_cmd_lists_list);
1126         mutex_unlock(&scst_suspend_mutex);
1127
1128         dev->p_cmd_lists = &dev->cmd_lists;
1129
1130 out:
1131         TRACE_EXIT_RES(res);
1132         return res;
1133 }
1134
1135 /* Called under scst_mutex */
1136 void scst_del_dev_threads(struct scst_device *dev, int num)
1137 {
1138         struct scst_cmd_thread_t *ct, *tmp;
1139         int i = 0;
1140
1141         TRACE_ENTRY();
1142
1143         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
1144                                 thread_list_entry) {
1145                 int rc = kthread_stop(ct->cmd_thread);
1146                 if (rc < 0)
1147                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1148                 list_del(&ct->thread_list_entry);
1149                 kfree(ct);
1150                 if ((num > 0) && (++i >= num))
1151                         break;
1152         }
1153
1154         TRACE_EXIT();
1155         return;
1156 }
1157
1158 /* Called under scst_mutex and suspended activity */
1159 static void scst_stop_dev_threads(struct scst_device *dev)
1160 {
1161         TRACE_ENTRY();
1162
1163         if (list_empty(&dev->threads_list))
1164                 goto out;
1165
1166         scst_del_dev_threads(dev, -1);
1167
1168         if (dev->p_cmd_lists == &dev->cmd_lists) {
1169                 mutex_lock(&scst_suspend_mutex);
1170                 list_del(&dev->cmd_lists.lists_list_entry);
1171                 mutex_unlock(&scst_suspend_mutex);
1172         }
1173
1174 out:
1175         TRACE_EXIT();
1176         return;
1177 }
1178
1179 /* The activity supposed to be suspended and scst_mutex held */
1180 int scst_assign_dev_handler(struct scst_device *dev,
1181         struct scst_dev_type *handler)
1182 {
1183         int res = 0;
1184         struct scst_tgt_dev *tgt_dev;
1185         LIST_HEAD(attached_tgt_devs);
1186
1187         TRACE_ENTRY();
1188
1189         sBUG_ON(handler == NULL);
1190
1191         if (dev->handler == handler)
1192                 goto out;
1193
1194         if (dev->handler && dev->handler->detach_tgt) {
1195                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1196                                 dev_tgt_dev_list_entry) {
1197                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1198                                 tgt_dev);
1199                         dev->handler->detach_tgt(tgt_dev);
1200                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1201                 }
1202         }
1203
1204         if (dev->handler && dev->handler->detach) {
1205                 TRACE_DBG("%s", "Calling dev handler's detach()");
1206                 dev->handler->detach(dev);
1207                 TRACE_DBG("%s", "Old handler's detach() returned");
1208         }
1209
1210         scst_stop_dev_threads(dev);
1211
1212         dev->handler = handler;
1213
1214         if (handler) {
1215                 res = scst_create_dev_threads(dev);
1216                 if (res != 0)
1217                         goto out_null;
1218         }
1219
1220         if (handler && handler->attach) {
1221                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1222                 res = handler->attach(dev);
1223                 TRACE_DBG("New dev handler's attach() returned %d", res);
1224                 if (res != 0) {
1225                         PRINT_ERROR("New device handler's %s attach() "
1226                                 "failed: %d", handler->name, res);
1227                 }
1228                 goto out_thr_null;
1229         }
1230
1231         if (handler && handler->attach_tgt) {
1232                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1233                                 dev_tgt_dev_list_entry) {
1234                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1235                                 tgt_dev);
1236                         res = handler->attach_tgt(tgt_dev);
1237                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1238                         if (res != 0) {
1239                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1240                                     "failed: %d", handler->name, res);
1241                                 goto out_err_detach_tgt;
1242                         }
1243                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1244                                 &attached_tgt_devs);
1245                 }
1246         }
1247
1248 out_thr_null:
1249         if (res != 0)
1250                 scst_stop_dev_threads(dev);
1251
1252 out_null:
1253         if (res != 0)
1254                 dev->handler = &scst_null_devtype;
1255
1256 out:
1257         TRACE_EXIT_RES(res);
1258         return res;
1259
1260 out_err_detach_tgt:
1261         if (handler && handler->detach_tgt) {
1262                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1263                                  extra_tgt_dev_list_entry)
1264                 {
1265                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1266                                 tgt_dev);
1267                         handler->detach_tgt(tgt_dev);
1268                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1269                 }
1270         }
1271         if (handler && handler->detach) {
1272                 TRACE_DBG("%s", "Calling handler's detach()");
1273                 handler->detach(dev);
1274                 TRACE_DBG("%s", "Handler's detach() returned");
1275         }
1276         goto out_null;
1277 }
1278
1279 int scst_cmd_threads_count(void)
1280 {
1281         int i;
1282
1283         /* Just to lower the race window, when user can get just changed value */
1284         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1285         i = scst_threads_info.nr_cmd_threads;
1286         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1287         return i;
1288 }
1289
1290 static void scst_threads_info_init(void)
1291 {
1292         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1293         mutex_init(&scst_threads_info.cmd_threads_mutex);
1294         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1295 }
1296
1297 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1298 void __scst_del_cmd_threads(int num)
1299 {
1300         struct scst_cmd_thread_t *ct, *tmp;
1301         int i;
1302
1303         TRACE_ENTRY();
1304
1305         i = scst_threads_info.nr_cmd_threads;
1306         if (num <= 0 || num > i) {
1307                 PRINT_ERROR("can not del %d cmd threads from %d", num, i);
1308                 return;
1309         }
1310
1311         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1312                                 thread_list_entry) {
1313                 int res;
1314
1315                 res = kthread_stop(ct->cmd_thread);
1316                 if (res < 0)
1317                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1318                 list_del(&ct->thread_list_entry);
1319                 kfree(ct);
1320                 scst_threads_info.nr_cmd_threads--;
1321                 --num;
1322                 if (num == 0)
1323                         break;
1324         }
1325
1326         TRACE_EXIT();
1327         return;
1328 }
1329
1330 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1331 int __scst_add_cmd_threads(int num)
1332 {
1333         int res = 0, i;
1334         static int scst_thread_num;
1335
1336         TRACE_ENTRY();
1337
1338         for (i = 0; i < num; i++) {
1339                 struct scst_cmd_thread_t *thr;
1340
1341                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1342                 if (!thr) {
1343                         res = -ENOMEM;
1344                         PRINT_ERROR("fail to allocate thr %d", res);
1345                         goto out_error;
1346                 }
1347                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1348                         &scst_main_cmd_lists, "scsi_tgt%d",
1349                         scst_thread_num++);
1350                 if (IS_ERR(thr->cmd_thread)) {
1351                         res = PTR_ERR(thr->cmd_thread);
1352                         PRINT_ERROR("kthread_create() failed: %d", res);
1353                         kfree(thr);
1354                         goto out_error;
1355                 }
1356                 list_add(&thr->thread_list_entry,
1357                         &scst_threads_info.cmd_threads_list);
1358                 scst_threads_info.nr_cmd_threads++;
1359         }
1360         res = 0;
1361
1362 out:
1363         TRACE_EXIT_RES(res);
1364         return res;
1365
1366 out_error:
1367         if (i > 0)
1368                 __scst_del_cmd_threads(i - 1);
1369         goto out;
1370 }
1371
1372 int scst_add_cmd_threads(int num)
1373 {
1374         int res;
1375
1376         TRACE_ENTRY();
1377
1378         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1379         res = __scst_add_cmd_threads(num);
1380         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1381
1382         TRACE_EXIT_RES(res);
1383         return res;
1384 }
1385 EXPORT_SYMBOL(scst_add_cmd_threads);
1386
1387 void scst_del_cmd_threads(int num)
1388 {
1389         TRACE_ENTRY();
1390
1391         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1392         __scst_del_cmd_threads(num);
1393         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1394
1395         TRACE_EXIT();
1396         return;
1397 }
1398 EXPORT_SYMBOL(scst_del_cmd_threads);
1399
1400 static void scst_stop_all_threads(void)
1401 {
1402         TRACE_ENTRY();
1403
1404         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1405         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1406         if (scst_threads_info.mgmt_cmd_thread)
1407                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1408         if (scst_threads_info.mgmt_thread)
1409                 kthread_stop(scst_threads_info.mgmt_thread);
1410         if (scst_threads_info.init_cmd_thread)
1411                 kthread_stop(scst_threads_info.init_cmd_thread);
1412         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1413
1414         TRACE_EXIT();
1415         return;
1416 }
1417
1418 static int scst_start_all_threads(int num)
1419 {
1420         int res;
1421
1422         TRACE_ENTRY();
1423
1424         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1425         res = __scst_add_cmd_threads(num);
1426         if (res < 0)
1427                 goto out;
1428
1429         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1430                 NULL, "scsi_tgt_init");
1431         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1432                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1433                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1434                 scst_threads_info.init_cmd_thread = NULL;
1435                 goto out;
1436         }
1437
1438         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1439                 NULL, "scsi_tgt_mc");
1440         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1441                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1442                 PRINT_ERROR("kthread_create() for mcmd failed: %d", res);
1443                 scst_threads_info.mgmt_cmd_thread = NULL;
1444                 goto out;
1445         }
1446
1447         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1448                 NULL, "scsi_tgt_mgmt");
1449         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1450                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1451                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1452                 scst_threads_info.mgmt_thread = NULL;
1453                 goto out;
1454         }
1455
1456 out:
1457         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1458         TRACE_EXIT_RES(res);
1459         return res;
1460 }
1461
1462 void scst_get(void)
1463 {
1464         __scst_get(0);
1465 }
1466 EXPORT_SYMBOL(scst_get);
1467
1468 void scst_put(void)
1469 {
1470         __scst_put();
1471 }
1472 EXPORT_SYMBOL(scst_put);
1473
1474 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1475 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1476 #else
1477 static int scst_add(struct device *cdev, struct class_interface *intf)
1478 #endif
1479 {
1480         struct scsi_device *scsidp;
1481         int res = 0;
1482
1483         TRACE_ENTRY();
1484
1485 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1486         scsidp = to_scsi_device(cdev->dev);
1487 #else
1488         scsidp = to_scsi_device(cdev->parent);
1489 #endif
1490
1491         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1492                 res = scst_register_device(scsidp);
1493
1494         TRACE_EXIT();
1495         return res;
1496 }
1497
1498 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1499 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1500 #else
1501 static void scst_remove(struct device *cdev, struct class_interface *intf)
1502 #endif
1503 {
1504         struct scsi_device *scsidp;
1505
1506         TRACE_ENTRY();
1507
1508 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1509         scsidp = to_scsi_device(cdev->dev);
1510 #else
1511         scsidp = to_scsi_device(cdev->parent);
1512 #endif
1513
1514         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1515                 scst_unregister_device(scsidp);
1516
1517         TRACE_EXIT();
1518         return;
1519 }
1520
1521 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1522 static struct class_interface scst_interface = {
1523         .add = scst_add,
1524         .remove = scst_remove,
1525 };
1526 #else
1527 static struct class_interface scst_interface = {
1528         .add_dev = scst_add,
1529         .remove_dev = scst_remove,
1530 };
1531 #endif
1532
1533 static void __init scst_print_config(void)
1534 {
1535         char buf[128];
1536         int i, j;
1537
1538         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1539         j = i;
1540
1541 #ifdef CONFIG_SCST_STRICT_SERIALIZING
1542         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1543 #endif
1544
1545 #ifdef CONFIG_SCST_EXTRACHECKS
1546         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1547                 (j == i) ? "" : ", ");
1548 #endif
1549
1550 #ifdef CONFIG_SCST_TRACING
1551         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1552                 (j == i) ? "" : ", ");
1553 #endif
1554
1555 #ifdef CONFIG_SCST_DEBUG
1556         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1557                 (j == i) ? "" : ", ");
1558 #endif
1559
1560 #ifdef CONFIG_SCST_DEBUG_TM
1561         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1562                 (j == i) ? "" : ", ");
1563 #endif
1564
1565 #ifdef CONFIG_SCST_DEBUG_RETRY
1566         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1567                 (j == i) ? "" : ", ");
1568 #endif
1569
1570 #ifdef CONFIG_SCST_DEBUG_OOM
1571         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1572                 (j == i) ? "" : ", ");
1573 #endif
1574
1575 #ifdef CONFIG_SCST_DEBUG_SN
1576         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1577                 (j == i) ? "" : ", ");
1578 #endif
1579
1580 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
1581         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1582                 (j == i) ? "" : ", ");
1583 #endif
1584
1585 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1586         i += snprintf(&buf[i], sizeof(buf) - i, "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1587                 (j == i) ? "" : ", ");
1588 #endif
1589
1590 #ifdef CONFIG_SCST_STRICT_SECURITY
1591         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1592                 (j == i) ? "" : ", ");
1593 #endif
1594
1595         if (j != i)
1596                 PRINT_INFO("%s", buf);
1597 }
1598
1599 static int __init init_scst(void)
1600 {
1601         int res = 0, i;
1602         int scst_num_cpus;
1603
1604         TRACE_ENTRY();
1605
1606 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1607         {
1608                 struct scsi_request *req;
1609                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1610                         sizeof(req->sr_sense_buffer));
1611         }
1612 #else
1613         {
1614                 struct scsi_sense_hdr *shdr;
1615                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1616         }
1617 #endif
1618         {
1619                 struct scst_tgt_dev *t;
1620                 struct scst_cmd *c;
1621                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1622                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1623         }
1624
1625         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1626         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1627         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1628         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1629
1630         mutex_init(&scst_mutex);
1631         INIT_LIST_HEAD(&scst_template_list);
1632         INIT_LIST_HEAD(&scst_dev_list);
1633         INIT_LIST_HEAD(&scst_dev_type_list);
1634         spin_lock_init(&scst_main_lock);
1635         INIT_LIST_HEAD(&scst_acg_list);
1636         spin_lock_init(&scst_init_lock);
1637         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1638         INIT_LIST_HEAD(&scst_init_cmd_list);
1639 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1640         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1641 #endif
1642         atomic_set(&scst_cmd_count, 0);
1643         spin_lock_init(&scst_mcmd_lock);
1644         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1645         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1646         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1647         init_waitqueue_head(&scst_mgmt_waitQ);
1648         spin_lock_init(&scst_mgmt_lock);
1649         INIT_LIST_HEAD(&scst_sess_init_list);
1650         INIT_LIST_HEAD(&scst_sess_shut_list);
1651         init_waitqueue_head(&scst_dev_cmd_waitQ);
1652         mutex_init(&scst_suspend_mutex);
1653         INIT_LIST_HEAD(&scst_cmd_lists_list);
1654         scst_virt_dev_last_id = 1;
1655         spin_lock_init(&scst_temp_UA_lock);
1656
1657         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1658         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1659         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1660         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1661                 &scst_cmd_lists_list);
1662
1663         scst_num_cpus = num_online_cpus();
1664
1665         /* ToDo: register_cpu_notifier() */
1666
1667         if (scst_threads == 0)
1668                 scst_threads = scst_num_cpus;
1669
1670         if (scst_threads < 1) {
1671                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1672                 scst_threads = scst_num_cpus;
1673         }
1674
1675         scst_threads_info_init();
1676
1677 #define INIT_CACHEP(p, s, o) do {                                       \
1678                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1679                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1680                           sizeof(struct s));                            \
1681                 if (p == NULL) {                                        \
1682                         res = -ENOMEM;                                  \
1683                         goto o;                                         \
1684                 }                                                       \
1685         } while (0)
1686
1687         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out);
1688         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1689                         out_destroy_mgmt_cache);
1690         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1691                         out_destroy_mgmt_stub_cache);
1692         {
1693                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1694                 INIT_CACHEP(scst_sense_cachep, scst_sense, out_destroy_ua_cache);
1695         }
1696         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_sense_cache);
1697         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1698         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1699         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1700
1701         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1702                 mempool_free_slab, scst_mgmt_cachep);
1703         if (scst_mgmt_mempool == NULL) {
1704                 res = -ENOMEM;
1705                 goto out_destroy_acg_cache;
1706         }
1707
1708         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1709                 mempool_free_slab, scst_mgmt_stub_cachep);
1710         if (scst_mgmt_stub_mempool == NULL) {
1711                 res = -ENOMEM;
1712                 goto out_destroy_mgmt_mempool;
1713         }
1714
1715         scst_ua_mempool = mempool_create(64, mempool_alloc_slab,
1716                 mempool_free_slab, scst_ua_cachep);
1717         if (scst_ua_mempool == NULL) {
1718                 res = -ENOMEM;
1719                 goto out_destroy_mgmt_stub_mempool;
1720         }
1721
1722         /* Loosing sense may have fatal consequences, so let's have a big pool */
1723         scst_sense_mempool = mempool_create(128, mempool_alloc_slab,
1724                 mempool_free_slab, scst_sense_cachep);
1725         if (scst_sense_mempool == NULL) {
1726                 res = -ENOMEM;
1727                 goto out_destroy_ua_mempool;
1728         }
1729
1730         if (scst_max_cmd_mem == 0) {
1731                 struct sysinfo si;
1732                 si_meminfo(&si);
1733 #if BITS_PER_LONG == 32
1734                 scst_max_cmd_mem = min(
1735                         (((uint64_t)si.totalram << PAGE_SHIFT) >> 20) >> 2,
1736                         (uint64_t)1 << 30);
1737 #else
1738                 scst_max_cmd_mem = ((si.totalram << PAGE_SHIFT) >> 20) >> 2;
1739 #endif
1740         }
1741
1742         if (scst_max_dev_cmd_mem != 0) {
1743                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
1744                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
1745                                 "scst_max_cmd_mem (%d)",
1746                                 scst_max_dev_cmd_mem,
1747                                 scst_max_cmd_mem);
1748                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
1749                 }
1750         } else
1751                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
1752
1753         res = scst_sgv_pools_init(
1754                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
1755         if (res != 0)
1756                 goto out_destroy_sense_mempool;
1757
1758         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1759         if (scst_default_acg == NULL) {
1760                 res = -ENOMEM;
1761                 goto out_destroy_sgv_pool;
1762         }
1763
1764         res = scsi_register_interface(&scst_interface);
1765         if (res != 0)
1766                 goto out_free_acg;
1767
1768         scst_scsi_op_list_init();
1769
1770         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1771                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1772                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1773                 tasklet_init(&scst_tasklets[i].tasklet, (void *)scst_cmd_tasklet,
1774                         (unsigned long)&scst_tasklets[i]);
1775         }
1776
1777         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1778                 scst_threads);
1779
1780         res = scst_start_all_threads(scst_threads);
1781         if (res < 0)
1782                 goto out_thread_free;
1783
1784         res = scst_proc_init_module();
1785         if (res != 0)
1786                 goto out_thread_free;
1787
1788
1789         PRINT_INFO("SCST version %s loaded successfully (max mem for "
1790                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
1791                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
1792
1793         scst_print_config();
1794
1795 out:
1796         TRACE_EXIT_RES(res);
1797         return res;
1798
1799 out_thread_free:
1800         scst_stop_all_threads();
1801
1802         scsi_unregister_interface(&scst_interface);
1803
1804 out_free_acg:
1805         scst_destroy_acg(scst_default_acg);
1806
1807 out_destroy_sgv_pool:
1808         scst_sgv_pools_deinit();
1809
1810 out_destroy_sense_mempool:
1811         mempool_destroy(scst_sense_mempool);
1812
1813 out_destroy_ua_mempool:
1814         mempool_destroy(scst_ua_mempool);
1815
1816 out_destroy_mgmt_stub_mempool:
1817         mempool_destroy(scst_mgmt_stub_mempool);
1818
1819 out_destroy_mgmt_mempool:
1820         mempool_destroy(scst_mgmt_mempool);
1821
1822 out_destroy_acg_cache:
1823         kmem_cache_destroy(scst_acgd_cachep);
1824
1825 out_destroy_tgt_cache:
1826         kmem_cache_destroy(scst_tgtd_cachep);
1827
1828 out_destroy_sess_cache:
1829         kmem_cache_destroy(scst_sess_cachep);
1830
1831 out_destroy_cmd_cache:
1832         kmem_cache_destroy(scst_cmd_cachep);
1833
1834 out_destroy_sense_cache:
1835         kmem_cache_destroy(scst_sense_cachep);
1836
1837 out_destroy_ua_cache:
1838         kmem_cache_destroy(scst_ua_cachep);
1839
1840 out_destroy_mgmt_stub_cache:
1841         kmem_cache_destroy(scst_mgmt_stub_cachep);
1842
1843 out_destroy_mgmt_cache:
1844         kmem_cache_destroy(scst_mgmt_cachep);
1845         goto out;
1846 }
1847
1848 static void __exit exit_scst(void)
1849 {
1850 #ifdef CONFIG_LOCKDEP
1851         static /* To hide the lockdep's warning about non-static key */
1852 #endif
1853         DECLARE_MUTEX_LOCKED(shm);
1854
1855         TRACE_ENTRY();
1856
1857         /* ToDo: unregister_cpu_notifier() */
1858
1859         scst_proc_cleanup_module();
1860
1861         scst_stop_all_threads();
1862
1863         scsi_unregister_interface(&scst_interface);
1864         scst_destroy_acg(scst_default_acg);
1865
1866         scst_sgv_pools_deinit();
1867
1868 #define DEINIT_CACHEP(p) do {           \
1869                 kmem_cache_destroy(p);  \
1870                 p = NULL;               \
1871         } while (0)
1872
1873         mempool_destroy(scst_mgmt_mempool);
1874         mempool_destroy(scst_mgmt_stub_mempool);
1875         mempool_destroy(scst_ua_mempool);
1876         mempool_destroy(scst_sense_mempool);
1877
1878         DEINIT_CACHEP(scst_mgmt_cachep);
1879         DEINIT_CACHEP(scst_mgmt_stub_cachep);
1880         DEINIT_CACHEP(scst_ua_cachep);
1881         DEINIT_CACHEP(scst_sense_cachep);
1882         DEINIT_CACHEP(scst_cmd_cachep);
1883         DEINIT_CACHEP(scst_sess_cachep);
1884         DEINIT_CACHEP(scst_tgtd_cachep);
1885         DEINIT_CACHEP(scst_acgd_cachep);
1886
1887         PRINT_INFO("%s", "SCST unloaded");
1888
1889         TRACE_EXIT();
1890         return;
1891 }
1892
1893
1894 module_init(init_scst);
1895 module_exit(exit_scst);
1896
1897 MODULE_AUTHOR("Vladislav Bolkhovitin");
1898 MODULE_LICENSE("GPL");
1899 MODULE_DESCRIPTION("SCSI target core");
1900 MODULE_VERSION(SCST_VERSION_STRING);