33708c306dbeedce9e6cefbad0acbd3ff332cca5
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/module.h>
20
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/list.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/sched.h>
28 #include <linux/unistd.h>
29 #include <linux/string.h>
30 #include <linux/kthread.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34 #include "scst_mem.h"
35
36 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
37 #warning "HIGHMEM kernel configurations are fully supported, but not\
38  recommended for performance reasons. Consider changing VMSPLIT\
39  option or use a 64-bit configuration instead. See README file for\
40  details."
41 #endif
42
43 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
44 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && \
45     !defined(CONFIG_SCST_STRICT_SERIALIZING)
46 #warning "Patch scst_exec_req_fifo-<kernel-version> was not applied on\
47  your kernel and CONFIG_SCST_STRICT_SERIALIZING isn't defined.\
48  Pass-through dev handlers will not work."
49 #endif
50 #endif
51
52 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
53 #if !defined(SCST_IO_CONTEXT)
54 #warning "Patch io_context-<kernel-version> was not applied\
55  on your kernel. SCST will be working with not the best performance."
56 #endif
57 #else
58 #warning "There is no patch io_context-<kernel-version>\
59  for your kernel version. For performance reasons it is strongly recommended\
60  to upgrade your kernel to version >= 2.6.27.x."
61 #endif
62
63 /**
64  ** SCST global variables. They are all uninitialized to have their layout in
65  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
66  ** variable separately from nonzero-initialized ones.
67  **/
68
69 /*
70  * All targets, devices and dev_types management is done under this mutex.
71  *
72  * It must NOT be used in any works (schedule_work(), etc.), because
73  * otherwise a deadlock (double lock, actually) is possible, e.g., with
74  * scst_user detach_tgt(), which is called under scst_mutex and calls
75  * flush_scheduled_work().
76  */
77 struct mutex scst_mutex;
78
79  /* All 3 protected by scst_mutex */
80 struct list_head scst_template_list;
81 struct list_head scst_dev_list;
82 struct list_head scst_dev_type_list;
83
84 spinlock_t scst_main_lock;
85
86 static struct kmem_cache *scst_mgmt_cachep;
87 mempool_t *scst_mgmt_mempool;
88 static struct kmem_cache *scst_mgmt_stub_cachep;
89 mempool_t *scst_mgmt_stub_mempool;
90 static struct kmem_cache *scst_ua_cachep;
91 mempool_t *scst_ua_mempool;
92 static struct kmem_cache *scst_sense_cachep;
93 mempool_t *scst_sense_mempool;
94 static struct kmem_cache *scst_aen_cachep;
95 mempool_t *scst_aen_mempool;
96 struct kmem_cache *scst_tgtd_cachep;
97 struct kmem_cache *scst_sess_cachep;
98 struct kmem_cache *scst_acgd_cachep;
99
100 struct list_head scst_acg_list;
101 struct scst_acg *scst_default_acg;
102
103 spinlock_t scst_init_lock;
104 wait_queue_head_t scst_init_cmd_list_waitQ;
105 struct list_head scst_init_cmd_list;
106 unsigned int scst_init_poll_cnt;
107
108 struct kmem_cache *scst_cmd_cachep;
109
110 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
111 unsigned long scst_trace_flag;
112 #endif
113
114 unsigned long scst_flags;
115 atomic_t scst_cmd_count;
116
117 struct scst_cmd_lists scst_main_cmd_lists;
118
119 struct scst_tasklet scst_tasklets[NR_CPUS];
120
121 spinlock_t scst_mcmd_lock;
122 struct list_head scst_active_mgmt_cmd_list;
123 struct list_head scst_delayed_mgmt_cmd_list;
124 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
125
126 wait_queue_head_t scst_mgmt_waitQ;
127 spinlock_t scst_mgmt_lock;
128 struct list_head scst_sess_init_list;
129 struct list_head scst_sess_shut_list;
130
131 wait_queue_head_t scst_dev_cmd_waitQ;
132
133 static struct mutex scst_suspend_mutex;
134 /* protected by scst_suspend_mutex */
135 static struct list_head scst_cmd_lists_list;
136
137 static int scst_threads;
138 struct mutex scst_global_threads_mutex;
139 u32 scst_nr_global_threads;
140 static struct list_head scst_global_threads_list;
141 static struct task_struct *scst_init_cmd_thread;
142 static struct task_struct *scst_mgmt_thread;
143 static struct task_struct *scst_mgmt_cmd_thread;
144
145 static int suspend_count;
146
147 static int scst_virt_dev_last_id; /* protected by scst_mutex */
148
149 static unsigned int scst_max_cmd_mem;
150 unsigned int scst_max_dev_cmd_mem;
151
152 module_param_named(scst_threads, scst_threads, int, 0);
153 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
154
155 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, S_IRUGO);
156 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
157         "all SCSI commands of all devices at any given time in MB");
158
159 module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, S_IRUGO);
160 MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
161         "by all SCSI commands of a device at any given time in MB");
162
163 struct scst_dev_type scst_null_devtype = {
164         .name = "none",
165 };
166
167 static void __scst_resume_activity(void);
168
169 int __scst_register_target_template(struct scst_tgt_template *vtt,
170         const char *version)
171 {
172         int res = 0;
173         struct scst_tgt_template *t;
174         static DEFINE_MUTEX(m);
175
176         TRACE_ENTRY();
177
178         INIT_LIST_HEAD(&vtt->tgt_list);
179
180         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
181                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
182                 res = -EINVAL;
183                 goto out_err;
184         }
185
186         if (!vtt->detect) {
187                 PRINT_ERROR("Target driver %s doesn't have a "
188                         "detect() method.", vtt->name);
189                 res = -EINVAL;
190                 goto out_err;
191         }
192
193         if (!vtt->release) {
194                 PRINT_ERROR("Target driver %s doesn't have a "
195                         "release() method.", vtt->name);
196                 res = -EINVAL;
197                 goto out_err;
198         }
199
200         if (!vtt->xmit_response) {
201                 PRINT_ERROR("Target driver %s doesn't have a "
202                         "xmit_response() method.", vtt->name);
203                 res = -EINVAL;
204                 goto out_err;
205         }
206
207         if (vtt->threads_num < 0) {
208                 PRINT_ERROR("Wrong threads_num value %d for "
209                         "target \"%s\"", vtt->threads_num,
210                         vtt->name);
211                 res = -EINVAL;
212                 goto out_err;
213         }
214
215         if (!vtt->no_proc_entry) {
216                 res = scst_build_proc_target_dir_entries(vtt);
217                 if (res < 0)
218                         goto out_err;
219         }
220
221         if (vtt->rdy_to_xfer == NULL)
222                 vtt->rdy_to_xfer_atomic = 1;
223
224         if (mutex_lock_interruptible(&m) != 0)
225                 goto out_err;
226
227         if (mutex_lock_interruptible(&scst_mutex) != 0)
228                 goto out_m_up;
229         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
230                 if (strcmp(t->name, vtt->name) == 0) {
231                         PRINT_ERROR("Target driver %s already registered",
232                                 vtt->name);
233                         mutex_unlock(&scst_mutex);
234                         goto out_cleanup;
235                 }
236         }
237         mutex_unlock(&scst_mutex);
238
239         TRACE_DBG("%s", "Calling target driver's detect()");
240         res = vtt->detect(vtt);
241         TRACE_DBG("Target driver's detect() returned %d", res);
242         if (res < 0) {
243                 PRINT_ERROR("%s", "The detect() routine failed");
244                 res = -EINVAL;
245                 goto out_cleanup;
246         }
247
248         mutex_lock(&scst_mutex);
249         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
250         mutex_unlock(&scst_mutex);
251
252         res = 0;
253
254         PRINT_INFO("Target template %s registered successfully", vtt->name);
255
256         mutex_unlock(&m);
257
258 out:
259         TRACE_EXIT_RES(res);
260         return res;
261
262 out_cleanup:
263         scst_cleanup_proc_target_dir_entries(vtt);
264
265 out_m_up:
266         mutex_unlock(&m);
267
268 out_err:
269         PRINT_ERROR("Failed to register target template %s", vtt->name);
270         goto out;
271 }
272 EXPORT_SYMBOL(__scst_register_target_template);
273
274 void scst_unregister_target_template(struct scst_tgt_template *vtt)
275 {
276         struct scst_tgt *tgt;
277         struct scst_tgt_template *t;
278         int found = 0;
279
280         TRACE_ENTRY();
281
282         mutex_lock(&scst_mutex);
283
284         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
285                 if (strcmp(t->name, vtt->name) == 0) {
286                         found = 1;
287                         break;
288                 }
289         }
290         if (!found) {
291                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
292                 goto out_up;
293         }
294
295 restart:
296         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
297                 mutex_unlock(&scst_mutex);
298                 scst_unregister(tgt);
299                 mutex_lock(&scst_mutex);
300                 goto restart;
301         }
302         list_del(&vtt->scst_template_list_entry);
303
304         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
305
306 out_up:
307         mutex_unlock(&scst_mutex);
308
309         scst_cleanup_proc_target_dir_entries(vtt);
310
311         TRACE_EXIT();
312         return;
313 }
314 EXPORT_SYMBOL(scst_unregister_target_template);
315
316 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
317         const char *target_name)
318 {
319         struct scst_tgt *tgt;
320         int rc = 0;
321
322         TRACE_ENTRY();
323
324         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
325         if (tgt == NULL) {
326                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of tgt failed");
327                 rc = -ENOMEM;
328                 goto out_err;
329         }
330
331         INIT_LIST_HEAD(&tgt->sess_list);
332         init_waitqueue_head(&tgt->unreg_waitQ);
333         tgt->tgtt = vtt;
334         tgt->sg_tablesize = vtt->sg_tablesize;
335         spin_lock_init(&tgt->tgt_lock);
336         INIT_LIST_HEAD(&tgt->retry_cmd_list);
337         atomic_set(&tgt->finished_cmds, 0);
338         init_timer(&tgt->retry_timer);
339         tgt->retry_timer.data = (unsigned long)tgt;
340         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
341
342         rc = scst_suspend_activity(true);
343         if (rc != 0)
344                 goto out_free_tgt_err;
345
346         if (mutex_lock_interruptible(&scst_mutex) != 0) {
347                 rc = -EINTR;
348                 goto out_resume_free;
349         }
350
351         if (target_name != NULL) {
352                 int len = strlen(target_name) + 1 +
353                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
354
355                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
356                 if (tgt->default_group_name == NULL) {
357                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
358                                 "group name failed");
359                         rc = -ENOMEM;
360                         goto out_unlock_resume;
361                 }
362                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
363                         target_name);
364         }
365
366         rc = scst_build_proc_target_entries(tgt);
367         if (rc < 0)
368                 goto out_free_name;
369         else
370                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
371
372         mutex_unlock(&scst_mutex);
373         scst_resume_activity();
374
375         PRINT_INFO("Target %s (%p) for template %s registered successfully",
376                 target_name, tgt, vtt->name);
377
378 out:
379         TRACE_EXIT();
380         return tgt;
381
382 out_free_name:
383         kfree(tgt->default_group_name);
384
385 out_unlock_resume:
386         mutex_unlock(&scst_mutex);
387
388 out_resume_free:
389         scst_resume_activity();
390
391 out_free_tgt_err:
392         kfree(tgt);
393         tgt = NULL;
394
395 out_err:
396         PRINT_ERROR("Failed to register target %s for template %s (error %d)",
397                 target_name, vtt->name, rc);
398         goto out;
399 }
400 EXPORT_SYMBOL(scst_register);
401
402 static inline int test_sess_list(struct scst_tgt *tgt)
403 {
404         int res;
405         mutex_lock(&scst_mutex);
406         res = list_empty(&tgt->sess_list);
407         mutex_unlock(&scst_mutex);
408         return res;
409 }
410
411 void scst_unregister(struct scst_tgt *tgt)
412 {
413         struct scst_session *sess;
414         struct scst_tgt_template *vtt = tgt->tgtt;
415
416         TRACE_ENTRY();
417
418         TRACE_DBG("%s", "Calling target driver's release()");
419         tgt->tgtt->release(tgt);
420         TRACE_DBG("%s", "Target driver's release() returned");
421
422         mutex_lock(&scst_mutex);
423 again:
424         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
425                 if (sess->shut_phase == SCST_SESS_SPH_READY) {
426                         /*
427                          * Sometimes it's hard for target driver to track all
428                          * its sessions (see scst_local, for example), so let's
429                          * help it.
430                          */
431                         mutex_unlock(&scst_mutex);
432                         scst_unregister_session(sess, 0, NULL);
433                         mutex_lock(&scst_mutex);
434                         goto again;
435                 }
436         }
437         mutex_unlock(&scst_mutex);
438
439         TRACE_DBG("%s", "Waiting for sessions shutdown");
440         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
441         TRACE_DBG("%s", "wait_event() returned");
442
443         scst_suspend_activity(false);
444         mutex_lock(&scst_mutex);
445
446         list_del(&tgt->tgt_list_entry);
447
448         scst_cleanup_proc_target_entries(tgt);
449
450         kfree(tgt->default_group_name);
451
452         mutex_unlock(&scst_mutex);
453         scst_resume_activity();
454
455         del_timer_sync(&tgt->retry_timer);
456
457         PRINT_INFO("Target %p for template %s unregistered successfully",
458                 tgt, vtt->name);
459
460         kfree(tgt);
461
462         TRACE_EXIT();
463         return;
464 }
465 EXPORT_SYMBOL(scst_unregister);
466
467 static int scst_susp_wait(bool interruptible)
468 {
469         int res = 0;
470
471         TRACE_ENTRY();
472
473         if (interruptible) {
474                 res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
475                         (atomic_read(&scst_cmd_count) == 0),
476                         SCST_SUSPENDING_TIMEOUT);
477                 if (res <= 0) {
478                         __scst_resume_activity();
479                         if (res == 0)
480                                 res = -EBUSY;
481                 } else
482                         res = 0;
483         } else
484                 wait_event(scst_dev_cmd_waitQ,
485                            atomic_read(&scst_cmd_count) == 0);
486
487         TRACE_MGMT_DBG("wait_event() returned %d", res);
488
489         TRACE_EXIT_RES(res);
490         return res;
491 }
492
493 int scst_suspend_activity(bool interruptible)
494 {
495         int res = 0;
496         bool rep = false;
497
498         TRACE_ENTRY();
499
500         if (interruptible) {
501                 if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
502                         res = -EINTR;
503                         goto out;
504                 }
505         } else
506                 mutex_lock(&scst_suspend_mutex);
507
508         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
509         suspend_count++;
510         if (suspend_count > 1)
511                 goto out_up;
512
513         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
514         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
515         /*
516          * Assignment of SCST_FLAG_SUSPENDING and SCST_FLAG_SUSPENDED must be
517          * ordered with scst_cmd_count. Otherwise lockless logic in
518          * scst_translate_lun() and scst_mgmt_translate_lun() won't work.
519          */
520         smp_mb__after_set_bit();
521
522         /*
523          * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
524          * information about scst_user behavior.
525          *
526          * ToDo: make the global suspending unneeded (switch to per-device
527          * reference counting? That would mean to switch off from lockless
528          * implementation of scst_translate_lun().. )
529          */
530
531         if (atomic_read(&scst_cmd_count) != 0) {
532                 PRINT_INFO("Waiting for %d active commands to complete... This "
533                         "might take few minutes for disks or few hours for "
534                         "tapes, if you use long executed commands, like "
535                         "REWIND or FORMAT. In case, if you have a hung user "
536                         "space device (i.e. made using scst_user module) not "
537                         "responding to any commands, if might take virtually "
538                         "forever until the corresponding user space "
539                         "program recovers and starts responding or gets "
540                         "killed.", atomic_read(&scst_cmd_count));
541                 rep = true;
542         }
543
544         res = scst_susp_wait(interruptible);
545         if (res != 0)
546                 goto out_clear;
547
548         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
549         /* See comment about smp_mb() above */
550         smp_mb__after_clear_bit();
551
552         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
553                 atomic_read(&scst_cmd_count));
554
555         res = scst_susp_wait(interruptible);
556         if (res != 0)
557                 goto out_clear;
558
559         if (rep)
560                 PRINT_INFO("%s", "All active commands completed");
561
562 out_up:
563         mutex_unlock(&scst_suspend_mutex);
564
565 out:
566         TRACE_EXIT_RES(res);
567         return res;
568
569 out_clear:
570         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
571         /* See comment about smp_mb() above */
572         smp_mb__after_clear_bit();
573         goto out_up;
574 }
575 EXPORT_SYMBOL(scst_suspend_activity);
576
577 static void __scst_resume_activity(void)
578 {
579         struct scst_cmd_lists *l;
580
581         TRACE_ENTRY();
582
583         suspend_count--;
584         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
585         if (suspend_count > 0)
586                 goto out;
587
588         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
589         /*
590          * The barrier is needed to make sure all woken up threads see the
591          * cleared flag. Not sure if it's really needed, but let's be safe.
592          */
593         smp_mb__after_clear_bit();
594
595         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
596                 wake_up_all(&l->cmd_list_waitQ);
597         }
598         wake_up_all(&scst_init_cmd_list_waitQ);
599
600         spin_lock_irq(&scst_mcmd_lock);
601         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
602                 struct scst_mgmt_cmd *m;
603                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
604                                 mgmt_cmd_list_entry);
605                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
606                         "mgmt cmd list", m);
607                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
608         }
609         spin_unlock_irq(&scst_mcmd_lock);
610         wake_up_all(&scst_mgmt_cmd_list_waitQ);
611
612 out:
613         TRACE_EXIT();
614         return;
615 }
616
617 void scst_resume_activity(void)
618 {
619         TRACE_ENTRY();
620
621         mutex_lock(&scst_suspend_mutex);
622         __scst_resume_activity();
623         mutex_unlock(&scst_suspend_mutex);
624
625         TRACE_EXIT();
626         return;
627 }
628 EXPORT_SYMBOL(scst_resume_activity);
629
630 static int scst_register_device(struct scsi_device *scsidp)
631 {
632         int res = 0;
633         struct scst_device *dev;
634         struct scst_dev_type *dt;
635
636         TRACE_ENTRY();
637
638         res = scst_suspend_activity(true);
639         if (res != 0)
640                 goto out_err;
641
642         if (mutex_lock_interruptible(&scst_mutex) != 0) {
643                 res = -EINTR;
644                 goto out_resume;
645         }
646
647         res = scst_alloc_device(GFP_KERNEL, &dev);
648         if (res != 0)
649                 goto out_up;
650
651         dev->type = scsidp->type;
652
653         dev->rq_disk = alloc_disk(1);
654         if (dev->rq_disk == NULL) {
655                 res = -ENOMEM;
656                 goto out_free_dev;
657         }
658         dev->rq_disk->major = SCST_MAJOR;
659
660         dev->scsi_dev = scsidp;
661
662         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
663
664         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
665                 if (dt->type == scsidp->type) {
666                         res = scst_assign_dev_handler(dev, dt);
667                         if (res != 0)
668                                 goto out_free;
669                         break;
670                 }
671         }
672
673 out_up:
674         mutex_unlock(&scst_mutex);
675
676 out_resume:
677         scst_resume_activity();
678
679 out_err:
680         if (res == 0) {
681                 PRINT_INFO("Attached to scsi%d, channel %d, id %d, lun %d, "
682                         "type %d", scsidp->host->host_no, scsidp->channel,
683                         scsidp->id, scsidp->lun, scsidp->type);
684         } else {
685                 PRINT_ERROR("Failed to to scsi%d, channel %d, id %d, lun %d, "
686                         "type %d", scsidp->host->host_no, scsidp->channel,
687                         scsidp->id, scsidp->lun, scsidp->type);
688         }
689
690         TRACE_EXIT_RES(res);
691         return res;
692
693 out_free:
694         list_del(&dev->dev_list_entry);
695         put_disk(dev->rq_disk);
696
697 out_free_dev:
698         scst_free_device(dev);
699         goto out_up;
700 }
701
702 static void scst_unregister_device(struct scsi_device *scsidp)
703 {
704         struct scst_device *d, *dev = NULL;
705         struct scst_acg_dev *acg_dev, *aa;
706
707         TRACE_ENTRY();
708
709         scst_suspend_activity(false);
710         mutex_lock(&scst_mutex);
711
712         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
713                 if (d->scsi_dev == scsidp) {
714                         dev = d;
715                         TRACE_DBG("Target device %p found", dev);
716                         break;
717                 }
718         }
719         if (dev == NULL) {
720                 PRINT_ERROR("%s", "Target device not found");
721                 goto out_unblock;
722         }
723
724         list_del(&dev->dev_list_entry);
725
726         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
727                                  dev_acg_dev_list_entry)
728         {
729                 scst_acg_remove_dev(acg_dev->acg, dev);
730         }
731
732         scst_assign_dev_handler(dev, &scst_null_devtype);
733
734         put_disk(dev->rq_disk);
735         scst_free_device(dev);
736
737         PRINT_INFO("Detached from scsi%d, channel %d, id %d, lun %d, type %d",
738                 scsidp->host->host_no, scsidp->channel, scsidp->id,
739                 scsidp->lun, scsidp->type);
740
741 out_unblock:
742         mutex_unlock(&scst_mutex);
743         scst_resume_activity();
744
745         TRACE_EXIT();
746         return;
747 }
748
749 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
750 {
751         int res = 0;
752
753         if (dev_handler->parse == NULL) {
754                 PRINT_ERROR("scst dev_type driver %s doesn't have a "
755                         "parse() method.", dev_handler->name);
756                 res = -EINVAL;
757                 goto out;
758         }
759
760         if (dev_handler->exec == NULL) {
761 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
762                 dev_handler->exec_atomic = 1;
763 #else
764                 dev_handler->exec_atomic = 0;
765 #endif
766         }
767
768         if (dev_handler->dev_done == NULL)
769                 dev_handler->dev_done_atomic = 1;
770
771 out:
772         TRACE_EXIT_RES(res);
773         return res;
774 }
775
776 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
777         const char *dev_name)
778 {
779         int res, rc;
780         struct scst_device *dev = NULL;
781
782         TRACE_ENTRY();
783
784         if (dev_handler == NULL) {
785                 PRINT_ERROR("%s: valid device handler must be supplied",
786                             __func__);
787                 res = -EINVAL;
788                 goto out;
789         }
790
791         if (dev_name == NULL) {
792                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
793                 res = -EINVAL;
794                 goto out;
795         }
796
797         res = scst_dev_handler_check(dev_handler);
798         if (res != 0)
799                 goto out;
800
801         res = scst_suspend_activity(true);
802         if (res != 0)
803                 goto out;
804
805         if (mutex_lock_interruptible(&scst_mutex) != 0) {
806                 res = -EINTR;
807                 goto out_resume;
808         }
809
810         res = scst_alloc_device(GFP_KERNEL, &dev);
811         if (res != 0)
812                 goto out_up;
813
814         dev->type = dev_handler->type;
815         dev->scsi_dev = NULL;
816         dev->virt_name = dev_name;
817         dev->virt_id = scst_virt_dev_last_id++;
818
819         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
820
821         res = dev->virt_id;
822
823         rc = scst_assign_dev_handler(dev, dev_handler);
824         if (rc != 0) {
825                 res = rc;
826                 goto out_free_del;
827         }
828
829 out_up:
830         mutex_unlock(&scst_mutex);
831
832 out_resume:
833         scst_resume_activity();
834
835 out:
836         if (res > 0)
837                 PRINT_INFO("Attached to virtual device %s (id %d)",
838                         dev_name, dev->virt_id);
839         else
840                 PRINT_INFO("Failed to attach to virtual device %s", dev_name);
841
842         TRACE_EXIT_RES(res);
843         return res;
844
845 out_free_del:
846         list_del(&dev->dev_list_entry);
847         scst_free_device(dev);
848         goto out_up;
849 }
850 EXPORT_SYMBOL(scst_register_virtual_device);
851
852 void scst_unregister_virtual_device(int id)
853 {
854         struct scst_device *d, *dev = NULL;
855         struct scst_acg_dev *acg_dev, *aa;
856
857         TRACE_ENTRY();
858
859         scst_suspend_activity(false);
860         mutex_lock(&scst_mutex);
861
862         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
863                 if (d->virt_id == id) {
864                         dev = d;
865                         TRACE_DBG("Target device %p (id %d) found", dev, id);
866                         break;
867                 }
868         }
869         if (dev == NULL) {
870                 PRINT_ERROR("Target virtual device (id %d) not found", id);
871                 goto out_unblock;
872         }
873
874         list_del(&dev->dev_list_entry);
875
876         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
877                                  dev_acg_dev_list_entry)
878         {
879                 scst_acg_remove_dev(acg_dev->acg, dev);
880         }
881
882         scst_assign_dev_handler(dev, &scst_null_devtype);
883
884         PRINT_INFO("Detached from virtual device %s (id %d)",
885                 dev->virt_name, dev->virt_id);
886
887         scst_free_device(dev);
888
889 out_unblock:
890         mutex_unlock(&scst_mutex);
891         scst_resume_activity();
892
893         TRACE_EXIT();
894         return;
895 }
896 EXPORT_SYMBOL(scst_unregister_virtual_device);
897
898 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
899         const char *version)
900 {
901         struct scst_dev_type *dt;
902         struct scst_device *dev;
903         int res;
904         int exist;
905
906         TRACE_ENTRY();
907
908         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
909                 PRINT_ERROR("Incorrect version of dev handler %s",
910                         dev_type->name);
911                 res = -EINVAL;
912                 goto out_error;
913         }
914
915         res = scst_dev_handler_check(dev_type);
916         if (res != 0)
917                 goto out_error;
918
919 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
920 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && \
921     !defined(CONFIG_SCST_STRICT_SERIALIZING)
922         if (dev_type->exec == NULL) {
923                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
924                         "supported. Consider applying on your kernel patch "
925                         "scst_exec_req_fifo-<kernel-version> or define "
926                         "CONFIG_SCST_STRICT_SERIALIZING", dev_type->name);
927                 res = -EINVAL;
928                 goto out_error;
929         }
930 #endif
931 #endif
932
933         res = scst_suspend_activity(true);
934         if (res != 0)
935                 goto out_error;
936
937         if (mutex_lock_interruptible(&scst_mutex) != 0) {
938                 res = -EINTR;
939                 goto out_err_res;
940         }
941
942         exist = 0;
943         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
944                 if (strcmp(dt->name, dev_type->name) == 0) {
945                         PRINT_ERROR("Device type handler \"%s\" already "
946                                 "exist", dt->name);
947                         exist = 1;
948                         break;
949                 }
950         }
951         if (exist)
952                 goto out_up;
953
954         res = scst_build_proc_dev_handler_dir_entries(dev_type);
955         if (res < 0)
956                 goto out_up;
957
958         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
959
960         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
961                 if (dev->scsi_dev == NULL || dev->handler != &scst_null_devtype)
962                         continue;
963                 if (dev->scsi_dev->type == dev_type->type)
964                         scst_assign_dev_handler(dev, dev_type);
965         }
966
967         mutex_unlock(&scst_mutex);
968         scst_resume_activity();
969
970         if (res == 0) {
971                 PRINT_INFO("Device handler \"%s\" for type %d registered "
972                         "successfully", dev_type->name, dev_type->type);
973         }
974
975 out:
976         TRACE_EXIT_RES(res);
977         return res;
978
979 out_up:
980         mutex_unlock(&scst_mutex);
981
982 out_err_res:
983         scst_resume_activity();
984
985 out_error:
986         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
987                 dev_type->name, dev_type->type);
988         goto out;
989 }
990 EXPORT_SYMBOL(__scst_register_dev_driver);
991
992 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
993 {
994         struct scst_device *dev;
995         struct scst_dev_type *dt;
996         int found = 0;
997
998         TRACE_ENTRY();
999
1000         scst_suspend_activity(false);
1001         mutex_lock(&scst_mutex);
1002
1003         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
1004                 if (strcmp(dt->name, dev_type->name) == 0) {
1005                         found = 1;
1006                         break;
1007                 }
1008         }
1009         if (!found) {
1010                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
1011                         dev_type->name);
1012                 goto out_up;
1013         }
1014
1015         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
1016                 if (dev->handler == dev_type) {
1017                         scst_assign_dev_handler(dev, &scst_null_devtype);
1018                         TRACE_DBG("Dev handler removed from device %p", dev);
1019                 }
1020         }
1021
1022         list_del(&dev_type->dev_type_list_entry);
1023
1024         mutex_unlock(&scst_mutex);
1025         scst_resume_activity();
1026
1027         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1028
1029         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
1030                    dev_type->name, dev_type->type);
1031
1032 out:
1033         TRACE_EXIT();
1034         return;
1035
1036 out_up:
1037         mutex_unlock(&scst_mutex);
1038         scst_resume_activity();
1039         goto out;
1040 }
1041 EXPORT_SYMBOL(scst_unregister_dev_driver);
1042
1043 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
1044         const char *version)
1045 {
1046         int res;
1047
1048         TRACE_ENTRY();
1049
1050         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
1051                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
1052                         dev_type->name);
1053                 res = -EINVAL;
1054                 goto out_err;
1055         }
1056
1057         res = scst_dev_handler_check(dev_type);
1058         if (res != 0)
1059                 goto out_err;
1060
1061         if (!dev_type->no_proc) {
1062                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
1063                 if (res < 0)
1064                         goto out_err;
1065         }
1066
1067         if (dev_type->type != -1) {
1068                 PRINT_INFO("Virtual device handler %s for type %d "
1069                         "registered successfully", dev_type->name,
1070                         dev_type->type);
1071         } else {
1072                 PRINT_INFO("Virtual device handler \"%s\" registered "
1073                         "successfully", dev_type->name);
1074         }
1075
1076 out:
1077         TRACE_EXIT_RES(res);
1078         return res;
1079
1080 out_err:
1081         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
1082                 dev_type->name);
1083         goto out;
1084 }
1085 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
1086
1087 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
1088 {
1089         TRACE_ENTRY();
1090
1091         if (!dev_type->no_proc)
1092                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
1093
1094         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
1095
1096         TRACE_EXIT();
1097         return;
1098 }
1099 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1100
1101 /* Called under scst_mutex */
1102 int scst_add_dev_threads(struct scst_device *dev, int num)
1103 {
1104         int i, res = 0;
1105         int n = 0;
1106         struct scst_cmd_thread_t *thr;
1107         char nm[12];
1108
1109         TRACE_ENTRY();
1110
1111         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
1112                 n++;
1113         }
1114
1115         for (i = 0; i < num; i++) {
1116                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1117                 if (!thr) {
1118                         res = -ENOMEM;
1119                         PRINT_ERROR("Failed to allocate thr %d", res);
1120                         goto out_del;
1121                 }
1122                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
1123                 nm[ARRAY_SIZE(nm)-1] = '\0';
1124                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1125                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
1126                 if (IS_ERR(thr->cmd_thread)) {
1127                         res = PTR_ERR(thr->cmd_thread);
1128                         PRINT_ERROR("kthread_create() failed: %d", res);
1129                         kfree(thr);
1130                         goto out_del;
1131                 }
1132
1133                 list_add(&thr->thread_list_entry, &dev->threads_list);
1134
1135                 /*
1136                  * ToDo: better to use tgt_dev_io_context instead, but we
1137                  * are not ready for that yet.
1138                  */
1139                 __exit_io_context(thr->cmd_thread->io_context);
1140                 thr->cmd_thread->io_context = ioc_task_link(dev->dev_io_ctx);
1141                 TRACE_DBG("Setting dev io ctx %p on thr %d", dev->dev_io_ctx,
1142                         thr->cmd_thread->pid);
1143
1144                 wake_up_process(thr->cmd_thread);
1145         }
1146
1147 out:
1148         TRACE_EXIT_RES(res);
1149         return res;
1150
1151 out_del:
1152         scst_del_dev_threads(dev, i);
1153         goto out;
1154 }
1155
1156 /* Called under scst_mutex and suspended activity */
1157 static int scst_create_dev_threads(struct scst_device *dev)
1158 {
1159         int res = 0;
1160         int threads_num;
1161
1162         TRACE_ENTRY();
1163
1164         if (dev->handler->threads_num <= 0)
1165                 goto out;
1166
1167         threads_num = dev->handler->threads_num;
1168
1169         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1170         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1171         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1172
1173         res = scst_add_dev_threads(dev, threads_num);
1174         if (res != 0)
1175                 goto out;
1176
1177         mutex_lock(&scst_suspend_mutex);
1178         list_add_tail(&dev->cmd_lists.lists_list_entry,
1179                 &scst_cmd_lists_list);
1180         mutex_unlock(&scst_suspend_mutex);
1181
1182         dev->p_cmd_lists = &dev->cmd_lists;
1183
1184 out:
1185         TRACE_EXIT_RES(res);
1186         return res;
1187 }
1188
1189 /* Called under scst_mutex */
1190 void scst_del_dev_threads(struct scst_device *dev, int num)
1191 {
1192         struct scst_cmd_thread_t *ct, *tmp;
1193         int i = 0;
1194
1195         TRACE_ENTRY();
1196
1197         if (num == 0)
1198                 goto out;
1199
1200         list_for_each_entry_safe_reverse(ct, tmp, &dev->threads_list,
1201                                 thread_list_entry) {
1202                 int rc;
1203                 struct scst_tgt_dev *tgt_dev;
1204
1205                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1206                                 dev_tgt_dev_list_entry) {
1207                         struct scst_thr_data_hdr *td;
1208                         td = __scst_find_thr_data(tgt_dev, ct->cmd_thread);
1209                         if (td != NULL) {
1210                                 scst_thr_data_put(td);
1211                                 break;
1212                         }
1213                 }
1214
1215                 rc = kthread_stop(ct->cmd_thread);
1216                 if (rc < 0)
1217                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1218
1219                 list_del(&ct->thread_list_entry);
1220                 kfree(ct);
1221
1222                 if ((num > 0) && (++i >= num))
1223                         break;
1224         }
1225
1226 out:
1227         TRACE_EXIT();
1228         return;
1229 }
1230
1231 /* Called under scst_mutex and suspended activity */
1232 static void scst_stop_dev_threads(struct scst_device *dev)
1233 {
1234         TRACE_ENTRY();
1235
1236         if (list_empty(&dev->threads_list))
1237                 goto out;
1238
1239         scst_del_dev_threads(dev, -1);
1240
1241         if (dev->p_cmd_lists == &dev->cmd_lists) {
1242                 mutex_lock(&scst_suspend_mutex);
1243                 list_del(&dev->cmd_lists.lists_list_entry);
1244                 mutex_unlock(&scst_suspend_mutex);
1245         }
1246
1247 out:
1248         TRACE_EXIT();
1249         return;
1250 }
1251
1252 /* The activity supposed to be suspended and scst_mutex held */
1253 int scst_assign_dev_handler(struct scst_device *dev,
1254         struct scst_dev_type *handler)
1255 {
1256         int res = 0;
1257         struct scst_tgt_dev *tgt_dev;
1258         LIST_HEAD(attached_tgt_devs);
1259
1260         TRACE_ENTRY();
1261
1262         sBUG_ON(handler == NULL);
1263
1264         if (dev->handler == handler)
1265                 goto out;
1266
1267         if (dev->handler && dev->handler->detach_tgt) {
1268                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1269                                 dev_tgt_dev_list_entry) {
1270                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1271                                 tgt_dev);
1272                         dev->handler->detach_tgt(tgt_dev);
1273                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1274                 }
1275         }
1276
1277         if (dev->handler && dev->handler->detach) {
1278                 TRACE_DBG("%s", "Calling dev handler's detach()");
1279                 dev->handler->detach(dev);
1280                 TRACE_DBG("%s", "Old handler's detach() returned");
1281         }
1282
1283         scst_stop_dev_threads(dev);
1284
1285         dev->handler = handler;
1286
1287         if (handler) {
1288                 res = scst_create_dev_threads(dev);
1289                 if (res != 0)
1290                         goto out_null;
1291         }
1292
1293         if (handler && handler->attach) {
1294                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1295                 res = handler->attach(dev);
1296                 TRACE_DBG("New dev handler's attach() returned %d", res);
1297                 if (res != 0) {
1298                         PRINT_ERROR("New device handler's %s attach() "
1299                                 "failed: %d", handler->name, res);
1300                 }
1301                 goto out_thr_null;
1302         }
1303
1304         if (handler && handler->attach_tgt) {
1305                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1306                                 dev_tgt_dev_list_entry) {
1307                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1308                                 tgt_dev);
1309                         res = handler->attach_tgt(tgt_dev);
1310                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1311                         if (res != 0) {
1312                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1313                                     "failed: %d", handler->name, res);
1314                                 goto out_err_detach_tgt;
1315                         }
1316                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1317                                 &attached_tgt_devs);
1318                 }
1319         }
1320
1321 out_thr_null:
1322         if (res != 0)
1323                 scst_stop_dev_threads(dev);
1324
1325 out_null:
1326         if (res != 0)
1327                 dev->handler = &scst_null_devtype;
1328
1329 out:
1330         TRACE_EXIT_RES(res);
1331         return res;
1332
1333 out_err_detach_tgt:
1334         if (handler && handler->detach_tgt) {
1335                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1336                                  extra_tgt_dev_list_entry)
1337                 {
1338                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1339                                 tgt_dev);
1340                         handler->detach_tgt(tgt_dev);
1341                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1342                 }
1343         }
1344         if (handler && handler->detach) {
1345                 TRACE_DBG("%s", "Calling handler's detach()");
1346                 handler->detach(dev);
1347                 TRACE_DBG("%s", "Handler's detach() returned");
1348         }
1349         goto out_null;
1350 }
1351
1352 int scst_global_threads_count(void)
1353 {
1354         int i;
1355
1356         /*
1357          * Just to lower the race window, when user can get just changed value
1358          */
1359         mutex_lock(&scst_global_threads_mutex);
1360         i = scst_nr_global_threads;
1361         mutex_unlock(&scst_global_threads_mutex);
1362         return i;
1363 }
1364
1365 static void scst_threads_info_init(void)
1366 {
1367         mutex_init(&scst_global_threads_mutex);
1368         INIT_LIST_HEAD(&scst_global_threads_list);
1369 }
1370
1371 /* scst_global_threads_mutex supposed to be held */
1372 void __scst_del_global_threads(int num)
1373 {
1374         struct scst_cmd_thread_t *ct, *tmp;
1375
1376         TRACE_ENTRY();
1377
1378         if (num == 0)
1379                 goto out;
1380
1381         list_for_each_entry_safe(ct, tmp, &scst_global_threads_list,
1382                                 thread_list_entry) {
1383                 int res;
1384
1385                 res = kthread_stop(ct->cmd_thread);
1386                 if (res < 0)
1387                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1388                 list_del(&ct->thread_list_entry);
1389                 kfree(ct);
1390                 scst_nr_global_threads--;
1391                 --num;
1392                 if (num == 0)
1393                         break;
1394         }
1395
1396 out:
1397         TRACE_EXIT();
1398         return;
1399 }
1400
1401 /* scst_global_threads_mutex supposed to be held */
1402 int __scst_add_global_threads(int num)
1403 {
1404         int res = 0, i;
1405         static int scst_thread_num;
1406
1407         TRACE_ENTRY();
1408
1409         for (i = 0; i < num; i++) {
1410                 struct scst_cmd_thread_t *thr;
1411
1412                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1413                 if (!thr) {
1414                         res = -ENOMEM;
1415                         PRINT_ERROR("fail to allocate thr %d", res);
1416                         goto out_error;
1417                 }
1418                 thr->cmd_thread = kthread_create(scst_cmd_thread,
1419                         &scst_main_cmd_lists, "scsi_tgt%d",
1420                         scst_thread_num++);
1421                 if (IS_ERR(thr->cmd_thread)) {
1422                         res = PTR_ERR(thr->cmd_thread);
1423                         PRINT_ERROR("kthread_create() failed: %d", res);
1424                         kfree(thr);
1425                         goto out_error;
1426                 }
1427
1428                 list_add(&thr->thread_list_entry, &scst_global_threads_list);
1429                 scst_nr_global_threads++;
1430
1431                 wake_up_process(thr->cmd_thread);
1432         }
1433         res = 0;
1434
1435 out:
1436         TRACE_EXIT_RES(res);
1437         return res;
1438
1439 out_error:
1440         __scst_del_global_threads(i);
1441         goto out;
1442 }
1443
1444 int scst_add_global_threads(int num)
1445 {
1446         int res;
1447
1448         TRACE_ENTRY();
1449
1450         mutex_lock(&scst_global_threads_mutex);
1451         res = __scst_add_global_threads(num);
1452         mutex_unlock(&scst_global_threads_mutex);
1453
1454         TRACE_EXIT_RES(res);
1455         return res;
1456 }
1457 EXPORT_SYMBOL(scst_add_global_threads);
1458
1459 void scst_del_global_threads(int num)
1460 {
1461         TRACE_ENTRY();
1462
1463         mutex_lock(&scst_global_threads_mutex);
1464         __scst_del_global_threads(num);
1465         mutex_unlock(&scst_global_threads_mutex);
1466
1467         TRACE_EXIT();
1468         return;
1469 }
1470 EXPORT_SYMBOL(scst_del_global_threads);
1471
1472 static void scst_stop_all_threads(void)
1473 {
1474         TRACE_ENTRY();
1475
1476         mutex_lock(&scst_global_threads_mutex);
1477         __scst_del_global_threads(-1);
1478         if (scst_mgmt_cmd_thread)
1479                 kthread_stop(scst_mgmt_cmd_thread);
1480         if (scst_mgmt_thread)
1481                 kthread_stop(scst_mgmt_thread);
1482         if (scst_init_cmd_thread)
1483                 kthread_stop(scst_init_cmd_thread);
1484         mutex_unlock(&scst_global_threads_mutex);
1485
1486         TRACE_EXIT();
1487         return;
1488 }
1489
1490 static int scst_start_all_threads(int num)
1491 {
1492         int res;
1493
1494         TRACE_ENTRY();
1495
1496         mutex_lock(&scst_global_threads_mutex);
1497         res = __scst_add_global_threads(num);
1498         if (res < 0)
1499                 goto out;
1500
1501         scst_init_cmd_thread = kthread_run(scst_init_thread,
1502                 NULL, "scsi_tgt_init");
1503         if (IS_ERR(scst_init_cmd_thread)) {
1504                 res = PTR_ERR(scst_init_cmd_thread);
1505                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1506                 scst_init_cmd_thread = NULL;
1507                 goto out;
1508         }
1509
1510         scst_mgmt_cmd_thread = kthread_run(scst_tm_thread,
1511                 NULL, "scsi_tm");
1512         if (IS_ERR(scst_mgmt_cmd_thread)) {
1513                 res = PTR_ERR(scst_mgmt_cmd_thread);
1514                 PRINT_ERROR("kthread_create() for TM failed: %d", res);
1515                 scst_mgmt_cmd_thread = NULL;
1516                 goto out;
1517         }
1518
1519         scst_mgmt_thread = kthread_run(scst_global_mgmt_thread,
1520                 NULL, "scsi_tgt_mgmt");
1521         if (IS_ERR(scst_mgmt_thread)) {
1522                 res = PTR_ERR(scst_mgmt_thread);
1523                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1524                 scst_mgmt_thread = NULL;
1525                 goto out;
1526         }
1527
1528 out:
1529         mutex_unlock(&scst_global_threads_mutex);
1530         TRACE_EXIT_RES(res);
1531         return res;
1532 }
1533
1534 void scst_get(void)
1535 {
1536         __scst_get(0);
1537 }
1538 EXPORT_SYMBOL(scst_get);
1539
1540 void scst_put(void)
1541 {
1542         __scst_put();
1543 }
1544 EXPORT_SYMBOL(scst_put);
1545
1546 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1547 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1548 #else
1549 static int scst_add(struct device *cdev, struct class_interface *intf)
1550 #endif
1551 {
1552         struct scsi_device *scsidp;
1553         int res = 0;
1554
1555         TRACE_ENTRY();
1556
1557 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1558         scsidp = to_scsi_device(cdev->dev);
1559 #else
1560         scsidp = to_scsi_device(cdev->parent);
1561 #endif
1562
1563         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1564                 res = scst_register_device(scsidp);
1565
1566         TRACE_EXIT();
1567         return res;
1568 }
1569
1570 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1571 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1572 #else
1573 static void scst_remove(struct device *cdev, struct class_interface *intf)
1574 #endif
1575 {
1576         struct scsi_device *scsidp;
1577
1578         TRACE_ENTRY();
1579
1580 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1581         scsidp = to_scsi_device(cdev->dev);
1582 #else
1583         scsidp = to_scsi_device(cdev->parent);
1584 #endif
1585
1586         if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
1587                 scst_unregister_device(scsidp);
1588
1589         TRACE_EXIT();
1590         return;
1591 }
1592
1593 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
1594 static struct class_interface scst_interface = {
1595         .add = scst_add,
1596         .remove = scst_remove,
1597 };
1598 #else
1599 static struct class_interface scst_interface = {
1600         .add_dev = scst_add,
1601         .remove_dev = scst_remove,
1602 };
1603 #endif
1604
1605 static void __init scst_print_config(void)
1606 {
1607         char buf[128];
1608         int i, j;
1609
1610         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1611         j = i;
1612
1613 #ifdef CONFIG_SCST_STRICT_SERIALIZING
1614         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1615 #endif
1616
1617 #ifdef CONFIG_SCST_EXTRACHECKS
1618         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1619                 (j == i) ? "" : ", ");
1620 #endif
1621
1622 #ifdef CONFIG_SCST_TRACING
1623         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1624                 (j == i) ? "" : ", ");
1625 #endif
1626
1627 #ifdef CONFIG_SCST_DEBUG
1628         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1629                 (j == i) ? "" : ", ");
1630 #endif
1631
1632 #ifdef CONFIG_SCST_DEBUG_TM
1633         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1634                 (j == i) ? "" : ", ");
1635 #endif
1636
1637 #ifdef CONFIG_SCST_DEBUG_RETRY
1638         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1639                 (j == i) ? "" : ", ");
1640 #endif
1641
1642 #ifdef CONFIG_SCST_DEBUG_OOM
1643         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1644                 (j == i) ? "" : ", ");
1645 #endif
1646
1647 #ifdef CONFIG_SCST_DEBUG_SN
1648         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1649                 (j == i) ? "" : ", ");
1650 #endif
1651
1652 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
1653         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1654                 (j == i) ? "" : ", ");
1655 #endif
1656
1657 #ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1658         i += snprintf(&buf[i], sizeof(buf) - i,
1659                 "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1660                 (j == i) ? "" : ", ");
1661 #endif
1662
1663 #ifdef CONFIG_SCST_STRICT_SECURITY
1664         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1665                 (j == i) ? "" : ", ");
1666 #endif
1667
1668         if (j != i)
1669                 PRINT_INFO("%s", buf);
1670 }
1671
1672 static int __init init_scst(void)
1673 {
1674         int res = 0, i;
1675         int scst_num_cpus;
1676
1677         TRACE_ENTRY();
1678
1679 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1680         {
1681                 struct scsi_request *req;
1682                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1683                         sizeof(req->sr_sense_buffer));
1684         }
1685 #else
1686         {
1687                 struct scsi_sense_hdr *shdr;
1688                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1689         }
1690 #endif
1691         {
1692                 struct scst_tgt_dev *t;
1693                 struct scst_cmd *c;
1694                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1695                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1696         }
1697
1698 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
1699 #if !defined(SCST_IO_CONTEXT)
1700         PRINT_WARNING("%s", "Patch io_context was not applied on "
1701                 "your kernel. SCST will be working with not the best "
1702                 "performance.");
1703 #endif
1704 #else
1705         PRINT_WARNING("%s", "There is no patch io_context for your kernel "
1706                 "version. For performance reasons it is strongly recommended "
1707                 "to upgrade your kernel to version >= 2.6.27.x.");
1708 #endif
1709
1710         mutex_init(&scst_mutex);
1711         INIT_LIST_HEAD(&scst_template_list);
1712         INIT_LIST_HEAD(&scst_dev_list);
1713         INIT_LIST_HEAD(&scst_dev_type_list);
1714         spin_lock_init(&scst_main_lock);
1715         INIT_LIST_HEAD(&scst_acg_list);
1716         spin_lock_init(&scst_init_lock);
1717         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1718         INIT_LIST_HEAD(&scst_init_cmd_list);
1719 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1720         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1721 #endif
1722         atomic_set(&scst_cmd_count, 0);
1723         spin_lock_init(&scst_mcmd_lock);
1724         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1725         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1726         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1727         init_waitqueue_head(&scst_mgmt_waitQ);
1728         spin_lock_init(&scst_mgmt_lock);
1729         INIT_LIST_HEAD(&scst_sess_init_list);
1730         INIT_LIST_HEAD(&scst_sess_shut_list);
1731         init_waitqueue_head(&scst_dev_cmd_waitQ);
1732         mutex_init(&scst_suspend_mutex);
1733         INIT_LIST_HEAD(&scst_cmd_lists_list);
1734         scst_virt_dev_last_id = 1;
1735         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1736         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1737         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1738         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1739                 &scst_cmd_lists_list);
1740
1741         scst_num_cpus = num_online_cpus();
1742
1743         /* ToDo: register_cpu_notifier() */
1744
1745         if (scst_threads == 0)
1746                 scst_threads = scst_num_cpus;
1747
1748         if (scst_threads < 1) {
1749                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1750                 scst_threads = scst_num_cpus;
1751         }
1752
1753         scst_threads_info_init();
1754
1755 #define INIT_CACHEP(p, s, o) do {                                       \
1756                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1757                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1758                           sizeof(struct s));                            \
1759                 if (p == NULL) {                                        \
1760                         res = -ENOMEM;                                  \
1761                         goto o;                                         \
1762                 }                                                       \
1763         } while (0)
1764
1765         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out);
1766         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1767                         out_destroy_mgmt_cache);
1768         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1769                         out_destroy_mgmt_stub_cache);
1770         {
1771                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1772                 INIT_CACHEP(scst_sense_cachep, scst_sense,
1773                             out_destroy_ua_cache);
1774         }
1775         INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
1776         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
1777         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1778         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1779         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1780
1781         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1782                 mempool_free_slab, scst_mgmt_cachep);
1783         if (scst_mgmt_mempool == NULL) {
1784                 res = -ENOMEM;
1785                 goto out_destroy_acg_cache;
1786         }
1787
1788         /*
1789          * All mgmt stubs, UAs and sense buffers are bursty and loosing them
1790          * may have fatal consequences, so let's have big pools for them.
1791          */
1792
1793         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1794                 mempool_free_slab, scst_mgmt_stub_cachep);
1795         if (scst_mgmt_stub_mempool == NULL) {
1796                 res = -ENOMEM;
1797                 goto out_destroy_mgmt_mempool;
1798         }
1799
1800         scst_ua_mempool = mempool_create(512, mempool_alloc_slab,
1801                 mempool_free_slab, scst_ua_cachep);
1802         if (scst_ua_mempool == NULL) {
1803                 res = -ENOMEM;
1804                 goto out_destroy_mgmt_stub_mempool;
1805         }
1806
1807         scst_sense_mempool = mempool_create(1024, mempool_alloc_slab,
1808                 mempool_free_slab, scst_sense_cachep);
1809         if (scst_sense_mempool == NULL) {
1810                 res = -ENOMEM;
1811                 goto out_destroy_ua_mempool;
1812         }
1813
1814         scst_aen_mempool = mempool_create(100, mempool_alloc_slab,
1815                 mempool_free_slab, scst_aen_cachep);
1816         if (scst_aen_mempool == NULL) {
1817                 res = -ENOMEM;
1818                 goto out_destroy_sense_mempool;
1819         }
1820
1821         if (scst_max_cmd_mem == 0) {
1822                 struct sysinfo si;
1823                 si_meminfo(&si);
1824 #if BITS_PER_LONG == 32
1825                 scst_max_cmd_mem = min(
1826                         (((uint64_t)(si.totalram - si.totalhigh) << PAGE_SHIFT)
1827                                 >> 20) >> 2, (uint64_t)1 << 30);
1828 #else
1829                 scst_max_cmd_mem = (((si.totalram - si.totalhigh) << PAGE_SHIFT)
1830                                         >> 20) >> 2;
1831 #endif
1832         }
1833
1834         if (scst_max_dev_cmd_mem != 0) {
1835                 if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
1836                         PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
1837                                 "scst_max_cmd_mem (%d)",
1838                                 scst_max_dev_cmd_mem,
1839                                 scst_max_cmd_mem);
1840                         scst_max_dev_cmd_mem = scst_max_cmd_mem;
1841                 }
1842         } else
1843                 scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
1844
1845         res = scst_sgv_pools_init(
1846                 ((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
1847         if (res != 0)
1848                 goto out_destroy_aen_mempool;
1849
1850         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1851         if (scst_default_acg == NULL) {
1852                 res = -ENOMEM;
1853                 goto out_destroy_sgv_pool;
1854         }
1855
1856         res = scsi_register_interface(&scst_interface);
1857         if (res != 0)
1858                 goto out_free_acg;
1859
1860         scst_scsi_op_list_init();
1861
1862         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1863                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1864                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1865                 tasklet_init(&scst_tasklets[i].tasklet,
1866                              (void *)scst_cmd_tasklet,
1867                              (unsigned long)&scst_tasklets[i]);
1868         }
1869
1870         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1871                 scst_threads);
1872
1873         res = scst_start_all_threads(scst_threads);
1874         if (res < 0)
1875                 goto out_thread_free;
1876
1877         res = scst_proc_init_module();
1878         if (res != 0)
1879                 goto out_thread_free;
1880
1881
1882         PRINT_INFO("SCST version %s loaded successfully (max mem for "
1883                 "commands %dMB, per device %dMB)", SCST_VERSION_STRING,
1884                 scst_max_cmd_mem, scst_max_dev_cmd_mem);
1885
1886         scst_print_config();
1887
1888 out:
1889         TRACE_EXIT_RES(res);
1890         return res;
1891
1892 out_thread_free:
1893         scst_stop_all_threads();
1894
1895         scsi_unregister_interface(&scst_interface);
1896
1897 out_free_acg:
1898         scst_destroy_acg(scst_default_acg);
1899
1900 out_destroy_sgv_pool:
1901         scst_sgv_pools_deinit();
1902
1903 out_destroy_aen_mempool:
1904         mempool_destroy(scst_aen_mempool);
1905
1906 out_destroy_sense_mempool:
1907         mempool_destroy(scst_sense_mempool);
1908
1909 out_destroy_ua_mempool:
1910         mempool_destroy(scst_ua_mempool);
1911
1912 out_destroy_mgmt_stub_mempool:
1913         mempool_destroy(scst_mgmt_stub_mempool);
1914
1915 out_destroy_mgmt_mempool:
1916         mempool_destroy(scst_mgmt_mempool);
1917
1918 out_destroy_acg_cache:
1919         kmem_cache_destroy(scst_acgd_cachep);
1920
1921 out_destroy_tgt_cache:
1922         kmem_cache_destroy(scst_tgtd_cachep);
1923
1924 out_destroy_sess_cache:
1925         kmem_cache_destroy(scst_sess_cachep);
1926
1927 out_destroy_cmd_cache:
1928         kmem_cache_destroy(scst_cmd_cachep);
1929
1930 out_destroy_aen_cache:
1931         kmem_cache_destroy(scst_aen_cachep);
1932
1933 out_destroy_sense_cache:
1934         kmem_cache_destroy(scst_sense_cachep);
1935
1936 out_destroy_ua_cache:
1937         kmem_cache_destroy(scst_ua_cachep);
1938
1939 out_destroy_mgmt_stub_cache:
1940         kmem_cache_destroy(scst_mgmt_stub_cachep);
1941
1942 out_destroy_mgmt_cache:
1943         kmem_cache_destroy(scst_mgmt_cachep);
1944         goto out;
1945 }
1946
1947 static void __exit exit_scst(void)
1948 {
1949         TRACE_ENTRY();
1950
1951         /* ToDo: unregister_cpu_notifier() */
1952
1953         scst_proc_cleanup_module();
1954
1955         scst_stop_all_threads();
1956
1957         scsi_unregister_interface(&scst_interface);
1958         scst_destroy_acg(scst_default_acg);
1959
1960         scst_sgv_pools_deinit();
1961
1962 #define DEINIT_CACHEP(p) do {           \
1963                 kmem_cache_destroy(p);  \
1964                 p = NULL;               \
1965         } while (0)
1966
1967         mempool_destroy(scst_mgmt_mempool);
1968         mempool_destroy(scst_mgmt_stub_mempool);
1969         mempool_destroy(scst_ua_mempool);
1970         mempool_destroy(scst_sense_mempool);
1971         mempool_destroy(scst_aen_mempool);
1972
1973         DEINIT_CACHEP(scst_mgmt_cachep);
1974         DEINIT_CACHEP(scst_mgmt_stub_cachep);
1975         DEINIT_CACHEP(scst_ua_cachep);
1976         DEINIT_CACHEP(scst_sense_cachep);
1977         DEINIT_CACHEP(scst_aen_cachep);
1978         DEINIT_CACHEP(scst_cmd_cachep);
1979         DEINIT_CACHEP(scst_sess_cachep);
1980         DEINIT_CACHEP(scst_tgtd_cachep);
1981         DEINIT_CACHEP(scst_acgd_cachep);
1982
1983         PRINT_INFO("%s", "SCST unloaded");
1984
1985         TRACE_EXIT();
1986         return;
1987 }
1988
1989
1990 module_init(init_scst);
1991 module_exit(exit_scst);
1992
1993 MODULE_AUTHOR("Vladislav Bolkhovitin");
1994 MODULE_LICENSE("GPL");
1995 MODULE_DESCRIPTION("SCSI target core");
1996 MODULE_VERSION(SCST_VERSION_STRING);