- Fixes 2 shutdown problems
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <asm/unistd.h>
28 #include <asm/string.h>
29 #include <linux/kthread.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
36 #warning HIGHMEM kernel configurations are fully supported, but not \
37         recommended for performance reasons. Consider change VMSPLIT \
38         option or use 64-bit configuration instead. See README file for \
39         details.
40 #endif
41
42 #ifdef SCST_HIGHMEM
43 #error SCST_HIGHMEM configuration isn't supported and broken, because there \
44         is no real point to support it, at least it definitely doesn't worth \
45         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
46         or in 64-bit configuration instead. See README file for details.
47 #endif
48
49 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
50 #warning Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
51         your kernel and STRICT_SERIALIZING isn't defined. Pass-through dev \
52         handlers will not be supported.
53 #endif
54
55 /*
56  * All targets, devices and dev_types management is done under this mutex.
57  *
58  * It must NOT be used in any works (schedule_work(), etc.), because
59  * otherwise a deadlock (double lock, actually) is possible, e.g., with
60  * scst_user detach_tgt(), which is called under scst_mutex and calls
61  * flush_scheduled_work().
62  */
63 DEFINE_MUTEX(scst_mutex);
64
65 LIST_HEAD(scst_template_list);
66 LIST_HEAD(scst_dev_list);
67 LIST_HEAD(scst_dev_type_list);
68
69 spinlock_t scst_main_lock = SPIN_LOCK_UNLOCKED;
70
71 struct kmem_cache *scst_mgmt_cachep;
72 mempool_t *scst_mgmt_mempool;
73 struct kmem_cache *scst_ua_cachep;
74 mempool_t *scst_ua_mempool;
75 struct kmem_cache *scst_tgtd_cachep;
76 struct kmem_cache *scst_sess_cachep;
77 struct kmem_cache *scst_acgd_cachep;
78
79 LIST_HEAD(scst_acg_list);
80 struct scst_acg *scst_default_acg;
81
82 spinlock_t scst_init_lock = SPIN_LOCK_UNLOCKED;
83 DECLARE_WAIT_QUEUE_HEAD(scst_init_cmd_list_waitQ);
84 LIST_HEAD(scst_init_cmd_list);
85 unsigned int scst_init_poll_cnt;
86
87 struct kmem_cache *scst_cmd_cachep;
88
89 #if defined(DEBUG) || defined(TRACING)
90 unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
91 #endif
92
93 unsigned long scst_flags;
94 atomic_t scst_cmd_count = ATOMIC_INIT(0);
95
96 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
97 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
98 unsigned long scst_max_cmd_mem;
99
100 struct scst_cmd_lists scst_main_cmd_lists;
101
102 struct scst_tasklet scst_tasklets[NR_CPUS];
103
104
105 spinlock_t scst_mcmd_lock = SPIN_LOCK_UNLOCKED;
106 LIST_HEAD(scst_active_mgmt_cmd_list);
107 LIST_HEAD(scst_delayed_mgmt_cmd_list);
108 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
109
110 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
111 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
112 LIST_HEAD(scst_sess_init_list);
113 LIST_HEAD(scst_sess_shut_list);
114
115 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
116
117 DEFINE_MUTEX(scst_suspend_mutex);
118 LIST_HEAD(scst_cmd_lists_list); /* protected by scst_suspend_mutex */
119
120 static int scst_threads;
121 struct scst_threads_info_t scst_threads_info;
122
123 static int suspend_count;
124
125 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
126
127 /* 
128  * This buffer and lock are intended to avoid memory allocation, which
129  * could fail in improper places.
130  */
131 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
132 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
133
134 module_param_named(scst_threads, scst_threads, int, 0);
135 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
136
137 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
138 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
139         "the SCST commands at any given time in Mb");
140
141 int scst_register_target_template(struct scst_tgt_template *vtt)
142 {
143         int res = 0;
144         struct scst_tgt_template *t;
145         static DEFINE_MUTEX(m);
146
147         TRACE_ENTRY();
148
149         INIT_LIST_HEAD(&vtt->tgt_list);
150
151         if (!vtt->detect) {
152                 PRINT_ERROR_PR("Target driver %s doesn't have a "
153                         "detect() method.", vtt->name);
154                 res = -EINVAL;
155                 goto out_err;
156         }
157         
158         if (!vtt->release) {
159                 PRINT_ERROR_PR("Target driver %s doesn't have a "
160                         "release() method.", vtt->name);
161                 res = -EINVAL;
162                 goto out_err;
163         }
164
165         if (!vtt->xmit_response) {
166                 PRINT_ERROR_PR("Target driver %s doesn't have a "
167                         "xmit_response() method.", vtt->name);
168                 res = -EINVAL;
169                 goto out_err;
170         }
171
172         if (vtt->threads_num < 0) {
173                 PRINT_ERROR_PR("Wrong threads_num value %d for "
174                         "target \"%s\"", vtt->threads_num,
175                         vtt->name);
176                 res = -EINVAL;
177                 goto out_err;
178         }
179
180         if (!vtt->no_proc_entry) {
181                 res = scst_build_proc_target_dir_entries(vtt);
182                 if (res < 0)
183                         goto out_err;
184         }
185
186         if (vtt->preprocessing_done == NULL)
187                 vtt->preprocessing_done_atomic = 1;
188
189         if (mutex_lock_interruptible(&m) != 0)
190                 goto out_err;
191
192         if (mutex_lock_interruptible(&scst_mutex) != 0)
193                 goto out_m_up;
194         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
195                 if (strcmp(t->name, vtt->name) == 0) {
196                         PRINT_ERROR_PR("Target driver %s already registered",
197                                 vtt->name);
198                         mutex_unlock(&scst_mutex);
199                         goto out_cleanup;
200                 }
201         }
202         mutex_unlock(&scst_mutex);
203
204         TRACE_DBG("%s", "Calling target driver's detect()");
205         res = vtt->detect(vtt);
206         TRACE_DBG("Target driver's detect() returned %d", res);
207         if (res < 0) {
208                 PRINT_ERROR_PR("%s", "The detect() routine failed");
209                 res = -EINVAL;
210                 goto out_cleanup;
211         }
212
213         mutex_lock(&scst_mutex);
214         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
215         mutex_unlock(&scst_mutex);
216
217         res = 0;
218
219         PRINT_INFO_PR("Target template %s registered successfully", vtt->name);
220
221         mutex_unlock(&m);
222
223 out:
224         TRACE_EXIT_RES(res);
225         return res;
226
227 out_m_up:
228         mutex_unlock(&m);
229
230 out_cleanup:
231         scst_cleanup_proc_target_dir_entries(vtt);
232
233 out_err:
234         PRINT_ERROR_PR("Failed to register target template %s", vtt->name);
235         goto out;
236 }
237
238 void scst_unregister_target_template(struct scst_tgt_template *vtt)
239 {
240         struct scst_tgt *tgt;
241         struct scst_tgt_template *t;
242         int found = 0;
243
244         TRACE_ENTRY();
245
246         mutex_lock(&scst_mutex);
247
248         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
249                 if (strcmp(t->name, vtt->name) == 0) {
250                         found = 1;
251                         break;
252                 }
253         }
254         if (!found) {
255                 PRINT_ERROR_PR("Target driver %s isn't registered", vtt->name);
256                 goto out_up;
257         }
258
259 restart:
260         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
261                 mutex_unlock(&scst_mutex);
262                 scst_unregister(tgt);
263                 mutex_lock(&scst_mutex);
264                 goto restart;
265         }
266         list_del(&vtt->scst_template_list_entry);
267
268         PRINT_INFO_PR("Target template %s unregistered successfully", vtt->name);
269
270 out_up:
271         mutex_unlock(&scst_mutex);
272
273         scst_cleanup_proc_target_dir_entries(vtt);
274
275         TRACE_EXIT();
276         return;
277 }
278
279 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
280         const char *target_name)
281 {
282         struct scst_tgt *tgt;
283
284         TRACE_ENTRY();
285
286         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
287         if (tgt == NULL) {
288                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
289                 goto out_err;
290         }
291
292         INIT_LIST_HEAD(&tgt->sess_list);
293         init_waitqueue_head(&tgt->unreg_waitQ);
294         tgt->tgtt = vtt;
295         tgt->sg_tablesize = vtt->sg_tablesize;
296         spin_lock_init(&tgt->tgt_lock);
297         INIT_LIST_HEAD(&tgt->retry_cmd_list);
298         atomic_set(&tgt->finished_cmds, 0);
299         init_timer(&tgt->retry_timer);
300         tgt->retry_timer.data = (unsigned long)tgt;
301         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
302
303         scst_suspend_activity();
304         mutex_lock(&scst_mutex);
305
306         if (target_name != NULL) {
307                 int len = strlen(target_name) + 1 +
308                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
309
310                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
311                 if (tgt->default_group_name == NULL) {
312                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
313                                 "group name failed");
314                         goto out_free_err;
315                 }
316                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
317                         target_name);
318         }
319
320         if (scst_build_proc_target_entries(tgt) < 0)
321                 goto out_free_name;
322         else
323                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
324
325         mutex_unlock(&scst_mutex);
326         scst_resume_activity();
327
328         PRINT_INFO_PR("Target %s (%p) for template %s registered successfully",
329                 target_name, tgt, vtt->name);
330
331 out:
332         TRACE_EXIT();
333         return tgt;
334
335 out_free_name:
336         if (tgt->default_group_name)
337                 kfree(tgt->default_group_name);
338
339 out_free_err:
340         mutex_unlock(&scst_mutex);
341         scst_resume_activity();
342
343         kfree(tgt);
344         tgt = NULL;
345
346 out_err:
347         PRINT_ERROR_PR("Failed to register target %s for template %s",
348                 target_name, vtt->name);
349         goto out;
350 }
351
352 static inline int test_sess_list(struct scst_tgt *tgt)
353 {
354         int res;
355         mutex_lock(&scst_mutex);
356         res = list_empty(&tgt->sess_list);
357         mutex_unlock(&scst_mutex);
358         return res;
359 }
360
361 void scst_unregister(struct scst_tgt *tgt)
362 {
363         struct scst_session *sess;
364         struct scst_tgt_template *vtt = tgt->tgtt;
365
366         TRACE_ENTRY();
367
368         TRACE_DBG("%s", "Calling target driver's release()");
369         tgt->tgtt->release(tgt);
370         TRACE_DBG("%s", "Target driver's release() returned");
371
372         mutex_lock(&scst_mutex);
373         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
374                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
375         }
376         mutex_unlock(&scst_mutex);
377
378         TRACE_DBG("%s", "Waiting for sessions shutdown");
379         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
380         TRACE_DBG("%s", "wait_event() returned");
381
382         scst_suspend_activity();
383         mutex_lock(&scst_mutex);
384
385         list_del(&tgt->tgt_list_entry);
386
387         scst_cleanup_proc_target_entries(tgt);
388
389         if (tgt->default_group_name)
390                 kfree(tgt->default_group_name);
391
392         mutex_unlock(&scst_mutex);
393         scst_resume_activity();
394
395         del_timer_sync(&tgt->retry_timer);
396
397         PRINT_INFO_PR("Target %p for template %s unregistered successfully",
398                 tgt, vtt->name);
399
400         kfree(tgt);
401
402         TRACE_EXIT();
403         return;
404 }
405
406 void scst_suspend_activity(void)
407 {
408         TRACE_ENTRY();
409
410         mutex_lock(&scst_suspend_mutex);
411
412         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
413         suspend_count++;
414         if (suspend_count > 1)
415                 goto out_up;
416
417         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
418         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
419         smp_mb__after_set_bit();
420
421         TRACE_MGMT_DBG("Waiting for %d active commands to complete",
422               atomic_read(&scst_cmd_count));
423         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
424         TRACE_MGMT_DBG("%s", "wait_event() returned");
425
426         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
427         smp_mb__after_clear_bit();
428
429         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
430               atomic_read(&scst_cmd_count));
431         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
432         TRACE_MGMT_DBG("%s", "wait_event() returned");
433
434 out_up:
435         mutex_unlock(&scst_suspend_mutex);
436
437         TRACE_EXIT();
438         return;
439 }
440
441 void scst_resume_activity(void)
442 {
443         struct scst_cmd_lists *l;
444
445         TRACE_ENTRY();
446
447         mutex_lock(&scst_suspend_mutex);
448
449         suspend_count--;
450         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
451         if (suspend_count > 0)
452                 goto out_up;
453
454         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
455         smp_mb__after_clear_bit();
456
457         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
458                 wake_up_all(&l->cmd_list_waitQ);
459         }
460         wake_up_all(&scst_init_cmd_list_waitQ);
461
462         spin_lock_irq(&scst_mcmd_lock);
463         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
464                 struct scst_mgmt_cmd *m;
465                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
466                                 mgmt_cmd_list_entry);
467                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
468                         "mgmt cmd list", m);
469                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
470         }
471         spin_unlock_irq(&scst_mcmd_lock);
472         wake_up_all(&scst_mgmt_cmd_list_waitQ);
473
474 out_up:
475         mutex_unlock(&scst_suspend_mutex);
476
477         TRACE_EXIT();
478         return;
479 }
480
481 static int scst_register_device(struct scsi_device *scsidp)
482 {
483         int res = 0;
484         struct scst_device *dev;
485         struct scst_dev_type *dt;
486
487         TRACE_ENTRY();
488
489         scst_suspend_activity();
490         mutex_lock(&scst_mutex);
491
492         res = scst_alloc_device(GFP_KERNEL, &dev);
493         if (res != 0)
494                 goto out_up;
495
496         dev->type = scsidp->type;
497
498         dev->rq_disk = alloc_disk(1);
499         if (dev->rq_disk == NULL) {
500                 res = -ENOMEM;
501                 goto out_free_dev;
502         }
503         dev->rq_disk->major = SCST_MAJOR;
504
505         dev->scsi_dev = scsidp;
506
507         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
508         
509         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
510                 if (dt->type == scsidp->type) {
511                         res = scst_assign_dev_handler(dev, dt);
512                         if (res != 0)
513                                 goto out_free;
514                         break;
515                 }
516         }
517
518 out_up:
519         mutex_unlock(&scst_mutex);
520         scst_resume_activity();
521
522         if (res == 0) {
523                 PRINT_INFO_PR("Attached SCSI target mid-level at "
524                     "scsi%d, channel %d, id %d, lun %d, type %d", 
525                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
526                     scsidp->lun, scsidp->type);
527         } 
528         else {
529                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
530                     "at scsi%d, channel %d, id %d, lun %d, type %d", 
531                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
532                     scsidp->lun, scsidp->type);
533         }
534
535         TRACE_EXIT_RES(res);
536         return res;
537
538 out_free:
539         list_del(&dev->dev_list_entry);
540         put_disk(dev->rq_disk);
541
542 out_free_dev:
543         scst_free_device(dev);
544         goto out_up;
545 }
546
547 static void scst_unregister_device(struct scsi_device *scsidp)
548 {
549         struct scst_device *d, *dev = NULL;
550         struct scst_acg_dev *acg_dev, *aa;
551
552         TRACE_ENTRY();
553         
554         scst_suspend_activity();
555         mutex_lock(&scst_mutex);
556
557         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
558                 if (d->scsi_dev == scsidp) {
559                         dev = d;
560                         TRACE_DBG("Target device %p found", dev);
561                         break;
562                 }
563         }
564         if (dev == NULL) {
565                 PRINT_ERROR_PR("%s", "Target device not found");
566                 goto out_unblock;
567         }
568
569         list_del(&dev->dev_list_entry);
570         
571         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
572                                  dev_acg_dev_list_entry) 
573         {
574                 scst_acg_remove_dev(acg_dev->acg, dev);
575         }
576
577         scst_assign_dev_handler(dev, NULL);
578
579         put_disk(dev->rq_disk);
580         scst_free_device(dev);
581
582         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
583                 "id %d, lun %d, type %d", scsidp->host->host_no,
584                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
585
586 out_unblock:
587         mutex_unlock(&scst_mutex);
588         scst_resume_activity();
589
590         TRACE_EXIT();
591         return;
592 }
593
594 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
595 {
596         int res = 0;
597
598         if (dev_handler->parse == NULL) {
599                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
600                         "parse() method.", dev_handler->name);
601                 res = -EINVAL;
602                 goto out;
603         }
604
605         if (dev_handler->exec == NULL)
606                 dev_handler->exec_atomic = 1;
607
608         if (dev_handler->dev_done == NULL)
609                 dev_handler->dev_done_atomic = 1;
610
611 out:
612         TRACE_EXIT_RES(res);
613         return res;
614 }
615
616 int scst_register_virtual_device(struct scst_dev_type *dev_handler, 
617         const char *dev_name)
618 {
619         int res, rc;
620         struct scst_device *dev = NULL;
621
622         TRACE_ENTRY();
623         
624         if (dev_handler == NULL) {
625                 PRINT_ERROR_PR("%s: valid device handler must be supplied", 
626                         __FUNCTION__);
627                 res = -EINVAL;
628                 goto out;
629         }
630         
631         if (dev_name == NULL) {
632                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
633                 res = -EINVAL;
634                 goto out;
635         }
636
637         res = scst_dev_handler_check(dev_handler);
638         if (res != 0)
639                 goto out;
640
641         scst_suspend_activity();
642         if (mutex_lock_interruptible(&scst_mutex) != 0) {
643                 res = -EINTR;
644                 goto out_resume;
645         }
646
647         res = scst_alloc_device(GFP_KERNEL, &dev);
648         if (res != 0)
649                 goto out_up;
650
651         dev->type = dev_handler->type;
652         dev->scsi_dev = NULL;
653         dev->virt_name = dev_name;
654         dev->virt_id = scst_virt_dev_last_id++;
655
656         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
657
658         res = dev->virt_id;
659
660         rc = scst_assign_dev_handler(dev, dev_handler);
661         if (rc != 0) {
662                 res = rc;
663                 goto out_free_del;
664         }
665
666 out_up:
667         mutex_unlock(&scst_mutex);
668
669 out_resume:
670         scst_resume_activity();
671
672 out:
673         if (res > 0) {
674                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
675                     "device %s (id %d)", dev_name, dev->virt_id);
676         } 
677         else {
678                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
679                     "virtual device %s", dev_name);
680         }
681
682         TRACE_EXIT_RES(res);
683         return res;
684
685 out_free_del:
686         list_del(&dev->dev_list_entry);
687         scst_free_device(dev);
688         goto out_up;
689 }
690
691 void scst_unregister_virtual_device(int id)
692 {
693         struct scst_device *d, *dev = NULL;
694         struct scst_acg_dev *acg_dev, *aa;
695
696         TRACE_ENTRY();
697
698         scst_suspend_activity();
699         mutex_lock(&scst_mutex);
700
701         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
702                 if (d->virt_id == id) {
703                         dev = d;
704                         TRACE_DBG("Target device %p found", dev);
705                         break;
706                 }
707         }
708         if (dev == NULL) {
709                 PRINT_ERROR_PR("%s", "Target device not found");
710                 goto out_unblock;
711         }
712
713         list_del(&dev->dev_list_entry);
714         
715         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
716                                  dev_acg_dev_list_entry) 
717         {
718                 scst_acg_remove_dev(acg_dev->acg, dev);
719         }
720
721         scst_assign_dev_handler(dev, NULL);
722
723         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
724                 "(id %d)", dev->virt_name, dev->virt_id);
725
726         scst_free_device(dev);
727
728 out_unblock:
729         mutex_unlock(&scst_mutex);
730         scst_resume_activity();
731
732         TRACE_EXIT();
733         return;
734 }
735
736 int scst_register_dev_driver(struct scst_dev_type *dev_type)
737 {
738         struct scst_dev_type *dt;
739         struct scst_device *dev;
740         int res;
741         int exist;
742
743         TRACE_ENTRY();
744
745         res = scst_dev_handler_check(dev_type);
746         if (res != 0)
747                 goto out_err;
748
749 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
750         if (dev_type->exec == NULL) {
751                 PRINT_ERROR_PR("Pass-through dev handlers (handler \"%s\") not "
752                         "supported. Consider applying on your kernel patch "
753                         "scst_exec_req_fifo-<kernel-version>.patch or define "
754                         "STRICT_SERIALIZING", dev_type->name);
755                 res = -EINVAL;
756                 goto out_err;
757         }
758 #endif
759
760         scst_suspend_activity();
761         if (mutex_lock_interruptible(&scst_mutex) != 0) {
762                 res = -EINTR;
763                 goto out_err;
764         }
765
766         exist = 0;
767         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
768                 if (strcmp(dt->name, dev_type->name) == 0) {
769                         PRINT_ERROR_PR("Device type handler \"%s\" already "
770                                 "exist", dt->name);
771                         exist = 1;
772                         break;
773                 }
774         }
775         if (exist)
776                 goto out_up;
777
778         res = scst_build_proc_dev_handler_dir_entries(dev_type);
779         if (res < 0) {
780                 goto out_up;
781         }
782
783         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
784
785         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
786                 if ((dev->scsi_dev == NULL) || (dev->handler != NULL))
787                         continue;
788                 if (dev->scsi_dev->type == dev_type->type)
789                         scst_assign_dev_handler(dev, dev_type);
790         }
791
792         mutex_unlock(&scst_mutex);
793         scst_resume_activity();
794
795         if (res == 0) {
796                 PRINT_INFO_PR("Device handler \"%s\" for type %d registered "
797                         "successfully", dev_type->name, dev_type->type);
798         }
799
800 out:
801         TRACE_EXIT_RES(res);
802         return res;
803
804 out_up:
805         mutex_unlock(&scst_mutex);
806
807 out_err:
808         scst_resume_activity();
809         PRINT_ERROR_PR("Failed to register device handler \"%s\" for type %d",
810                 dev_type->name, dev_type->type);
811         goto out;
812 }
813
814 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
815 {
816         struct scst_device *dev;
817         struct scst_dev_type *dt;
818         int found = 0;
819
820         TRACE_ENTRY();
821
822         scst_suspend_activity();
823         mutex_lock(&scst_mutex);
824
825         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
826                 if (strcmp(dt->name, dev_type->name) == 0) {
827                         found = 1;
828                         break;
829                 }
830         }
831         if (!found) {
832                 PRINT_ERROR_PR("Dev handler \"%s\" isn't registered",
833                         dev_type->name);
834                 goto out_up;
835         }
836
837         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
838                 if (dev->handler == dev_type) {
839                         scst_assign_dev_handler(dev, NULL);
840                         TRACE_DBG("Dev handler removed from device %p", dev);
841                 }
842         }
843
844         list_del(&dev_type->dev_type_list_entry);
845
846         mutex_unlock(&scst_mutex);
847         scst_resume_activity();
848
849         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
850
851         PRINT_INFO_PR("Device handler \"%s\" for type %d unloaded",
852                    dev_type->name, dev_type->type);
853
854 out:
855         TRACE_EXIT();
856         return;
857
858 out_up:
859         mutex_unlock(&scst_mutex);
860         scst_resume_activity();
861         goto out;
862 }
863
864 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
865 {
866         int res;
867
868         TRACE_ENTRY();
869
870         res = scst_dev_handler_check(dev_type);
871         if (res != 0)
872                 goto out_err;
873
874         if (!dev_type->no_proc) {
875                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
876                 if (res < 0)
877                         goto out_err;
878         }
879
880         if (dev_type->type != -1) {
881                 PRINT_INFO_PR("Virtual device handler %s for type %d "
882                         "registered successfully", dev_type->name,
883                         dev_type->type);
884         } else {
885                 PRINT_INFO_PR("Virtual device handler \"%s\" registered "
886                         "successfully", dev_type->name);
887         }
888
889 out:
890         TRACE_EXIT_RES(res);
891         return res;
892
893 out_err:
894         PRINT_ERROR_PR("Failed to register virtual device handler \"%s\"",
895                 dev_type->name);
896         goto out;
897 }
898
899 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
900 {
901         TRACE_ENTRY();
902
903         if (!dev_type->no_proc)
904                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
905
906         PRINT_INFO_PR("Device handler \"%s\" unloaded", dev_type->name);
907
908         TRACE_EXIT();
909         return;
910 }
911
912 /* Called under scst_mutex and suspended activity */
913 int scst_add_dev_threads(struct scst_device *dev, int num)
914 {
915         int i, res = 0;
916         int n = 0;
917         struct scst_cmd_thread_t *thr;
918         char nm[12];
919
920         TRACE_ENTRY();
921
922         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
923                 n++;
924         }
925
926         for (i = 0; i < num; i++) {
927                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
928                 if (!thr) {
929                         res = -ENOMEM;
930                         PRINT_ERROR_PR("Failed to allocate thr %d", res);
931                         goto out;
932                 }
933                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
934                 nm[ARRAY_SIZE(nm)-1] = '\0';
935                 thr->cmd_thread = kthread_run(scst_cmd_thread,
936                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
937                 if (IS_ERR(thr->cmd_thread)) {
938                         res = PTR_ERR(thr->cmd_thread);
939                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
940                         kfree(thr);
941                         goto out;
942                 }
943                 list_add(&thr->thread_list_entry, &dev->threads_list);
944         }
945
946 out:
947         TRACE_EXIT_RES(res);
948         return res;
949 }
950
951 /* Called under scst_mutex and suspended activity */
952 static int scst_create_dev_threads(struct scst_device *dev)
953 {
954         int res = 0;
955         int threads_num;
956
957         TRACE_ENTRY();
958
959         if (dev->handler->threads_num <= 0)
960                 goto out;
961
962         threads_num = dev->handler->threads_num;
963
964         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
965         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
966         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
967
968         res = scst_add_dev_threads(dev, threads_num);
969         if (res != 0)
970                 goto out;
971
972         mutex_lock(&scst_suspend_mutex);
973         list_add_tail(&dev->cmd_lists.lists_list_entry,
974                 &scst_cmd_lists_list);
975         mutex_unlock(&scst_suspend_mutex);
976
977         dev->p_cmd_lists = &dev->cmd_lists;
978
979 out:
980         TRACE_EXIT_RES(res);
981         return res;
982 }
983
984 /* Called under scst_mutex and suspended activity */
985 void scst_del_dev_threads(struct scst_device *dev, int num)
986 {
987         struct scst_cmd_thread_t *ct, *tmp;
988         int i = 0;
989
990         TRACE_ENTRY();
991
992         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
993                                 thread_list_entry) {
994                 int rc = kthread_stop(ct->cmd_thread);
995                 if (rc < 0) {
996                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
997                 }
998                 list_del(&ct->thread_list_entry);
999                 kfree(ct);
1000                 if ((num >0) && (++i >= num))
1001                         break;
1002         }
1003
1004         TRACE_EXIT();
1005         return;
1006 }
1007
1008 /* Called under scst_mutex and suspended activity */
1009 static void scst_stop_dev_threads(struct scst_device *dev)
1010 {
1011         TRACE_ENTRY();
1012
1013         if (list_empty(&dev->threads_list))
1014                 goto out;
1015
1016         scst_del_dev_threads(dev, -1);
1017
1018         if (dev->p_cmd_lists == &dev->cmd_lists) {
1019                 mutex_lock(&scst_suspend_mutex);
1020                 list_del(&dev->cmd_lists.lists_list_entry);
1021                 mutex_unlock(&scst_suspend_mutex);
1022         }
1023
1024 out:
1025         TRACE_EXIT();
1026         return;
1027 }
1028
1029 /* The activity supposed to be suspended and scst_mutex held */
1030 int scst_assign_dev_handler(struct scst_device *dev, 
1031         struct scst_dev_type *handler)
1032 {
1033         int res = 0;
1034         struct scst_tgt_dev *tgt_dev;
1035         LIST_HEAD(attached_tgt_devs);
1036         
1037         TRACE_ENTRY();
1038         
1039         if (dev->handler == handler)
1040                 goto out;
1041         
1042         if (dev->handler && dev->handler->detach_tgt) {
1043                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1044                                 dev_tgt_dev_list_entry) {
1045                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1046                                 tgt_dev);
1047                         dev->handler->detach_tgt(tgt_dev);
1048                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1049                 }
1050         }
1051
1052         if (dev->handler && dev->handler->detach) {
1053                 TRACE_DBG("%s", "Calling dev handler's detach()");
1054                 dev->handler->detach(dev);
1055                 TRACE_DBG("%s", "Old handler's detach() returned");
1056         }
1057
1058         scst_stop_dev_threads(dev);
1059
1060         dev->handler = handler;
1061
1062         if (handler) {
1063                 res = scst_create_dev_threads(dev);
1064                 if (res != 0)
1065                         goto out_null;
1066         }
1067
1068         if (handler && handler->attach) {
1069                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1070                 res = handler->attach(dev);
1071                 TRACE_DBG("New dev handler's attach() returned %d", res);
1072                 if (res != 0) {
1073                         PRINT_ERROR_PR("New device handler's %s attach() "
1074                                 "failed: %d", handler->name, res);
1075                 }
1076                 goto out_thr_null;
1077         }
1078         
1079         if (handler && handler->attach_tgt) {
1080                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1081                                 dev_tgt_dev_list_entry) {
1082                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1083                                 tgt_dev);
1084                         res = handler->attach_tgt(tgt_dev);
1085                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1086                         if (res != 0) {
1087                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
1088                                     "failed: %d", handler->name, res);
1089                                 goto out_err_detach_tgt;
1090                         }
1091                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1092                                 &attached_tgt_devs);
1093                 }
1094         }
1095
1096 out_thr_null:
1097         if (res != 0)
1098                 scst_stop_dev_threads(dev);
1099
1100 out_null:
1101         if (res != 0)
1102                 dev->handler = NULL;
1103         
1104 out:
1105         TRACE_EXIT_RES(res);
1106         return res;
1107
1108 out_err_detach_tgt:
1109         if (handler && handler->detach_tgt) {
1110                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1111                                  extra_tgt_dev_list_entry) 
1112                 {
1113                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1114                                 tgt_dev);
1115                         handler->detach_tgt(tgt_dev);
1116                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1117                 }
1118         }
1119         if (handler && handler->detach) {
1120                 TRACE_DBG("%s", "Calling handler's detach()");
1121                 handler->detach(dev);
1122                 TRACE_DBG("%s", "Handler's detach() returned");
1123         }
1124         goto out_null;
1125 }
1126
1127 int scst_cmd_threads_count(void)
1128 {
1129         int i;
1130
1131         /* Just to lower the race window, when user can get just changed value */
1132         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1133         i = scst_threads_info.nr_cmd_threads;
1134         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1135         return i;
1136 }
1137
1138 static void scst_threads_info_init(void)
1139 {
1140         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1141         mutex_init(&scst_threads_info.cmd_threads_mutex);
1142         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1143 }
1144
1145 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1146 void __scst_del_cmd_threads(int num)
1147 {
1148         struct scst_cmd_thread_t *ct, *tmp;
1149         int i;
1150
1151         TRACE_ENTRY();
1152
1153         i = scst_threads_info.nr_cmd_threads;
1154         if (num <= 0 || num > i) {
1155                 PRINT_ERROR_PR("can not del %d cmd threads from %d", num, i);
1156                 return;
1157         }
1158
1159         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1160                                 thread_list_entry) {
1161                 int res;
1162
1163                 res = kthread_stop(ct->cmd_thread);
1164                 if (res < 0) {
1165                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1166                 }
1167                 list_del(&ct->thread_list_entry);
1168                 kfree(ct);
1169                 scst_threads_info.nr_cmd_threads--;
1170                 --num;
1171                 if (num == 0)
1172                         break;
1173         }
1174
1175         TRACE_EXIT();
1176         return;
1177 }
1178
1179 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1180 int __scst_add_cmd_threads(int num)
1181 {
1182         int res = 0, i;
1183         static int scst_thread_num = 0;
1184         
1185         TRACE_ENTRY();
1186
1187         for (i = 0; i < num; i++) {
1188                 struct scst_cmd_thread_t *thr;
1189
1190                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1191                 if (!thr) {
1192                         res = -ENOMEM;
1193                         PRINT_ERROR_PR("fail to allocate thr %d", res);
1194                         goto out_error;
1195                 }
1196                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1197                         &scst_main_cmd_lists, "scsi_tgt%d",
1198                         scst_thread_num++);
1199                 if (IS_ERR(thr->cmd_thread)) {
1200                         res = PTR_ERR(thr->cmd_thread);
1201                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
1202                         kfree(thr);
1203                         goto out_error;
1204                 }
1205                 list_add(&thr->thread_list_entry,
1206                         &scst_threads_info.cmd_threads_list);
1207                 scst_threads_info.nr_cmd_threads++;
1208         }
1209         res = 0;
1210
1211 out:
1212         TRACE_EXIT_RES(res);
1213         return res;
1214
1215 out_error:
1216         if (i > 0)
1217                 __scst_del_cmd_threads(i - 1);
1218         goto out;
1219 }
1220
1221 int scst_add_cmd_threads(int num)
1222 {
1223         int res;
1224
1225         TRACE_ENTRY();
1226
1227         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1228         res = __scst_add_cmd_threads(num);
1229         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1230
1231         TRACE_EXIT_RES(res);
1232         return res;
1233 }
1234
1235 void scst_del_cmd_threads(int num)
1236 {
1237         TRACE_ENTRY();
1238
1239         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1240         __scst_del_cmd_threads(num);
1241         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1242
1243         TRACE_EXIT();
1244         return;
1245 }
1246
1247 static void scst_stop_all_threads(void)
1248 {
1249         TRACE_ENTRY();
1250
1251         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1252         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1253         if (scst_threads_info.mgmt_cmd_thread)
1254                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1255         if (scst_threads_info.mgmt_thread)
1256                 kthread_stop(scst_threads_info.mgmt_thread);
1257         if (scst_threads_info.init_cmd_thread)
1258                 kthread_stop(scst_threads_info.init_cmd_thread);
1259         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1260
1261         TRACE_EXIT();
1262         return;
1263 }
1264
1265 static int scst_start_all_threads(int num)
1266 {
1267         int res;
1268
1269         TRACE_ENTRY();
1270
1271         mutex_lock(&scst_threads_info.cmd_threads_mutex);               
1272         res = __scst_add_cmd_threads(num);
1273         if (res < 0)
1274                 goto out;
1275
1276         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1277                 NULL, "scsi_tgt_init");
1278         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1279                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1280                 PRINT_ERROR_PR("kthread_create() for init cmd failed: %d", res);
1281                 scst_threads_info.init_cmd_thread = NULL;
1282                 goto out;
1283         }
1284
1285         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1286                 NULL, "scsi_tgt_mc");
1287         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1288                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1289                 PRINT_ERROR_PR("kthread_create() for mcmd failed: %d", res);
1290                 scst_threads_info.mgmt_cmd_thread = NULL;
1291                 goto out;
1292         }
1293
1294         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1295                 NULL, "scsi_tgt_mgmt");
1296         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1297                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1298                 PRINT_ERROR_PR("kthread_create() for mgmt failed: %d", res);
1299                 scst_threads_info.mgmt_thread = NULL;
1300                 goto out;
1301         }
1302
1303 out:
1304         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1305         TRACE_EXIT_RES(res);
1306         return res;     
1307 }
1308
1309 void scst_get(void)
1310 {
1311         __scst_get(0);
1312 }
1313
1314 void scst_put(void)
1315 {
1316         __scst_put();
1317 }
1318
1319 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1320 static int scst_add(struct class_device *cdev)
1321 #else
1322 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1323 #endif
1324 {
1325         struct scsi_device *scsidp;
1326         int res = 0;
1327
1328         TRACE_ENTRY();
1329         
1330         scsidp = to_scsi_device(cdev->dev);
1331         res = scst_register_device(scsidp);
1332
1333         TRACE_EXIT();
1334         return res;
1335 }
1336
1337 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1338 static void scst_remove(struct class_device *cdev)
1339 #else
1340 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1341 #endif
1342 {
1343         struct scsi_device *scsidp;
1344
1345         TRACE_ENTRY();
1346
1347         scsidp = to_scsi_device(cdev->dev);
1348         scst_unregister_device(scsidp);
1349
1350         TRACE_EXIT();
1351         return;
1352 }
1353
1354 static struct class_interface scst_interface = {
1355         .add = scst_add,
1356         .remove = scst_remove,
1357 };
1358
1359 static int __init init_scst(void)
1360 {
1361         int res = 0, i;
1362         struct scst_cmd *cmd;
1363         int scst_num_cpus;
1364
1365         TRACE_ENTRY();
1366
1367 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1368         {
1369                 struct scsi_request *req;
1370                 BUILD_BUG_ON(sizeof(cmd->sense_buffer) !=
1371                         sizeof(req->sr_sense_buffer));
1372         }
1373 #else
1374         {
1375                 struct scsi_sense_hdr *shdr;
1376                 BUILD_BUG_ON((sizeof(cmd->sense_buffer) < sizeof(*shdr)) &&
1377                         (sizeof(cmd->sense_buffer) >= SCST_SENSE_BUFFERSIZE));
1378         }
1379 #endif
1380         {
1381                 struct scst_tgt_dev *t;
1382                 struct scst_cmd *c;
1383                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1384                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1385         }
1386
1387         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1388         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1389         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1390         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1391
1392         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1393         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1394         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1395         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1396                 &scst_cmd_lists_list);
1397
1398         scst_num_cpus = num_online_cpus();
1399
1400         /* ToDo: register_cpu_notifier() */
1401         
1402         if (scst_threads == 0)
1403                 scst_threads = scst_num_cpus;
1404                 
1405         if (scst_threads < scst_num_cpus) {
1406                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
1407                         "CPUs count");
1408                 scst_threads = scst_num_cpus;
1409         }
1410
1411         scst_threads_info_init();
1412
1413 #define INIT_CACHEP(p, s, t, o) do {                                    \
1414                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
1415                                       SCST_SLAB_FLAGS, NULL, NULL);     \
1416                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
1417                           sizeof(struct t));                            \
1418                 if (p == NULL) { res = -ENOMEM; goto o; }               \
1419         } while (0)
1420           
1421         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING, 
1422                     scst_mgmt_cmd, out);
1423         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING, 
1424                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
1425         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING, 
1426                     scst_cmd, out_destroy_ua_cache);
1427         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
1428                     scst_session, out_destroy_cmd_cache);
1429         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
1430                     scst_tgt_dev, out_destroy_sess_cache);
1431         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
1432                     scst_acg_dev, out_destroy_tgt_cache);
1433
1434         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
1435                 mempool_free_slab, scst_mgmt_cachep);
1436         if (scst_mgmt_mempool == NULL) {
1437                 res = -ENOMEM;
1438                 goto out_destroy_acg_cache;
1439         }
1440
1441         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1442                 mempool_free_slab, scst_ua_cachep);
1443         if (scst_ua_mempool == NULL) {
1444                 res = -ENOMEM;
1445                 goto out_destroy_mgmt_mempool;
1446         }
1447
1448         if (scst_max_cmd_mem == 0) {
1449                 struct sysinfo si;
1450                 si_meminfo(&si);
1451 #if BITS_PER_LONG == 32
1452                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1453                                         (uint64_t)1 << 30);
1454 #else
1455                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1456 #endif
1457         } else
1458                 scst_max_cmd_mem <<= 20;
1459
1460         res = scst_sgv_pools_init(scst_max_cmd_mem, 0);
1461         if (res != 0)
1462                 goto out_destroy_ua_mempool;
1463
1464         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1465         if (scst_default_acg == NULL) {
1466                 res = -ENOMEM;
1467                 goto out_destroy_sgv_pool;
1468         }
1469         
1470         res = scsi_register_interface(&scst_interface);
1471         if (res != 0)
1472                 goto out_free_acg;
1473
1474         scst_scsi_op_list_init();
1475
1476         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1477                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1478                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1479                 tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet,
1480                         (unsigned long)&scst_tasklets[i]);
1481         }
1482
1483         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1484                 scst_threads);
1485
1486         res = scst_start_all_threads(scst_threads);
1487         if (res < 0)
1488                 goto out_thread_free;
1489
1490         res = scst_proc_init_module();
1491         if (res != 0)
1492                 goto out_thread_free;
1493
1494
1495         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1496                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1497
1498 out:
1499         TRACE_EXIT_RES(res);
1500         return res;
1501
1502 out_thread_free:
1503         scst_stop_all_threads();
1504
1505         scsi_unregister_interface(&scst_interface);
1506
1507 out_free_acg:
1508         scst_destroy_acg(scst_default_acg);
1509
1510 out_destroy_sgv_pool:
1511         scst_sgv_pools_deinit();
1512
1513 out_destroy_ua_mempool:
1514         mempool_destroy(scst_ua_mempool);
1515
1516 out_destroy_mgmt_mempool:
1517         mempool_destroy(scst_mgmt_mempool);
1518
1519 out_destroy_acg_cache:
1520         kmem_cache_destroy(scst_acgd_cachep);
1521
1522 out_destroy_tgt_cache:
1523         kmem_cache_destroy(scst_tgtd_cachep);
1524
1525 out_destroy_sess_cache:
1526         kmem_cache_destroy(scst_sess_cachep);
1527
1528 out_destroy_cmd_cache:
1529         kmem_cache_destroy(scst_cmd_cachep);
1530
1531 out_destroy_ua_cache:
1532         kmem_cache_destroy(scst_ua_cachep);
1533
1534 out_destroy_mgmt_cache:
1535         kmem_cache_destroy(scst_mgmt_cachep);
1536         goto out;
1537 }
1538
1539 static void __exit exit_scst(void)
1540 {
1541 #ifdef CONFIG_LOCKDEP
1542         static /* To hide the lockdep's warning about non-static key */
1543 #endif
1544         DECLARE_MUTEX_LOCKED(shm);
1545
1546         TRACE_ENTRY();
1547         
1548         /* ToDo: unregister_cpu_notifier() */
1549
1550         scst_proc_cleanup_module();
1551
1552         scst_stop_all_threads();
1553
1554         scsi_unregister_interface(&scst_interface);
1555         scst_destroy_acg(scst_default_acg);
1556
1557         scst_sgv_pools_deinit();
1558
1559 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
1560 #define DEINIT_CACHEP(p, s) do {                        \
1561                 if (kmem_cache_destroy(p)) {            \
1562                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1563                                 "error", s);            \
1564                 }                                       \
1565                 p = NULL;                               \
1566         } while (0)
1567 #else
1568 #define DEINIT_CACHEP(p, s) do {                        \
1569                 kmem_cache_destroy(p);                  \
1570                 p = NULL;                               \
1571         } while (0)
1572 #endif
1573
1574         mempool_destroy(scst_mgmt_mempool);
1575         mempool_destroy(scst_ua_mempool);
1576
1577         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1578         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1579         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1580         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1581         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1582         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1583
1584         PRINT_INFO_PR("%s", "SCST unloaded");
1585
1586         TRACE_EXIT();
1587         return;
1588 }
1589
1590 /*
1591  * Device Handler Side (i.e. scst_vdisk)
1592  */
1593 EXPORT_SYMBOL(scst_register_dev_driver);
1594 EXPORT_SYMBOL(scst_unregister_dev_driver);
1595 EXPORT_SYMBOL(scst_register);
1596 EXPORT_SYMBOL(scst_unregister);
1597
1598 EXPORT_SYMBOL(scst_register_virtual_device);
1599 EXPORT_SYMBOL(scst_unregister_virtual_device);
1600 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1601 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1602
1603 EXPORT_SYMBOL(scst_set_busy);
1604 EXPORT_SYMBOL(scst_set_cmd_error_status);
1605 EXPORT_SYMBOL(scst_set_cmd_error);
1606 EXPORT_SYMBOL(scst_set_resp_data_len);
1607
1608 EXPORT_SYMBOL(scst_process_active_cmd);
1609
1610 /*
1611  * Target Driver Side (i.e. HBA)
1612  */
1613 EXPORT_SYMBOL(scst_register_session);
1614 EXPORT_SYMBOL(scst_unregister_session);
1615
1616 EXPORT_SYMBOL(scst_register_target_template);
1617 EXPORT_SYMBOL(scst_unregister_target_template);
1618
1619 EXPORT_SYMBOL(scst_cmd_init_done);
1620 EXPORT_SYMBOL(scst_tgt_cmd_done);
1621 EXPORT_SYMBOL(scst_restart_cmd);
1622 EXPORT_SYMBOL(scst_rx_cmd);
1623 EXPORT_SYMBOL(scst_rx_data);
1624 EXPORT_SYMBOL(scst_rx_mgmt_fn);
1625
1626 EXPORT_SYMBOL(scst_find_cmd);
1627 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1628
1629 /*
1630  * Global Commands
1631  */
1632 EXPORT_SYMBOL(scst_suspend_activity);
1633 EXPORT_SYMBOL(scst_resume_activity);
1634
1635 EXPORT_SYMBOL(scst_add_cmd_threads);
1636 EXPORT_SYMBOL(scst_del_cmd_threads);
1637
1638 #if defined(DEBUG) || defined(TRACING)
1639 EXPORT_SYMBOL(scst_proc_log_entry_read);
1640 EXPORT_SYMBOL(scst_proc_log_entry_write);
1641 #endif
1642
1643 EXPORT_SYMBOL(scst_create_proc_entry);
1644 EXPORT_SYMBOL(scst_single_seq_open);
1645
1646 EXPORT_SYMBOL(__scst_get_buf);
1647 EXPORT_SYMBOL(scst_get);
1648 EXPORT_SYMBOL(scst_put);
1649
1650 EXPORT_SYMBOL(scst_alloc);
1651 EXPORT_SYMBOL(scst_free);
1652
1653 EXPORT_SYMBOL(scst_check_local_events);
1654
1655 /* Tgt_dev's threads local storage */
1656 EXPORT_SYMBOL(scst_add_thr_data);
1657 EXPORT_SYMBOL(scst_del_all_thr_data);
1658 EXPORT_SYMBOL(scst_dev_del_all_thr_data);
1659 EXPORT_SYMBOL(scst_find_thr_data);
1660
1661 /* SGV pool routines */
1662 EXPORT_SYMBOL(sgv_pool_create);
1663 EXPORT_SYMBOL(sgv_pool_destroy);
1664 EXPORT_SYMBOL(sgv_pool_set_allocator);
1665 EXPORT_SYMBOL(sgv_pool_alloc);
1666 EXPORT_SYMBOL(sgv_pool_free);
1667 EXPORT_SYMBOL(sgv_get_priv);
1668
1669 /* Generic parse() routines */
1670 EXPORT_SYMBOL(scst_calc_block_shift);
1671 EXPORT_SYMBOL(scst_sbc_generic_parse);
1672 EXPORT_SYMBOL(scst_cdrom_generic_parse);
1673 EXPORT_SYMBOL(scst_modisk_generic_parse);
1674 EXPORT_SYMBOL(scst_tape_generic_parse);
1675 EXPORT_SYMBOL(scst_changer_generic_parse);
1676 EXPORT_SYMBOL(scst_processor_generic_parse);
1677 EXPORT_SYMBOL(scst_raid_generic_parse);
1678
1679 /* Generic dev_done() routines */
1680 EXPORT_SYMBOL(scst_block_generic_dev_done);
1681 EXPORT_SYMBOL(scst_tape_generic_dev_done);
1682
1683 /*
1684  * Other Commands
1685  */
1686 EXPORT_SYMBOL(scst_get_cdb_info);
1687 EXPORT_SYMBOL(scst_cmd_get_tgt_priv_lock);
1688 EXPORT_SYMBOL(scst_cmd_set_tgt_priv_lock);
1689
1690 #ifdef DEBUG
1691 EXPORT_SYMBOL(scst_random);
1692 #endif
1693
1694 module_init(init_scst);
1695 module_exit(exit_scst);
1696
1697 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1698 MODULE_LICENSE("GPL");
1699 MODULE_DESCRIPTION("SCSI target core");
1700 MODULE_VERSION(SCST_VERSION_STRING);