bfcd1417e42e59fa023265f226abdf549fb1fc26
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <asm/unistd.h>
28 #include <asm/string.h>
29 #include <linux/kthread.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
36 #warning HIGHMEM kernel configurations are fully supported, but not \
37         recommended for performance reasons. Consider change VMSPLIT \
38         option or use 64-bit configuration instead. See README file for \
39         details.
40 #endif
41
42 #ifdef SCST_HIGHMEM
43 #error SCST_HIGHMEM configuration isn't supported and broken, because there \
44         is no real point to support it, at least it definitely doesn't worth \
45         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
46         or in 64-bit configuration instead. See README file for details.
47 #endif
48
49 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
50 #warning Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
51         your kernel and STRICT_SERIALIZING isn't defined. Pass-through dev \
52         handlers will not be supported.
53 #endif
54
55 /*
56  * All targets, devices and dev_types management is done under this mutex.
57  *
58  * It must NOT be used in any works (schedule_work(), etc.), because
59  * otherwise a deadlock (double lock, actually) is possible, e.g., with
60  * scst_user detach_tgt(), which is called under scst_mutex and calls
61  * flush_scheduled_work().
62  */
63 DEFINE_MUTEX(scst_mutex);
64
65 LIST_HEAD(scst_template_list);
66 LIST_HEAD(scst_dev_list);
67 LIST_HEAD(scst_dev_type_list);
68
69 spinlock_t scst_main_lock = SPIN_LOCK_UNLOCKED;
70
71 struct kmem_cache *scst_mgmt_cachep;
72 mempool_t *scst_mgmt_mempool;
73 struct kmem_cache *scst_ua_cachep;
74 mempool_t *scst_ua_mempool;
75 struct kmem_cache *scst_tgtd_cachep;
76 struct kmem_cache *scst_sess_cachep;
77 struct kmem_cache *scst_acgd_cachep;
78
79 LIST_HEAD(scst_acg_list);
80 struct scst_acg *scst_default_acg;
81
82 spinlock_t scst_init_lock = SPIN_LOCK_UNLOCKED;
83 DECLARE_WAIT_QUEUE_HEAD(scst_init_cmd_list_waitQ);
84 LIST_HEAD(scst_init_cmd_list);
85 unsigned int scst_init_poll_cnt;
86
87 struct kmem_cache *scst_cmd_cachep;
88
89 #if defined(DEBUG) || defined(TRACING)
90 unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
91 #endif
92
93 unsigned long scst_flags;
94 atomic_t scst_cmd_count = ATOMIC_INIT(0);
95
96 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
97 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
98 unsigned long scst_max_cmd_mem;
99
100 struct scst_cmd_lists scst_main_cmd_lists;
101
102 struct scst_tasklet scst_tasklets[NR_CPUS];
103
104
105 spinlock_t scst_mcmd_lock = SPIN_LOCK_UNLOCKED;
106 LIST_HEAD(scst_active_mgmt_cmd_list);
107 LIST_HEAD(scst_delayed_mgmt_cmd_list);
108 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
109
110 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
111 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
112 LIST_HEAD(scst_sess_init_list);
113 LIST_HEAD(scst_sess_shut_list);
114
115 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
116
117 DEFINE_MUTEX(scst_suspend_mutex);
118 LIST_HEAD(scst_cmd_lists_list); /* protected by scst_suspend_mutex */
119
120 static int scst_threads;
121 struct scst_threads_info_t scst_threads_info;
122
123 static int suspend_count;
124
125 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
126
127 /* 
128  * This buffer and lock are intended to avoid memory allocation, which
129  * could fail in improper places.
130  */
131 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
132 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
133
134 module_param_named(scst_threads, scst_threads, int, 0);
135 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
136
137 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
138 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
139         "the SCST commands at any given time in Mb");
140
141 int scst_register_target_template(struct scst_tgt_template *vtt)
142 {
143         int res = 0;
144         struct scst_tgt_template *t;
145         static DEFINE_MUTEX(m);
146
147         TRACE_ENTRY();
148
149         INIT_LIST_HEAD(&vtt->tgt_list);
150
151         if (!vtt->detect) {
152                 PRINT_ERROR_PR("Target driver %s doesn't have a "
153                         "detect() method.", vtt->name);
154                 res = -EINVAL;
155                 goto out_err;
156         }
157         
158         if (!vtt->release) {
159                 PRINT_ERROR_PR("Target driver %s doesn't have a "
160                         "release() method.", vtt->name);
161                 res = -EINVAL;
162                 goto out_err;
163         }
164
165         if (!vtt->xmit_response) {
166                 PRINT_ERROR_PR("Target driver %s doesn't have a "
167                         "xmit_response() method.", vtt->name);
168                 res = -EINVAL;
169                 goto out_err;
170         }
171
172         if (vtt->threads_num < 0) {
173                 PRINT_ERROR_PR("Wrong threads_num value %d for "
174                         "target \"%s\"", vtt->threads_num,
175                         vtt->name);
176                 res = -EINVAL;
177                 goto out_err;
178         }
179
180         if (!vtt->no_proc_entry) {
181                 res = scst_build_proc_target_dir_entries(vtt);
182                 if (res < 0)
183                         goto out_err;
184         }
185
186         if (vtt->preprocessing_done == NULL)
187                 vtt->preprocessing_done_atomic = 1;
188
189         if (mutex_lock_interruptible(&m) != 0)
190                 goto out_err;
191
192         if (mutex_lock_interruptible(&scst_mutex) != 0)
193                 goto out_m_up;
194         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
195                 if (strcmp(t->name, vtt->name) == 0) {
196                         PRINT_ERROR_PR("Target driver %s already registered",
197                                 vtt->name);
198                         mutex_unlock(&scst_mutex);
199                         goto out_cleanup;
200                 }
201         }
202         mutex_unlock(&scst_mutex);
203
204         TRACE_DBG("%s", "Calling target driver's detect()");
205         res = vtt->detect(vtt);
206         TRACE_DBG("Target driver's detect() returned %d", res);
207         if (res < 0) {
208                 PRINT_ERROR_PR("%s", "The detect() routine failed");
209                 res = -EINVAL;
210                 goto out_cleanup;
211         }
212
213         mutex_lock(&scst_mutex);
214         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
215         mutex_unlock(&scst_mutex);
216
217         res = 0;
218
219         PRINT_INFO_PR("Target template %s registered successfully", vtt->name);
220
221         mutex_unlock(&m);
222
223 out:
224         TRACE_EXIT_RES(res);
225         return res;
226
227 out_m_up:
228         mutex_unlock(&m);
229
230 out_cleanup:
231         scst_cleanup_proc_target_dir_entries(vtt);
232
233 out_err:
234         PRINT_ERROR_PR("Failed to register target template %s", vtt->name);
235         goto out;
236 }
237
238 void scst_unregister_target_template(struct scst_tgt_template *vtt)
239 {
240         struct scst_tgt *tgt;
241         struct scst_tgt_template *t;
242         int found = 0;
243
244         TRACE_ENTRY();
245
246         mutex_lock(&scst_mutex);
247
248         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
249                 if (strcmp(t->name, vtt->name) == 0) {
250                         found = 1;
251                         break;
252                 }
253         }
254         if (!found) {
255                 PRINT_ERROR_PR("Target driver %s isn't registered", vtt->name);
256                 goto out_up;
257         }
258
259 restart:
260         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
261                 mutex_unlock(&scst_mutex);
262                 scst_unregister(tgt);
263                 mutex_lock(&scst_mutex);
264                 goto restart;
265         }
266         list_del(&vtt->scst_template_list_entry);
267
268         PRINT_INFO_PR("Target template %s unregistered successfully", vtt->name);
269
270 out_up:
271         mutex_unlock(&scst_mutex);
272
273         scst_cleanup_proc_target_dir_entries(vtt);
274
275         TRACE_EXIT();
276         return;
277 }
278
279 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
280         const char *target_name)
281 {
282         struct scst_tgt *tgt;
283
284         TRACE_ENTRY();
285
286         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
287         if (tgt == NULL) {
288                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
289                 goto out_err;
290         }
291
292         INIT_LIST_HEAD(&tgt->sess_list);
293         init_waitqueue_head(&tgt->unreg_waitQ);
294         tgt->tgtt = vtt;
295         tgt->sg_tablesize = vtt->sg_tablesize;
296         spin_lock_init(&tgt->tgt_lock);
297         INIT_LIST_HEAD(&tgt->retry_cmd_list);
298         atomic_set(&tgt->finished_cmds, 0);
299         init_timer(&tgt->retry_timer);
300         tgt->retry_timer.data = (unsigned long)tgt;
301         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
302
303         scst_suspend_activity();
304         mutex_lock(&scst_mutex);
305
306         if (target_name != NULL) {
307                 int len = strlen(target_name) + 1 +
308                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
309
310                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
311                 if (tgt->default_group_name == NULL) {
312                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
313                                 "group name failed");
314                         goto out_free_err;
315                 }
316                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
317                         target_name);
318         }
319
320         if (scst_build_proc_target_entries(tgt) < 0)
321                 goto out_free_name;
322         else
323                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
324
325         mutex_unlock(&scst_mutex);
326         scst_resume_activity();
327
328         PRINT_INFO_PR("Target %s for template %s registered successfully",
329                 target_name, vtt->name);
330
331 out:
332         TRACE_EXIT();
333         return tgt;
334
335 out_free_name:
336         if (tgt->default_group_name)
337                 kfree(tgt->default_group_name);
338
339 out_free_err:
340         mutex_unlock(&scst_mutex);
341         scst_resume_activity();
342
343         kfree(tgt);
344         tgt = NULL;
345
346 out_err:
347         PRINT_ERROR_PR("Failed to register target for template %s", vtt->name);
348         goto out;
349 }
350
351 static inline int test_sess_list(struct scst_tgt *tgt)
352 {
353         int res;
354         mutex_lock(&scst_mutex);
355         res = list_empty(&tgt->sess_list);
356         mutex_unlock(&scst_mutex);
357         return res;
358 }
359
360 void scst_unregister(struct scst_tgt *tgt)
361 {
362         struct scst_session *sess;
363         struct scst_tgt_template *vtt = tgt->tgtt;
364
365         TRACE_ENTRY();
366
367         TRACE_DBG("%s", "Calling target driver's release()");
368         tgt->tgtt->release(tgt);
369         TRACE_DBG("%s", "Target driver's release() returned");
370
371         mutex_lock(&scst_mutex);
372         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
373                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
374         }
375         mutex_unlock(&scst_mutex);
376
377         TRACE_DBG("%s", "Waiting for sessions shutdown");
378         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
379         TRACE_DBG("%s", "wait_event() returned");
380
381         scst_suspend_activity();
382         mutex_lock(&scst_mutex);
383
384         list_del(&tgt->tgt_list_entry);
385
386         scst_cleanup_proc_target_entries(tgt);
387
388         if (tgt->default_group_name)
389                 kfree(tgt->default_group_name);
390
391         mutex_unlock(&scst_mutex);
392         scst_resume_activity();
393
394         del_timer_sync(&tgt->retry_timer);
395
396         kfree(tgt);
397
398         PRINT_INFO_PR("Target for template %s unregistered successfully",
399                 vtt->name);
400
401         TRACE_EXIT();
402         return;
403 }
404
405 void scst_suspend_activity(void)
406 {
407         TRACE_ENTRY();
408
409         mutex_lock(&scst_suspend_mutex);
410
411         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
412         suspend_count++;
413         if (suspend_count > 1)
414                 goto out_up;
415
416         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
417         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
418         smp_mb__after_set_bit();
419
420         TRACE_MGMT_DBG("Waiting for %d active commands to complete",
421               atomic_read(&scst_cmd_count));
422         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
423         TRACE_MGMT_DBG("%s", "wait_event() returned");
424
425         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
426         smp_mb__after_clear_bit();
427
428         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
429               atomic_read(&scst_cmd_count));
430         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
431         TRACE_MGMT_DBG("%s", "wait_event() returned");
432
433 out_up:
434         mutex_unlock(&scst_suspend_mutex);
435
436         TRACE_EXIT();
437         return;
438 }
439
440 void scst_resume_activity(void)
441 {
442         struct scst_cmd_lists *l;
443
444         TRACE_ENTRY();
445
446         mutex_lock(&scst_suspend_mutex);
447
448         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
449         suspend_count--;
450         if (suspend_count > 0)
451                 goto out_up;
452
453         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
454         smp_mb__after_clear_bit();
455
456         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
457                 wake_up_all(&l->cmd_list_waitQ);
458         }
459         wake_up_all(&scst_init_cmd_list_waitQ);
460
461         spin_lock_irq(&scst_mcmd_lock);
462         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
463                 struct scst_mgmt_cmd *m;
464                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
465                                 mgmt_cmd_list_entry);
466                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
467                         "mgmt cmd list", m);
468                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
469         }
470         spin_unlock_irq(&scst_mcmd_lock);
471         wake_up_all(&scst_mgmt_cmd_list_waitQ);
472
473 out_up:
474         mutex_unlock(&scst_suspend_mutex);
475
476         TRACE_EXIT();
477         return;
478 }
479
480 static int scst_register_device(struct scsi_device *scsidp)
481 {
482         int res = 0;
483         struct scst_device *dev;
484         struct scst_dev_type *dt;
485
486         TRACE_ENTRY();
487
488         scst_suspend_activity();
489         mutex_lock(&scst_mutex);
490
491         res = scst_alloc_device(GFP_KERNEL, &dev);
492         if (res != 0)
493                 goto out_up;
494
495         dev->type = scsidp->type;
496
497         dev->rq_disk = alloc_disk(1);
498         if (dev->rq_disk == NULL) {
499                 res = -ENOMEM;
500                 goto out_free_dev;
501         }
502         dev->rq_disk->major = SCST_MAJOR;
503
504         dev->scsi_dev = scsidp;
505
506         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
507         
508         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
509                 if (dt->type == scsidp->type) {
510                         res = scst_assign_dev_handler(dev, dt);
511                         if (res != 0)
512                                 goto out_free;
513                         break;
514                 }
515         }
516
517 out_up:
518         mutex_unlock(&scst_mutex);
519         scst_resume_activity();
520
521         if (res == 0) {
522                 PRINT_INFO_PR("Attached SCSI target mid-level at "
523                     "scsi%d, channel %d, id %d, lun %d, type %d", 
524                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
525                     scsidp->lun, scsidp->type);
526         } 
527         else {
528                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
529                     "at scsi%d, channel %d, id %d, lun %d, type %d", 
530                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
531                     scsidp->lun, scsidp->type);
532         }
533
534         TRACE_EXIT_RES(res);
535         return res;
536
537 out_free:
538         list_del(&dev->dev_list_entry);
539         put_disk(dev->rq_disk);
540
541 out_free_dev:
542         scst_free_device(dev);
543         goto out_up;
544 }
545
546 static void scst_unregister_device(struct scsi_device *scsidp)
547 {
548         struct scst_device *d, *dev = NULL;
549         struct scst_acg_dev *acg_dev, *aa;
550
551         TRACE_ENTRY();
552         
553         scst_suspend_activity();
554         mutex_lock(&scst_mutex);
555
556         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
557                 if (d->scsi_dev == scsidp) {
558                         dev = d;
559                         TRACE_DBG("Target device %p found", dev);
560                         break;
561                 }
562         }
563         if (dev == NULL) {
564                 PRINT_ERROR_PR("%s", "Target device not found");
565                 goto out_unblock;
566         }
567
568         list_del(&dev->dev_list_entry);
569         
570         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
571                                  dev_acg_dev_list_entry) 
572         {
573                 scst_acg_remove_dev(acg_dev->acg, dev);
574         }
575
576         scst_assign_dev_handler(dev, NULL);
577
578         put_disk(dev->rq_disk);
579         scst_free_device(dev);
580
581         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
582                 "id %d, lun %d, type %d", scsidp->host->host_no,
583                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
584
585 out_unblock:
586         mutex_unlock(&scst_mutex);
587         scst_resume_activity();
588
589         TRACE_EXIT();
590         return;
591 }
592
593 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
594 {
595         int res = 0;
596
597         if (dev_handler->parse == NULL) {
598                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
599                         "parse() method.", dev_handler->name);
600                 res = -EINVAL;
601                 goto out;
602         }
603
604         if (dev_handler->exec == NULL)
605                 dev_handler->exec_atomic = 1;
606
607         if (dev_handler->dev_done == NULL)
608                 dev_handler->dev_done_atomic = 1;
609
610 out:
611         TRACE_EXIT_RES(res);
612         return res;
613 }
614
615 int scst_register_virtual_device(struct scst_dev_type *dev_handler, 
616         const char *dev_name)
617 {
618         int res, rc;
619         struct scst_device *dev = NULL;
620
621         TRACE_ENTRY();
622         
623         if (dev_handler == NULL) {
624                 PRINT_ERROR_PR("%s: valid device handler must be supplied", 
625                         __FUNCTION__);
626                 res = -EINVAL;
627                 goto out;
628         }
629         
630         if (dev_name == NULL) {
631                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
632                 res = -EINVAL;
633                 goto out;
634         }
635
636         res = scst_dev_handler_check(dev_handler);
637         if (res != 0)
638                 goto out;
639
640         scst_suspend_activity();
641         if (mutex_lock_interruptible(&scst_mutex) != 0) {
642                 res = -EINTR;
643                 goto out_resume;
644         }
645
646         res = scst_alloc_device(GFP_KERNEL, &dev);
647         if (res != 0)
648                 goto out_up;
649
650         dev->type = dev_handler->type;
651         dev->scsi_dev = NULL;
652         dev->virt_name = dev_name;
653         dev->virt_id = scst_virt_dev_last_id++;
654
655         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
656
657         res = dev->virt_id;
658
659         rc = scst_assign_dev_handler(dev, dev_handler);
660         if (rc != 0) {
661                 res = rc;
662                 goto out_free_del;
663         }
664
665 out_up:
666         mutex_unlock(&scst_mutex);
667
668 out_resume:
669         scst_resume_activity();
670
671 out:
672         if (res > 0) {
673                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
674                     "device %s (id %d)", dev_name, dev->virt_id);
675         } 
676         else {
677                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
678                     "virtual device %s", dev_name);
679         }
680
681         TRACE_EXIT_RES(res);
682         return res;
683
684 out_free_del:
685         list_del(&dev->dev_list_entry);
686         scst_free_device(dev);
687         goto out_up;
688 }
689
690 void scst_unregister_virtual_device(int id)
691 {
692         struct scst_device *d, *dev = NULL;
693         struct scst_acg_dev *acg_dev, *aa;
694
695         TRACE_ENTRY();
696
697         scst_suspend_activity();
698         mutex_lock(&scst_mutex);
699
700         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
701                 if (d->virt_id == id) {
702                         dev = d;
703                         TRACE_DBG("Target device %p found", dev);
704                         break;
705                 }
706         }
707         if (dev == NULL) {
708                 PRINT_ERROR_PR("%s", "Target device not found");
709                 goto out_unblock;
710         }
711
712         list_del(&dev->dev_list_entry);
713         
714         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
715                                  dev_acg_dev_list_entry) 
716         {
717                 scst_acg_remove_dev(acg_dev->acg, dev);
718         }
719
720         scst_assign_dev_handler(dev, NULL);
721
722         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
723                 "(id %d)", dev->virt_name, dev->virt_id);
724
725         scst_free_device(dev);
726
727 out_unblock:
728         mutex_unlock(&scst_mutex);
729         scst_resume_activity();
730
731         TRACE_EXIT();
732         return;
733 }
734
735 int scst_register_dev_driver(struct scst_dev_type *dev_type)
736 {
737         struct scst_dev_type *dt;
738         struct scst_device *dev;
739         int res;
740         int exist;
741
742         TRACE_ENTRY();
743
744         res = scst_dev_handler_check(dev_type);
745         if (res != 0)
746                 goto out_err;
747
748 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
749         if (dev_type->exec == NULL) {
750                 PRINT_ERROR_PR("Pass-through dev handlers (handler \"%s\") not "
751                         "supported. Consider applying on your kernel patch "
752                         "scst_exec_req_fifo-<kernel-version>.patch or define "
753                         "STRICT_SERIALIZING", dev_type->name);
754                 res = -EINVAL;
755                 goto out_err;
756         }
757 #endif
758
759         scst_suspend_activity();
760         if (mutex_lock_interruptible(&scst_mutex) != 0) {
761                 res = -EINTR;
762                 goto out_err;
763         }
764
765         exist = 0;
766         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
767                 if (strcmp(dt->name, dev_type->name) == 0) {
768                         PRINT_ERROR_PR("Device type handler \"%s\" already "
769                                 "exist", dt->name);
770                         exist = 1;
771                         break;
772                 }
773         }
774         if (exist)
775                 goto out_up;
776
777         res = scst_build_proc_dev_handler_dir_entries(dev_type);
778         if (res < 0) {
779                 goto out_up;
780         }
781
782         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
783
784         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
785                 if ((dev->scsi_dev == NULL) || (dev->handler != NULL))
786                         continue;
787                 if (dev->scsi_dev->type == dev_type->type)
788                         scst_assign_dev_handler(dev, dev_type);
789         }
790
791         mutex_unlock(&scst_mutex);
792         scst_resume_activity();
793
794         if (res == 0) {
795                 PRINT_INFO_PR("Device handler \"%s\" for type %d registered "
796                         "successfully", dev_type->name, dev_type->type);
797         }
798
799 out:
800         TRACE_EXIT_RES(res);
801         return res;
802
803 out_up:
804         mutex_unlock(&scst_mutex);
805
806 out_err:
807         scst_resume_activity();
808         PRINT_ERROR_PR("Failed to register device handler \"%s\" for type %d",
809                 dev_type->name, dev_type->type);
810         goto out;
811 }
812
813 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
814 {
815         struct scst_device *dev;
816         struct scst_dev_type *dt;
817         int found = 0;
818
819         TRACE_ENTRY();
820
821         scst_suspend_activity();
822         mutex_lock(&scst_mutex);
823
824         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
825                 if (strcmp(dt->name, dev_type->name) == 0) {
826                         found = 1;
827                         break;
828                 }
829         }
830         if (!found) {
831                 PRINT_ERROR_PR("Dev handler \"%s\" isn't registered",
832                         dev_type->name);
833                 goto out_up;
834         }
835
836         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
837                 if (dev->handler == dev_type) {
838                         scst_assign_dev_handler(dev, NULL);
839                         TRACE_DBG("Dev handler removed from device %p", dev);
840                 }
841         }
842
843         list_del(&dev_type->dev_type_list_entry);
844
845         mutex_unlock(&scst_mutex);
846         scst_resume_activity();
847
848         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
849
850         PRINT_INFO_PR("Device handler \"%s\" for type %d unloaded",
851                    dev_type->name, dev_type->type);
852
853 out:
854         TRACE_EXIT();
855         return;
856
857 out_up:
858         mutex_unlock(&scst_mutex);
859         scst_resume_activity();
860         goto out;
861 }
862
863 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
864 {
865         int res;
866
867         TRACE_ENTRY();
868
869         res = scst_dev_handler_check(dev_type);
870         if (res != 0)
871                 goto out_err;
872
873         if (!dev_type->no_proc) {
874                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
875                 if (res < 0)
876                         goto out_err;
877         }
878
879         if (dev_type->type != -1) {
880                 PRINT_INFO_PR("Virtual device handler %s for type %d "
881                         "registered successfully", dev_type->name,
882                         dev_type->type);
883         } else {
884                 PRINT_INFO_PR("Virtual device handler \"%s\" registered "
885                         "successfully", dev_type->name);
886         }
887
888 out:
889         TRACE_EXIT_RES(res);
890         return res;
891
892 out_err:
893         PRINT_ERROR_PR("Failed to register virtual device handler \"%s\"",
894                 dev_type->name);
895         goto out;
896 }
897
898 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
899 {
900         TRACE_ENTRY();
901
902         if (!dev_type->no_proc)
903                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
904
905         PRINT_INFO_PR("Device handler \"%s\" unloaded", dev_type->name);
906
907         TRACE_EXIT();
908         return;
909 }
910
911 /* Called under scst_mutex and suspended activity */
912 int scst_add_dev_threads(struct scst_device *dev, int num)
913 {
914         int i, res = 0;
915         int n = 0;
916         struct scst_cmd_thread_t *thr;
917         char nm[12];
918
919         TRACE_ENTRY();
920
921         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
922                 n++;
923         }
924
925         for (i = 0; i < num; i++) {
926                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
927                 if (!thr) {
928                         res = -ENOMEM;
929                         PRINT_ERROR_PR("Failed to allocate thr %d", res);
930                         goto out;
931                 }
932                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
933                 nm[ARRAY_SIZE(nm)-1] = '\0';
934                 thr->cmd_thread = kthread_run(scst_cmd_thread,
935                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
936                 if (IS_ERR(thr->cmd_thread)) {
937                         res = PTR_ERR(thr->cmd_thread);
938                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
939                         kfree(thr);
940                         goto out;
941                 }
942                 list_add(&thr->thread_list_entry, &dev->threads_list);
943         }
944
945 out:
946         TRACE_EXIT_RES(res);
947         return res;
948 }
949
950 /* Called under scst_mutex and suspended activity */
951 static int scst_create_dev_threads(struct scst_device *dev)
952 {
953         int res = 0;
954         int threads_num;
955
956         TRACE_ENTRY();
957
958         if (dev->handler->threads_num <= 0)
959                 goto out;
960
961         threads_num = dev->handler->threads_num;
962
963         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
964         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
965         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
966
967         res = scst_add_dev_threads(dev, threads_num);
968         if (res != 0)
969                 goto out;
970
971         mutex_lock(&scst_suspend_mutex);
972         list_add_tail(&dev->cmd_lists.lists_list_entry,
973                 &scst_cmd_lists_list);
974         mutex_unlock(&scst_suspend_mutex);
975
976         dev->p_cmd_lists = &dev->cmd_lists;
977
978 out:
979         TRACE_EXIT_RES(res);
980         return res;
981 }
982
983 /* Called under scst_mutex and suspended activity */
984 void scst_del_dev_threads(struct scst_device *dev, int num)
985 {
986         struct scst_cmd_thread_t *ct, *tmp;
987         int i = 0;
988
989         TRACE_ENTRY();
990
991         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
992                                 thread_list_entry) {
993                 int rc = kthread_stop(ct->cmd_thread);
994                 if (rc < 0) {
995                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
996                 }
997                 list_del(&ct->thread_list_entry);
998                 kfree(ct);
999                 if ((num >0) && (++i >= num))
1000                         break;
1001         }
1002
1003         TRACE_EXIT();
1004         return;
1005 }
1006
1007 /* Called under scst_mutex and suspended activity */
1008 static void scst_stop_dev_threads(struct scst_device *dev)
1009 {
1010         TRACE_ENTRY();
1011
1012         if (list_empty(&dev->threads_list))
1013                 goto out;
1014
1015         scst_del_dev_threads(dev, -1);
1016
1017         if (dev->p_cmd_lists == &dev->cmd_lists) {
1018                 mutex_lock(&scst_suspend_mutex);
1019                 list_del(&dev->cmd_lists.lists_list_entry);
1020                 mutex_unlock(&scst_suspend_mutex);
1021         }
1022
1023 out:
1024         TRACE_EXIT();
1025         return;
1026 }
1027
1028 /* The activity supposed to be suspended and scst_mutex held */
1029 int scst_assign_dev_handler(struct scst_device *dev, 
1030         struct scst_dev_type *handler)
1031 {
1032         int res = 0;
1033         struct scst_tgt_dev *tgt_dev;
1034         LIST_HEAD(attached_tgt_devs);
1035         
1036         TRACE_ENTRY();
1037         
1038         if (dev->handler == handler)
1039                 goto out;
1040         
1041         if (dev->handler && dev->handler->detach_tgt) {
1042                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1043                                 dev_tgt_dev_list_entry) {
1044                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1045                                 tgt_dev);
1046                         dev->handler->detach_tgt(tgt_dev);
1047                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1048                 }
1049         }
1050
1051         if (dev->handler && dev->handler->detach) {
1052                 TRACE_DBG("%s", "Calling dev handler's detach()");
1053                 dev->handler->detach(dev);
1054                 TRACE_DBG("%s", "Old handler's detach() returned");
1055         }
1056
1057         scst_stop_dev_threads(dev);
1058
1059         dev->handler = handler;
1060
1061         if (handler) {
1062                 res = scst_create_dev_threads(dev);
1063                 if (res != 0)
1064                         goto out_null;
1065         }
1066
1067         if (handler && handler->attach) {
1068                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1069                 res = handler->attach(dev);
1070                 TRACE_DBG("New dev handler's attach() returned %d", res);
1071                 if (res != 0) {
1072                         PRINT_ERROR_PR("New device handler's %s attach() "
1073                                 "failed: %d", handler->name, res);
1074                 }
1075                 goto out_thr_null;
1076         }
1077         
1078         if (handler && handler->attach_tgt) {
1079                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
1080                                 dev_tgt_dev_list_entry) {
1081                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1082                                 tgt_dev);
1083                         res = handler->attach_tgt(tgt_dev);
1084                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1085                         if (res != 0) {
1086                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
1087                                     "failed: %d", handler->name, res);
1088                                 goto out_err_detach_tgt;
1089                         }
1090                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1091                                 &attached_tgt_devs);
1092                 }
1093         }
1094
1095 out_thr_null:
1096         if (res != 0)
1097                 scst_stop_dev_threads(dev);
1098
1099 out_null:
1100         if (res != 0)
1101                 dev->handler = NULL;
1102         
1103 out:
1104         TRACE_EXIT_RES(res);
1105         return res;
1106
1107 out_err_detach_tgt:
1108         if (handler && handler->detach_tgt) {
1109                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1110                                  extra_tgt_dev_list_entry) 
1111                 {
1112                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1113                                 tgt_dev);
1114                         handler->detach_tgt(tgt_dev);
1115                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1116                 }
1117         }
1118         if (handler && handler->detach) {
1119                 TRACE_DBG("%s", "Calling handler's detach()");
1120                 handler->detach(dev);
1121                 TRACE_DBG("%s", "Handler's detach() returned");
1122         }
1123         goto out_null;
1124 }
1125
1126 int scst_cmd_threads_count(void)
1127 {
1128         int i;
1129
1130         /* Just to lower the race window, when user can get just changed value */
1131         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1132         i = scst_threads_info.nr_cmd_threads;
1133         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1134         return i;
1135 }
1136
1137 static void scst_threads_info_init(void)
1138 {
1139         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1140         mutex_init(&scst_threads_info.cmd_threads_mutex);
1141         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1142 }
1143
1144 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1145 void __scst_del_cmd_threads(int num)
1146 {
1147         struct scst_cmd_thread_t *ct, *tmp;
1148         int i;
1149
1150         TRACE_ENTRY();
1151
1152         i = scst_threads_info.nr_cmd_threads;
1153         if (num <= 0 || num > i) {
1154                 PRINT_ERROR_PR("can not del %d cmd threads from %d", num, i);
1155                 return;
1156         }
1157
1158         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1159                                 thread_list_entry) {
1160                 int res;
1161
1162                 res = kthread_stop(ct->cmd_thread);
1163                 if (res < 0) {
1164                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1165                 }
1166                 list_del(&ct->thread_list_entry);
1167                 kfree(ct);
1168                 scst_threads_info.nr_cmd_threads--;
1169                 --num;
1170                 if (num == 0)
1171                         break;
1172         }
1173
1174         TRACE_EXIT();
1175         return;
1176 }
1177
1178 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1179 int __scst_add_cmd_threads(int num)
1180 {
1181         int res = 0, i;
1182         static int scst_thread_num = 0;
1183         
1184         TRACE_ENTRY();
1185
1186         for (i = 0; i < num; i++) {
1187                 struct scst_cmd_thread_t *thr;
1188
1189                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1190                 if (!thr) {
1191                         res = -ENOMEM;
1192                         PRINT_ERROR_PR("fail to allocate thr %d", res);
1193                         goto out_error;
1194                 }
1195                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1196                         &scst_main_cmd_lists, "scsi_tgt%d",
1197                         scst_thread_num++);
1198                 if (IS_ERR(thr->cmd_thread)) {
1199                         res = PTR_ERR(thr->cmd_thread);
1200                         PRINT_ERROR_PR("kthread_create() failed: %d", res);
1201                         kfree(thr);
1202                         goto out_error;
1203                 }
1204                 list_add(&thr->thread_list_entry,
1205                         &scst_threads_info.cmd_threads_list);
1206                 scst_threads_info.nr_cmd_threads++;
1207         }
1208         res = 0;
1209
1210 out:
1211         TRACE_EXIT_RES(res);
1212         return res;
1213
1214 out_error:
1215         if (i > 0)
1216                 __scst_del_cmd_threads(i - 1);
1217         goto out;
1218 }
1219
1220 int scst_add_cmd_threads(int num)
1221 {
1222         int res;
1223
1224         TRACE_ENTRY();
1225
1226         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1227         res = __scst_add_cmd_threads(num);
1228         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1229
1230         TRACE_EXIT_RES(res);
1231         return res;
1232 }
1233
1234 void scst_del_cmd_threads(int num)
1235 {
1236         TRACE_ENTRY();
1237
1238         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1239         __scst_del_cmd_threads(num);
1240         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1241
1242         TRACE_EXIT();
1243         return;
1244 }
1245
1246 static void scst_stop_all_threads(void)
1247 {
1248         TRACE_ENTRY();
1249
1250         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1251         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1252         if (scst_threads_info.mgmt_cmd_thread)
1253                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1254         if (scst_threads_info.mgmt_thread)
1255                 kthread_stop(scst_threads_info.mgmt_thread);
1256         if (scst_threads_info.init_cmd_thread)
1257                 kthread_stop(scst_threads_info.init_cmd_thread);
1258         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1259
1260         TRACE_EXIT();
1261         return;
1262 }
1263
1264 static int scst_start_all_threads(int num)
1265 {
1266         int res;
1267
1268         TRACE_ENTRY();
1269
1270         mutex_lock(&scst_threads_info.cmd_threads_mutex);               
1271         res = __scst_add_cmd_threads(num);
1272         if (res < 0)
1273                 goto out;
1274
1275         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1276                 NULL, "scsi_tgt_init");
1277         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1278                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1279                 PRINT_ERROR_PR("kthread_create() for init cmd failed: %d", res);
1280                 scst_threads_info.init_cmd_thread = NULL;
1281                 goto out;
1282         }
1283
1284         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1285                 NULL, "scsi_tgt_mc");
1286         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1287                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1288                 PRINT_ERROR_PR("kthread_create() for mcmd failed: %d", res);
1289                 scst_threads_info.mgmt_cmd_thread = NULL;
1290                 goto out;
1291         }
1292
1293         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1294                 NULL, "scsi_tgt_mgmt");
1295         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1296                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1297                 PRINT_ERROR_PR("kthread_create() for mgmt failed: %d", res);
1298                 scst_threads_info.mgmt_thread = NULL;
1299                 goto out;
1300         }
1301
1302 out:
1303         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1304         TRACE_EXIT_RES(res);
1305         return res;     
1306 }
1307
1308 void scst_get(void)
1309 {
1310         __scst_get(0);
1311 }
1312
1313 void scst_put(void)
1314 {
1315         __scst_put();
1316 }
1317
1318 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1319 static int scst_add(struct class_device *cdev)
1320 #else
1321 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1322 #endif
1323 {
1324         struct scsi_device *scsidp;
1325         int res = 0;
1326
1327         TRACE_ENTRY();
1328         
1329         scsidp = to_scsi_device(cdev->dev);
1330         res = scst_register_device(scsidp);
1331
1332         TRACE_EXIT();
1333         return res;
1334 }
1335
1336 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
1337 static void scst_remove(struct class_device *cdev)
1338 #else
1339 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1340 #endif
1341 {
1342         struct scsi_device *scsidp;
1343
1344         TRACE_ENTRY();
1345
1346         scsidp = to_scsi_device(cdev->dev);
1347         scst_unregister_device(scsidp);
1348
1349         TRACE_EXIT();
1350         return;
1351 }
1352
1353 static struct class_interface scst_interface = {
1354         .add = scst_add,
1355         .remove = scst_remove,
1356 };
1357
1358 static int __init init_scst(void)
1359 {
1360         int res = 0, i;
1361         struct scst_cmd *cmd;
1362         int scst_num_cpus;
1363
1364         TRACE_ENTRY();
1365
1366 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1367         {
1368                 struct scsi_request *req;
1369                 BUILD_BUG_ON(sizeof(cmd->sense_buffer) !=
1370                         sizeof(req->sr_sense_buffer));
1371         }
1372 #else
1373         {
1374                 struct scsi_sense_hdr *shdr;
1375                 BUILD_BUG_ON((sizeof(cmd->sense_buffer) < sizeof(*shdr)) &&
1376                         (sizeof(cmd->sense_buffer) >= SCST_SENSE_BUFFERSIZE));
1377         }
1378 #endif
1379         {
1380                 struct scst_tgt_dev *t;
1381                 struct scst_cmd *c;
1382                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1383                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1384         }
1385
1386         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1387         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1388         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1389         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1390
1391         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1392         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1393         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1394         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1395                 &scst_cmd_lists_list);
1396
1397         scst_num_cpus = num_online_cpus();
1398
1399         /* ToDo: register_cpu_notifier() */
1400         
1401         if (scst_threads == 0)
1402                 scst_threads = scst_num_cpus;
1403                 
1404         if (scst_threads < scst_num_cpus) {
1405                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
1406                         "CPUs count");
1407                 scst_threads = scst_num_cpus;
1408         }
1409
1410         scst_threads_info_init();
1411
1412 #define INIT_CACHEP(p, s, t, o) do {                                    \
1413                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
1414                                       SCST_SLAB_FLAGS, NULL, NULL);     \
1415                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
1416                           sizeof(struct t));                            \
1417                 if (p == NULL) { res = -ENOMEM; goto o; }               \
1418         } while (0)
1419           
1420         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING, 
1421                     scst_mgmt_cmd, out);
1422         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING, 
1423                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
1424         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING, 
1425                     scst_cmd, out_destroy_ua_cache);
1426         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
1427                     scst_session, out_destroy_cmd_cache);
1428         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
1429                     scst_tgt_dev, out_destroy_sess_cache);
1430         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
1431                     scst_acg_dev, out_destroy_tgt_cache);
1432
1433         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
1434                 mempool_free_slab, scst_mgmt_cachep);
1435         if (scst_mgmt_mempool == NULL) {
1436                 res = -ENOMEM;
1437                 goto out_destroy_acg_cache;
1438         }
1439
1440         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1441                 mempool_free_slab, scst_ua_cachep);
1442         if (scst_ua_mempool == NULL) {
1443                 res = -ENOMEM;
1444                 goto out_destroy_mgmt_mempool;
1445         }
1446
1447         if (scst_max_cmd_mem == 0) {
1448                 struct sysinfo si;
1449                 si_meminfo(&si);
1450 #if BITS_PER_LONG == 32
1451                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1452                                         (uint64_t)1 << 30);
1453 #else
1454                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1455 #endif
1456         } else
1457                 scst_max_cmd_mem <<= 20;
1458
1459         res = scst_sgv_pools_init(scst_max_cmd_mem, 0);
1460         if (res != 0)
1461                 goto out_destroy_ua_mempool;
1462
1463         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1464         if (scst_default_acg == NULL) {
1465                 res = -ENOMEM;
1466                 goto out_destroy_sgv_pool;
1467         }
1468         
1469         res = scsi_register_interface(&scst_interface);
1470         if (res != 0)
1471                 goto out_free_acg;
1472
1473         scst_scsi_op_list_init();
1474
1475         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1476                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1477                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1478                 tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet,
1479                         (unsigned long)&scst_tasklets[i]);
1480         }
1481
1482         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1483                 scst_threads);
1484
1485         res = scst_start_all_threads(scst_threads);
1486         if (res < 0)
1487                 goto out_thread_free;
1488
1489         res = scst_proc_init_module();
1490         if (res != 0)
1491                 goto out_thread_free;
1492
1493
1494         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1495                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1496
1497 out:
1498         TRACE_EXIT_RES(res);
1499         return res;
1500
1501 out_thread_free:
1502         scst_stop_all_threads();
1503
1504         scsi_unregister_interface(&scst_interface);
1505
1506 out_free_acg:
1507         scst_destroy_acg(scst_default_acg);
1508
1509 out_destroy_sgv_pool:
1510         scst_sgv_pools_deinit();
1511
1512 out_destroy_ua_mempool:
1513         mempool_destroy(scst_ua_mempool);
1514
1515 out_destroy_mgmt_mempool:
1516         mempool_destroy(scst_mgmt_mempool);
1517
1518 out_destroy_acg_cache:
1519         kmem_cache_destroy(scst_acgd_cachep);
1520
1521 out_destroy_tgt_cache:
1522         kmem_cache_destroy(scst_tgtd_cachep);
1523
1524 out_destroy_sess_cache:
1525         kmem_cache_destroy(scst_sess_cachep);
1526
1527 out_destroy_cmd_cache:
1528         kmem_cache_destroy(scst_cmd_cachep);
1529
1530 out_destroy_ua_cache:
1531         kmem_cache_destroy(scst_ua_cachep);
1532
1533 out_destroy_mgmt_cache:
1534         kmem_cache_destroy(scst_mgmt_cachep);
1535         goto out;
1536 }
1537
1538 static void __exit exit_scst(void)
1539 {
1540 #ifdef CONFIG_LOCKDEP
1541         static /* To hide the lockdep's warning about non-static key */
1542 #endif
1543         DECLARE_MUTEX_LOCKED(shm);
1544
1545         TRACE_ENTRY();
1546         
1547         /* ToDo: unregister_cpu_notifier() */
1548
1549         scst_proc_cleanup_module();
1550
1551         scst_stop_all_threads();
1552
1553         scsi_unregister_interface(&scst_interface);
1554         scst_destroy_acg(scst_default_acg);
1555
1556         scst_sgv_pools_deinit();
1557
1558 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
1559 #define DEINIT_CACHEP(p, s) do {                        \
1560                 if (kmem_cache_destroy(p)) {            \
1561                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1562                                 "error", s);            \
1563                 }                                       \
1564                 p = NULL;                               \
1565         } while (0)
1566 #else
1567 #define DEINIT_CACHEP(p, s) do {                        \
1568                 kmem_cache_destroy(p);                  \
1569                 p = NULL;                               \
1570         } while (0)
1571 #endif
1572
1573         mempool_destroy(scst_mgmt_mempool);
1574         mempool_destroy(scst_ua_mempool);
1575
1576         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1577         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1578         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1579         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1580         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1581         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1582
1583         PRINT_INFO_PR("%s", "SCST unloaded");
1584
1585         TRACE_EXIT();
1586         return;
1587 }
1588
1589 /*
1590  * Device Handler Side (i.e. scst_vdisk)
1591  */
1592 EXPORT_SYMBOL(scst_register_dev_driver);
1593 EXPORT_SYMBOL(scst_unregister_dev_driver);
1594 EXPORT_SYMBOL(scst_register);
1595 EXPORT_SYMBOL(scst_unregister);
1596
1597 EXPORT_SYMBOL(scst_register_virtual_device);
1598 EXPORT_SYMBOL(scst_unregister_virtual_device);
1599 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1600 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1601
1602 EXPORT_SYMBOL(scst_set_busy);
1603 EXPORT_SYMBOL(scst_set_cmd_error_status);
1604 EXPORT_SYMBOL(scst_set_cmd_error);
1605 EXPORT_SYMBOL(scst_set_resp_data_len);
1606
1607 EXPORT_SYMBOL(scst_process_active_cmd);
1608
1609 /*
1610  * Target Driver Side (i.e. HBA)
1611  */
1612 EXPORT_SYMBOL(scst_register_session);
1613 EXPORT_SYMBOL(scst_unregister_session);
1614
1615 EXPORT_SYMBOL(scst_register_target_template);
1616 EXPORT_SYMBOL(scst_unregister_target_template);
1617
1618 EXPORT_SYMBOL(scst_cmd_init_done);
1619 EXPORT_SYMBOL(scst_tgt_cmd_done);
1620 EXPORT_SYMBOL(scst_restart_cmd);
1621 EXPORT_SYMBOL(scst_rx_cmd);
1622 EXPORT_SYMBOL(scst_rx_data);
1623 EXPORT_SYMBOL(scst_rx_mgmt_fn);
1624
1625 EXPORT_SYMBOL(scst_find_cmd);
1626 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1627
1628 /*
1629  * Global Commands
1630  */
1631 EXPORT_SYMBOL(scst_suspend_activity);
1632 EXPORT_SYMBOL(scst_resume_activity);
1633
1634 EXPORT_SYMBOL(scst_add_cmd_threads);
1635 EXPORT_SYMBOL(scst_del_cmd_threads);
1636
1637 #if defined(DEBUG) || defined(TRACING)
1638 EXPORT_SYMBOL(scst_proc_log_entry_read);
1639 EXPORT_SYMBOL(scst_proc_log_entry_write);
1640 #endif
1641
1642 EXPORT_SYMBOL(scst_create_proc_entry);
1643 EXPORT_SYMBOL(scst_single_seq_open);
1644
1645 EXPORT_SYMBOL(__scst_get_buf);
1646 EXPORT_SYMBOL(scst_get);
1647 EXPORT_SYMBOL(scst_put);
1648
1649 EXPORT_SYMBOL(scst_alloc);
1650 EXPORT_SYMBOL(scst_free);
1651
1652 EXPORT_SYMBOL(scst_check_local_events);
1653
1654 /* Tgt_dev's threads local storage */
1655 EXPORT_SYMBOL(scst_add_thr_data);
1656 EXPORT_SYMBOL(scst_del_all_thr_data);
1657 EXPORT_SYMBOL(scst_dev_del_all_thr_data);
1658 EXPORT_SYMBOL(scst_find_thr_data);
1659
1660 /* SGV pool routines */
1661 EXPORT_SYMBOL(sgv_pool_create);
1662 EXPORT_SYMBOL(sgv_pool_destroy);
1663 EXPORT_SYMBOL(sgv_pool_set_allocator);
1664 EXPORT_SYMBOL(sgv_pool_alloc);
1665 EXPORT_SYMBOL(sgv_pool_free);
1666 EXPORT_SYMBOL(sgv_get_priv);
1667
1668 /* Generic parse() routines */
1669 EXPORT_SYMBOL(scst_calc_block_shift);
1670 EXPORT_SYMBOL(scst_sbc_generic_parse);
1671 EXPORT_SYMBOL(scst_cdrom_generic_parse);
1672 EXPORT_SYMBOL(scst_modisk_generic_parse);
1673 EXPORT_SYMBOL(scst_tape_generic_parse);
1674 EXPORT_SYMBOL(scst_changer_generic_parse);
1675 EXPORT_SYMBOL(scst_processor_generic_parse);
1676 EXPORT_SYMBOL(scst_raid_generic_parse);
1677
1678 /* Generic dev_done() routines */
1679 EXPORT_SYMBOL(scst_block_generic_dev_done);
1680 EXPORT_SYMBOL(scst_tape_generic_dev_done);
1681
1682 /*
1683  * Other Commands
1684  */
1685 EXPORT_SYMBOL(scst_get_cdb_info);
1686 EXPORT_SYMBOL(scst_cmd_get_tgt_priv_lock);
1687 EXPORT_SYMBOL(scst_cmd_set_tgt_priv_lock);
1688
1689 #ifdef DEBUG
1690 EXPORT_SYMBOL(scst_random);
1691 #endif
1692
1693 module_init(init_scst);
1694 module_exit(exit_scst);
1695
1696 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1697 MODULE_LICENSE("GPL");
1698 MODULE_DESCRIPTION("SCSI target core");
1699 MODULE_VERSION(SCST_VERSION_STRING);