More intelligent IO flow control implemented
[mirror/scst/.git] / scst / src / scst.c
1 /*
2  *  scst.c
3  *  
4  *  Copyright (C) 2004-2006 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <asm/unistd.h>
28 #include <asm/string.h>
29
30 #include "scst_debug.h"
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #include "scst_debug.c"
36
37 #if defined(DEBUG) || defined(TRACING)
38 unsigned long trace_flag = SCST_DEFAULT_LOG_FLAGS;
39 #endif
40
41 /*
42  * All targets, devices and dev_types management is done under
43  * this mutex.
44  */
45 DECLARE_MUTEX(scst_mutex);
46
47 int scst_num_cpus;
48 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
49 LIST_HEAD(scst_dev_wait_sess_list);
50
51 LIST_HEAD(scst_template_list);
52 LIST_HEAD(scst_dev_list);
53 LIST_HEAD(scst_dev_type_list);
54
55 kmem_cache_t *scst_mgmt_cachep;
56 mempool_t *scst_mgmt_mempool;
57 kmem_cache_t *scst_ua_cachep;
58 mempool_t *scst_ua_mempool;
59 kmem_cache_t *scst_tgtd_cachep;
60 kmem_cache_t *scst_sess_cachep;
61 kmem_cache_t *scst_acgd_cachep;
62
63 LIST_HEAD(scst_acg_list);
64 struct scst_acg *scst_default_acg;
65
66 kmem_cache_t *scst_cmd_cachep;
67
68 unsigned long scst_flags;
69 atomic_t scst_cmd_count = ATOMIC_INIT(0);
70 spinlock_t scst_list_lock = SPIN_LOCK_UNLOCKED;
71 LIST_HEAD(scst_active_cmd_list);
72 LIST_HEAD(scst_init_cmd_list);
73 LIST_HEAD(scst_cmd_list);
74 DECLARE_WAIT_QUEUE_HEAD(scst_list_waitQ);
75
76 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
77 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
78
79 struct tasklet_struct scst_tasklets[NR_CPUS];
80
81 struct scst_sgv_pools scst_sgv;
82
83 DECLARE_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn, 0);
84
85 unsigned long scst_max_cmd_mem;
86
87 LIST_HEAD(scst_mgmt_cmd_list);
88 LIST_HEAD(scst_active_mgmt_cmd_list);
89 LIST_HEAD(scst_delayed_mgmt_cmd_list);
90 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
91
92 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
93 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
94 LIST_HEAD(scst_sess_mgmt_list);
95
96 struct semaphore *scst_shutdown_mutex = NULL;
97
98 int scst_threads;
99 atomic_t scst_threads_count = ATOMIC_INIT(0);
100 int scst_shut_threads_count;
101 int scst_thread_num;
102 static int suspend_count;
103
104 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
105
106 /* 
107  * This buffer and lock are intended to avoid memory allocation, which
108  * could fail in improper places.
109  */
110 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
111 uint8_t scst_temp_UA[SCSI_SENSE_BUFFERSIZE];
112
113 module_param_named(scst_threads, scst_threads, int, 0);
114 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
115
116 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
117 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
118         "the SCST commands at any given time in Mb");
119
120 int scst_register_target_template(struct scst_tgt_template *vtt)
121 {
122         int res = 0;
123         struct scst_tgt_template *t;
124
125         TRACE_ENTRY();
126
127         INIT_LIST_HEAD(&vtt->tgt_list);
128
129         if (!vtt->detect) {
130                 PRINT_ERROR_PR("Target driver %s doesn't have a "
131                         "detect() method.", vtt->name);
132                 res = -EINVAL;
133                 goto out;
134         }
135         
136         if (!vtt->release) {
137                 PRINT_ERROR_PR("Target driver %s doesn't have a "
138                         "release() method.", vtt->name);
139                 res = -EINVAL;
140                 goto out;
141         }
142
143         if (!vtt->xmit_response) {
144                 PRINT_ERROR_PR("Target driver %s doesn't have a "
145                         "xmit_response() method.", vtt->name);
146                 res = -EINVAL;
147                 goto out;
148         }
149
150         if (!vtt->rdy_to_xfer) {
151                 PRINT_ERROR_PR("Target driver %s doesn't have a "
152                         "rdy_to_xfer() method.", vtt->name);
153                 res = -EINVAL;
154                 goto out;
155         }
156         
157         if (!vtt->on_free_cmd) {
158                 PRINT_ERROR_PR("Target driver %s doesn't have a "
159                         "on_free_cmd() method.", vtt->name);
160                 res = -EINVAL;
161                 goto out;
162         }
163
164         if (!vtt->no_proc_entry) {
165                 res = scst_build_proc_target_dir_entries(vtt);
166                 if (res < 0) {
167                         goto out;
168                 }
169         }
170
171         if (down_interruptible(&scst_mutex) != 0)
172                 goto out;
173         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
174                 if (strcmp(t->name, vtt->name) == 0) {
175                         PRINT_ERROR_PR("Target driver %s already registered",
176                                 vtt->name);
177                         up(&scst_mutex);
178                         goto out_cleanup;
179                 }
180         }
181         /* That's OK to drop it. The race doesn't matter */
182         up(&scst_mutex);
183
184         TRACE_DBG("%s", "Calling target driver's detect()");
185         res = vtt->detect(vtt);
186         TRACE_DBG("Target driver's detect() returned %d", res);
187         if (res < 0) {
188                 PRINT_ERROR_PR("%s", "The detect() routine failed");
189                 res = -EINVAL;
190                 goto out_cleanup;
191         }
192
193         down(&scst_mutex);
194         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
195         up(&scst_mutex);
196
197         res = 0;
198
199 out:
200         TRACE_EXIT_RES(res);
201         return res;
202
203 out_cleanup:
204         scst_cleanup_proc_target_dir_entries(vtt);
205         goto out;
206 }
207
208 void scst_unregister_target_template(struct scst_tgt_template *vtt)
209 {
210         struct scst_tgt *tgt;
211
212         TRACE_ENTRY();
213
214         if (down_interruptible(&scst_mutex) != 0)
215                 goto out;
216
217 restart:
218         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
219                 up(&scst_mutex);
220                 scst_unregister(tgt);
221                 down(&scst_mutex);
222                 goto restart;
223         }
224         list_del(&vtt->scst_template_list_entry);
225         up(&scst_mutex);
226
227         scst_cleanup_proc_target_dir_entries(vtt);
228
229 out:
230         TRACE_EXIT();
231         return;
232 }
233
234 struct scst_tgt *scst_register(struct scst_tgt_template *vtt)
235 {
236         struct scst_tgt *tgt;
237
238         TRACE_ENTRY();
239
240         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
241         TRACE_MEM("kzalloc(GFP_KERNEL) for tgt (%zd): %p",
242               sizeof(*tgt), tgt);
243         if (tgt == NULL) {
244                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
245                 goto out;
246         }
247
248         INIT_LIST_HEAD(&tgt->sess_list);
249         init_waitqueue_head(&tgt->unreg_waitQ);
250         tgt->tgtt = vtt;
251         tgt->sg_tablesize = vtt->sg_tablesize;
252         spin_lock_init(&tgt->tgt_lock);
253         INIT_LIST_HEAD(&tgt->retry_cmd_list);
254         atomic_set(&tgt->finished_cmds, 0);
255         init_timer(&tgt->retry_timer);
256         tgt->retry_timer.data = (unsigned long)tgt;
257         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
258
259         down(&scst_mutex);
260
261         if (scst_build_proc_target_entries(tgt) < 0) {
262                 tgt = NULL;
263                 goto out_free;
264         }
265
266         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
267
268         up(&scst_mutex);
269
270 out:
271         TRACE_EXIT();
272         return tgt;
273
274 out_free:
275         TRACE_MEM("kfree() for tgt %p", tgt);
276         kfree(tgt);
277         goto out;
278 }
279
280 static inline int test_sess_list(struct scst_tgt *tgt)
281 {
282         int res;
283         down(&scst_mutex);
284         res = list_empty(&tgt->sess_list);
285         up(&scst_mutex);
286         return res;
287 }
288
289 void scst_unregister(struct scst_tgt *tgt)
290 {
291         struct scst_session *sess;
292
293         TRACE_ENTRY();
294
295         TRACE_DBG("%s", "Calling target driver's release()");
296         tgt->tgtt->release(tgt);
297         TRACE_DBG("%s", "Target driver's release() returned");
298
299         down(&scst_mutex);
300         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
301                 BUG_ON(!sess->shutting_down);
302         }
303         up(&scst_mutex);
304
305         TRACE_DBG("%s", "Waiting for sessions shutdown");
306         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
307         TRACE_DBG("%s", "wait_event() returned");
308
309         down(&scst_mutex);
310         list_del(&tgt->tgt_list_entry);
311         up(&scst_mutex);
312
313         scst_cleanup_proc_target_entries(tgt);
314
315         del_timer_sync(&tgt->retry_timer);
316
317         TRACE_MEM("kfree for tgt: %p", tgt);
318         kfree(tgt);
319
320         TRACE_EXIT();
321         return;
322 }
323
324 /* scst_mutex supposed to be held */
325 void __scst_suspend_activity(void)
326 {
327         TRACE_ENTRY();
328
329         suspend_count++;
330         if (suspend_count > 1)
331                 goto out;
332
333         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
334         smp_mb__after_set_bit();
335
336         TRACE_DBG("Waiting for all %d active commands to complete",
337               atomic_read(&scst_cmd_count));
338         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
339         TRACE_DBG("%s", "wait_event() returned");
340
341 out:
342         TRACE_EXIT();
343         return;
344 }
345
346 void scst_suspend_activity(void)
347 {
348         down(&scst_mutex);
349         __scst_suspend_activity();
350 }
351
352 /* scst_mutex supposed to be held */
353 void __scst_resume_activity(void)
354 {
355         struct scst_session *sess, *tsess;
356
357         TRACE_ENTRY();
358
359         suspend_count--;
360         if (suspend_count > 0)
361                 goto out;
362
363         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
364         smp_mb__after_clear_bit();
365
366         spin_lock_irq(&scst_list_lock);
367         list_for_each_entry_safe(sess, tsess, &scst_dev_wait_sess_list,
368                             dev_wait_sess_list_entry) 
369         {
370                 sess->waiting = 0;
371                 list_del(&sess->dev_wait_sess_list_entry);
372         }
373         spin_unlock_irq(&scst_list_lock);
374
375         wake_up_all(&scst_list_waitQ);
376         wake_up_all(&scst_mgmt_cmd_list_waitQ);
377
378 out:
379         TRACE_EXIT();
380         return;
381 }
382
383 void scst_resume_activity(void)
384 {
385         __scst_resume_activity();
386         up(&scst_mutex);
387 }
388
389 /* Called under scst_mutex */
390 static int scst_register_device(struct scsi_device *scsidp)
391 {
392         int res = 0;
393         struct scst_device *dev;
394         struct scst_dev_type *dt;
395
396         TRACE_ENTRY();
397
398         dev = scst_alloc_device(GFP_KERNEL);
399         if (dev == NULL) {
400                 res = -ENOMEM;
401                 goto out;
402         }
403         
404         dev->rq_disk = alloc_disk(1);
405         if (dev->rq_disk == NULL) {
406                 res = -ENOMEM;
407                 scst_free_device(dev);
408                 goto out;
409         }
410         dev->rq_disk->major = SCST_MAJOR;
411
412         dev->scsi_dev = scsidp;
413
414         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
415         
416         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
417                 if (dt->type == scsidp->type) {
418                         res = scst_assign_dev_handler(dev, dt);
419                         if (res != 0)
420                                 goto out_free;
421                         break;
422                 }
423         }
424
425 out:
426         if (res == 0) {
427                 PRINT_INFO_PR("Attached SCSI target mid-level at "
428                     "scsi%d, channel %d, id %d, lun %d, type %d", 
429                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
430                     scsidp->lun, scsidp->type);
431         } 
432         else {
433                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
434                     "at scsi%d, channel %d, id %d, lun %d, type %d", 
435                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
436                     scsidp->lun, scsidp->type);
437         }
438
439         TRACE_EXIT_RES(res);
440         return res;
441
442 out_free:
443         put_disk(dev->rq_disk);
444
445         list_del(&dev->dev_list_entry);
446         scst_assign_dev_handler(dev, NULL);
447         goto out;
448 }
449
450 /* Called under scst_mutex */
451 static void scst_unregister_device(struct scsi_device *scsidp)
452 {
453         struct scst_device *d, *dev = NULL;
454         struct scst_acg_dev *acg_dev, *aa;
455
456         TRACE_ENTRY();
457         
458         __scst_suspend_activity();
459
460         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
461                 if (d->scsi_dev == scsidp) {
462                         dev = d;
463                         TRACE_DBG("Target device %p found", dev);
464                         break;
465                 }
466         }
467
468         if (dev == NULL) {
469                 PRINT_ERROR_PR("%s", "Target device not found");
470                 goto out_unblock;
471         }
472
473         list_del(&dev->dev_list_entry);
474         
475         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
476                                  dev_acg_dev_list_entry) 
477         {
478                 scst_acg_remove_dev(acg_dev->acg, dev);
479         }
480
481         scst_assign_dev_handler(dev, NULL);
482
483         put_disk(dev->rq_disk);
484         scst_free_device(dev);
485
486 out_unblock:
487         __scst_resume_activity();
488
489         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
490              "id %d, lun %d, type %d", scsidp->host->host_no, scsidp->channel, 
491              scsidp->id, scsidp->lun, scsidp->type);
492
493         TRACE_EXIT();
494         return;
495 }
496
497 int scst_register_virtual_device(struct scst_dev_type *dev_handler, 
498         const char *dev_name)
499 {
500         int res = -EINVAL, rc;
501         struct scst_device *dev = NULL;
502
503         TRACE_ENTRY();
504         
505         if (dev_handler == NULL) {
506                 PRINT_ERROR_PR("%s: valid device handler must be supplied", 
507                         __FUNCTION__);
508                 res = -EINVAL;
509                 goto out;
510         }
511         
512         if (dev_name == NULL) {
513                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
514                 res = -EINVAL;
515                 goto out;
516         }
517
518         dev = scst_alloc_device(GFP_KERNEL);
519         if (dev == NULL) {
520                 res = -ENOMEM;
521                 goto out;
522         }
523
524         dev->scsi_dev = NULL;
525         dev->virt_name = dev_name;
526
527         if (down_interruptible(&scst_mutex) != 0) {
528                 res = -EINTR;
529                 goto out_free_dev;
530         }
531
532         dev->virt_id = scst_virt_dev_last_id++;
533
534         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
535
536         res = dev->virt_id;
537
538         rc = scst_assign_dev_handler(dev, dev_handler);
539         if (rc != 0) {
540                 res = rc;
541                 goto out_free_del;
542         }
543         
544         up(&scst_mutex);
545
546 out:
547         if (res > 0) {
548                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
549                     "device %s (id %d)", dev_name, dev->virt_id);
550         } 
551         else {
552                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
553                     "virtual device %s", dev_name);
554         }
555
556         TRACE_EXIT_RES(res);
557         return res;
558
559 out_free_del:
560         list_del(&dev->dev_list_entry);
561         up(&scst_mutex);
562
563 out_free_dev:
564         scst_free_device(dev);
565         goto out;
566 }
567
568 void scst_unregister_virtual_device(int id)
569 {
570         struct scst_device *d, *dev = NULL;
571         struct scst_acg_dev *acg_dev, *aa;
572
573         TRACE_ENTRY();
574
575         if (down_interruptible(&scst_mutex) != 0)
576                 goto out;
577
578         __scst_suspend_activity();
579
580         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
581                 if (d->virt_id == id) {
582                         dev = d;
583                         TRACE_DBG("Target device %p found", dev);
584                         break;
585                 }
586         }
587
588         if (dev == NULL) {
589                 PRINT_ERROR_PR("%s", "Target device not found");
590                 goto out_unblock;
591         }
592
593         list_del(&dev->dev_list_entry);
594         
595         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
596                                  dev_acg_dev_list_entry) 
597         {
598                 scst_acg_remove_dev(acg_dev->acg, dev);
599         }
600
601         scst_assign_dev_handler(dev, NULL);
602
603         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
604                 "(id %d)", dev->virt_name, dev->virt_id);
605
606         scst_free_device(dev);
607
608 out_unblock:
609         __scst_resume_activity();
610
611         up(&scst_mutex);
612
613 out:
614         TRACE_EXIT();
615         return;
616 }
617
618 int scst_register_dev_driver(struct scst_dev_type *dev_type)
619 {
620         struct scst_dev_type *dt;
621         struct scst_device *dev;
622         int res = 0;
623         int exist;
624
625         TRACE_ENTRY();
626
627         if (dev_type->parse == NULL) {
628                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
629                         "parse() method.", dev_type->name);
630                 res = -EINVAL;
631                 goto out;
632         }
633
634         if (down_interruptible(&scst_mutex) != 0) {
635                 res = -EINTR;
636                 goto out;
637         }
638
639         exist = 0;
640         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
641                 if (dt->type == dev_type->type) {
642                         TRACE_DBG("Device type handler for type %d "
643                                 "already exist", dt->type);
644                         exist = 1;
645                         break;
646                 }
647         }
648         
649         res = scst_build_proc_dev_handler_dir_entries(dev_type);
650         if (res < 0) {
651                 goto out_up;
652         }
653         
654         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
655         
656         if (exist)
657                 goto out_up;
658
659         __scst_suspend_activity();
660         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
661                 if (dev->scsi_dev == NULL)
662                         continue;
663                 if (dev->scsi_dev->type == dev_type->type)
664                         scst_assign_dev_handler(dev, dev_type);
665         }
666         __scst_resume_activity();
667
668 out_up:
669         up(&scst_mutex);
670
671         if (res == 0) {
672                 PRINT_INFO_PR("Device handler %s for type %d loaded "
673                         "successfully", dev_type->name, dev_type->type);
674         }
675
676 out:
677         TRACE_EXIT_RES(res);
678         return res;
679 }
680
681 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
682 {
683         struct scst_device *dev;
684
685         TRACE_ENTRY();
686
687         if (down_interruptible(&scst_mutex) != 0)
688                 goto out;
689
690         __scst_suspend_activity();
691         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
692                 if (dev->handler == dev_type) {
693                         scst_assign_dev_handler(dev, NULL);
694                         TRACE_DBG("Dev handler removed from device %p", dev);
695                 }
696         }
697         __scst_resume_activity();
698
699         list_del(&dev_type->dev_type_list_entry);
700
701         up(&scst_mutex);
702
703         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
704
705         PRINT_INFO_PR("Device handler %s for type %d unloaded",
706                    dev_type->name, dev_type->type);
707
708 out:
709         TRACE_EXIT();
710         return;
711 }
712
713 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
714 {
715         int res = 0;
716
717         TRACE_ENTRY();
718
719         if (dev_type->parse == NULL) {
720                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
721                         "parse() method.", dev_type->name);
722                 res = -EINVAL;
723                 goto out;
724         }
725
726         res = scst_build_proc_dev_handler_dir_entries(dev_type);
727         if (res < 0) {
728                 goto out;
729         }
730         
731         PRINT_INFO_PR("Device handler %s for type %d loaded "
732                 "successfully", dev_type->name, dev_type->type);
733
734 out:
735         TRACE_EXIT_RES(res);
736         return res;
737 }
738
739 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
740 {
741         TRACE_ENTRY();
742
743         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
744
745         PRINT_INFO_PR("Device handler %s for type %d unloaded",
746                    dev_type->name, dev_type->type);
747
748         TRACE_EXIT();
749         return;
750 }
751
752 /* scst_mutex supposed to be held */
753 int scst_assign_dev_handler(struct scst_device *dev, 
754         struct scst_dev_type *handler)
755 {
756         int res = 0;
757         struct scst_tgt_dev *tgt_dev;
758         LIST_HEAD(attached_tgt_devs);
759         
760         TRACE_ENTRY();
761         
762         if (dev->handler == handler)
763                 goto out;
764         
765         __scst_suspend_activity();
766
767         if (dev->handler && dev->handler->detach_tgt) {
768                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
769                         dev_tgt_dev_list_entry) 
770                 {
771                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
772                                 tgt_dev);
773                         dev->handler->detach_tgt(tgt_dev);
774                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
775                 }
776         }
777
778         if (dev->handler && dev->handler->detach) {
779                 TRACE_DBG("%s", "Calling dev handler's detach()");
780                 dev->handler->detach(dev);
781                 TRACE_DBG("%s", "Old handler's detach() returned");
782         }
783
784         dev->handler = handler;
785
786         if (handler && handler->attach) {
787                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
788                 res = handler->attach(dev);
789                 TRACE_DBG("New dev handler's attach() returned %d", res);
790                 if (res != 0) {
791                         PRINT_ERROR_PR("New device handler's %s attach() "
792                                 "failed: %d", handler->name, res);
793                 }
794                 goto out_resume;
795         }
796         
797         if (handler && handler->attach_tgt) {
798                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
799                         dev_tgt_dev_list_entry) 
800                 {
801                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
802                                 tgt_dev);
803                         res = handler->attach_tgt(tgt_dev);
804                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
805                         if (res != 0) {
806                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
807                                     "failed: %d", handler->name, res);
808                                 goto out_err_detach_tgt;
809                         }
810                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
811                                 &attached_tgt_devs);
812                 }
813         }
814         
815 out_resume:
816         if (res != 0)
817                 dev->handler = NULL;
818         __scst_resume_activity();
819         
820 out:
821         TRACE_EXIT_RES(res);
822         return res;
823
824 out_err_detach_tgt:
825         if (handler && handler->detach_tgt) {
826                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
827                                  extra_tgt_dev_list_entry) 
828                 {
829                         TRACE_DBG("Calling handler's detach_tgt(%p)",
830                                 tgt_dev);
831                         handler->detach_tgt(tgt_dev);
832                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
833                 }
834         }
835         if (handler && handler->detach) {
836                 TRACE_DBG("%s", "Calling handler's detach()");
837                 handler->detach(dev);
838                 TRACE_DBG("%s", "Hhandler's detach() returned");
839         }
840         goto out_resume;
841 }
842
843 int scst_add_threads(int num)
844 {
845         int res = 0, i;
846         
847         TRACE_ENTRY();
848         
849         for (i = 0; i < num; i++) {
850                 res = kernel_thread(scst_cmd_thread, 0, SCST_THREAD_FLAGS);
851                 if (res < 0) {
852                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
853                         goto out_error;
854                 }
855                 atomic_inc(&scst_threads_count);
856         }
857
858         res = 0;
859
860 out:
861         TRACE_EXIT_RES(res);
862         return res;
863
864 out_error:
865         if (i > 0)
866                 scst_del_threads(i-1);
867         goto out;
868 }
869
870 void scst_del_threads(int num)
871 {
872         TRACE_ENTRY();
873         
874         spin_lock_irq(&scst_list_lock);
875         scst_shut_threads_count += num;
876         spin_unlock_irq(&scst_list_lock);
877         
878         wake_up_nr(&scst_list_waitQ, num);
879
880         TRACE_EXIT();
881         return;
882 }
883
884 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
885 static int scst_add(struct class_device *cdev)
886 #else
887 static int scst_add(struct class_device *cdev, struct class_interface *intf)
888 #endif
889 {
890         struct scsi_device *scsidp;
891         int res = 0;
892
893         TRACE_ENTRY();
894         
895         scsidp = to_scsi_device(cdev->dev);
896         
897         down(&scst_mutex);
898         res = scst_register_device(scsidp);
899         up(&scst_mutex);
900
901         TRACE_EXIT();
902         return res;
903 }
904
905 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
906 static void scst_remove(struct class_device *cdev)
907 #else
908 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
909 #endif
910 {
911         struct scsi_device *scsidp;
912
913         TRACE_ENTRY();
914
915         scsidp = to_scsi_device(cdev->dev);
916
917         down(&scst_mutex);
918         scst_unregister_device(scsidp);
919         up(&scst_mutex);
920
921         TRACE_EXIT();
922         return;
923 }
924
925 static struct class_interface scst_interface = {
926         .add = scst_add,
927         .remove = scst_remove,
928 };
929
930 static inline int get_cpus_count(void)
931 {
932 #ifdef CONFIG_SMP
933         return cpus_weight(cpu_online_map);
934 #else
935         return 1;
936 #endif
937 }
938
939 static int __init init_scst(void)
940 {
941         int res = 0, i;
942         struct scst_cmd *cmd;
943         struct scsi_request *req;
944
945         TRACE_ENTRY();
946
947         BUILD_BUG_ON(sizeof(cmd->sense_buffer) != sizeof(req->sr_sense_buffer));
948
949         scst_num_cpus = get_cpus_count();
950         
951         /* ToDo: register_cpu_notifier() */
952         
953         if (scst_threads == 0)
954                 scst_threads = scst_num_cpus;
955                 
956         if (scst_threads < scst_num_cpus) {
957                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
958                         "CPUs count");
959                 res = -EFAULT;
960                 goto out;
961         }
962         
963 #define INIT_CACHEP(p, s, t, o) do {                                    \
964                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
965                                       SCST_SLAB_FLAGS, NULL, NULL);     \
966                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
967                           sizeof(struct t));                            \
968                 if (p == NULL) { res = -ENOMEM; goto o; }               \
969         } while (0)
970           
971         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING, 
972                     scst_mgmt_cmd, out);
973         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING, 
974                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
975         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING, 
976                     scst_cmd, out_destroy_ua_cache);
977         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
978                     scst_session, out_destroy_cmd_cache);
979         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
980                     scst_tgt_dev, out_destroy_sess_cache);
981         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
982                     scst_acg_dev, out_destroy_tgt_cache);
983
984         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
985                 mempool_free_slab, scst_mgmt_cachep);
986         if (scst_mgmt_mempool == NULL) {
987                 res = -ENOMEM;
988                 goto out_destroy_acg_cache;
989         }
990
991         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
992                 mempool_free_slab, scst_ua_cachep);
993         if (scst_ua_mempool == NULL) {
994                 res = -ENOMEM;
995                 goto out_destroy_mgmt_mempool;
996         }
997
998         res = scst_sgv_pools_init(&scst_sgv);
999         if (res != 0)
1000                 goto out_destroy_ua_mempool;
1001
1002         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1003         if (scst_default_acg == NULL) {
1004                 res = -ENOMEM;
1005                 goto out_destroy_sgv_pool;
1006         }
1007         
1008         res = scsi_register_interface(&scst_interface);
1009         if (res != 0)
1010                 goto out_free_acg;
1011
1012         scst_scsi_op_list_init();
1013
1014         res = scst_proc_init_module();
1015         if (res != 0)
1016                 goto out_unreg_interface;
1017
1018         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1019                 scst_threads);
1020
1021         for (i = 0; i < scst_threads; i++) {
1022                 res = kernel_thread(scst_cmd_thread, NULL, SCST_THREAD_FLAGS);
1023                 if (res < 0) {
1024                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
1025                         goto out_thread_free;
1026                 }
1027                 atomic_inc(&scst_threads_count);
1028         }
1029
1030         for (i = 0; i < sizeof(scst_tasklets) / sizeof(scst_tasklets[0]); i++)
1031                 tasklet_init(&scst_tasklets[i], (void *)scst_cmd_tasklet, 0);
1032
1033         res = kernel_thread(scst_mgmt_cmd_thread, NULL, SCST_THREAD_FLAGS);
1034         if (res < 0) {
1035                 PRINT_ERROR_PR("kernel_thread() for mcmd failed: %d", res);
1036                 goto out_thread_free;
1037         }
1038         atomic_inc(&scst_threads_count);
1039
1040         res = kernel_thread(scst_mgmt_thread, NULL, SCST_THREAD_FLAGS);
1041         if (res < 0) {
1042                 PRINT_ERROR_PR("kernel_thread() for mgmt failed: %d", res);
1043                 goto out_thread_free;
1044         }
1045         atomic_inc(&scst_threads_count);
1046
1047         if (scst_max_cmd_mem == 0) {
1048                 struct sysinfo si;
1049                 si_meminfo(&si);
1050 #if BITS_PER_LONG == 32
1051                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1052                                         (uint64_t)1 << 30);
1053 #else
1054                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1055 #endif
1056         } else
1057                 scst_max_cmd_mem <<= 20;
1058
1059         scst_cur_max_cmd_mem = scst_max_cmd_mem;
1060
1061         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1062                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1063
1064 out:
1065         TRACE_EXIT_RES(res);
1066         return res;
1067
1068 out_thread_free:
1069         if (atomic_read(&scst_threads_count)) {
1070                 DECLARE_MUTEX_LOCKED(shm);
1071                 scst_shutdown_mutex = &shm;
1072                 smp_mb();
1073                 set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1074
1075                 wake_up_all(&scst_list_waitQ);
1076                 wake_up_all(&scst_mgmt_cmd_list_waitQ);
1077                 wake_up_all(&scst_mgmt_waitQ);
1078
1079                 TRACE_DBG("Waiting for %d threads to complete",
1080                         atomic_read(&scst_threads_count));
1081                 down(&shm);
1082         }
1083         
1084         scst_proc_cleanup_module();
1085
1086 out_unreg_interface:
1087         scsi_unregister_interface(&scst_interface);
1088
1089 out_free_acg:
1090         scst_destroy_acg(scst_default_acg);
1091
1092 out_destroy_sgv_pool:
1093         scst_sgv_pools_deinit(&scst_sgv);
1094
1095 out_destroy_ua_mempool:
1096         mempool_destroy(scst_ua_mempool);
1097
1098 out_destroy_mgmt_mempool:
1099         mempool_destroy(scst_mgmt_mempool);
1100
1101 out_destroy_acg_cache:
1102         kmem_cache_destroy(scst_acgd_cachep);
1103
1104 out_destroy_tgt_cache:
1105         kmem_cache_destroy(scst_tgtd_cachep);
1106
1107 out_destroy_sess_cache:
1108         kmem_cache_destroy(scst_sess_cachep);
1109
1110 out_destroy_cmd_cache:
1111         kmem_cache_destroy(scst_cmd_cachep);
1112
1113 out_destroy_ua_cache:
1114         kmem_cache_destroy(scst_ua_cachep);
1115
1116 out_destroy_mgmt_cache:
1117         kmem_cache_destroy(scst_mgmt_cachep);
1118         goto out;
1119 }
1120
1121 static void __exit exit_scst(void)
1122 {
1123         DECLARE_MUTEX_LOCKED(shm);
1124
1125         TRACE_ENTRY();
1126         
1127         /* ToDo: unregister_cpu_notifier() */
1128
1129         scst_shutdown_mutex = &shm;
1130         smp_mb();
1131         set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1132
1133         wake_up_all(&scst_list_waitQ);
1134         wake_up_all(&scst_mgmt_cmd_list_waitQ);
1135         wake_up_all(&scst_mgmt_waitQ);
1136
1137         if (atomic_read(&scst_threads_count)) {
1138                 TRACE_DBG("Waiting for %d threads to complete",
1139                       atomic_read(&scst_threads_count));
1140                 down(&shm);
1141         
1142                 /* 
1143                  * Wait for one sec. to allow the thread(s) actually exit,
1144                  * otherwise we can get Oops. Any better way?
1145                  */
1146                 {
1147                         unsigned long t = jiffies;
1148                         TRACE_DBG("%s", "Waiting 1 sec...");
1149                         while((jiffies - t) < HZ)
1150                                 schedule();
1151                 }
1152         }
1153
1154         if (test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
1155                 cancel_delayed_work(&scst_cmd_mem_work);
1156                 flush_scheduled_work();
1157         }
1158
1159         scst_proc_cleanup_module();
1160         scsi_unregister_interface(&scst_interface);
1161         scst_destroy_acg(scst_default_acg);
1162
1163         scst_sgv_pools_deinit(&scst_sgv);
1164
1165 #define DEINIT_CACHEP(p, s) do {                        \
1166                 if (kmem_cache_destroy(p)) {            \
1167                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1168                                 "error", s);            \
1169                 }                                       \
1170                 p = NULL;                               \
1171         } while (0)
1172
1173         mempool_destroy(scst_mgmt_mempool);
1174         mempool_destroy(scst_ua_mempool);
1175
1176         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1177         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1178         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1179         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1180         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1181         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1182
1183         PRINT_INFO_PR("%s", "SCST unloaded");
1184
1185         TRACE_EXIT();
1186         return;
1187 }
1188
1189 /*
1190  * Device Handler Side (i.e. scst_fileio)
1191  */
1192 EXPORT_SYMBOL(scst_register_dev_driver);
1193 EXPORT_SYMBOL(scst_unregister_dev_driver);
1194 EXPORT_SYMBOL(scst_register);
1195 EXPORT_SYMBOL(scst_unregister);
1196
1197 EXPORT_SYMBOL(scst_register_virtual_device);
1198 EXPORT_SYMBOL(scst_unregister_virtual_device);
1199 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1200 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1201
1202 EXPORT_SYMBOL(scst_set_busy);
1203 EXPORT_SYMBOL(scst_set_cmd_error_status);
1204 EXPORT_SYMBOL(scst_set_cmd_error);
1205 EXPORT_SYMBOL(scst_set_resp_data_len);
1206
1207 /*
1208  * Target Driver Side (i.e. HBA)
1209  */
1210 EXPORT_SYMBOL(scst_register_session);
1211 EXPORT_SYMBOL(scst_unregister_session);
1212
1213 EXPORT_SYMBOL(scst_register_target_template);
1214 EXPORT_SYMBOL(scst_unregister_target_template);
1215
1216 EXPORT_SYMBOL(scst_cmd_init_done);
1217 EXPORT_SYMBOL(scst_tgt_cmd_done);
1218 EXPORT_SYMBOL(scst_rx_cmd);
1219 EXPORT_SYMBOL(scst_rx_data);
1220 EXPORT_SYMBOL(scst_rx_mgmt_fn_tag);
1221 EXPORT_SYMBOL(scst_rx_mgmt_fn_lun);
1222
1223 EXPORT_SYMBOL(scst_find_cmd);
1224 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1225
1226 /*
1227  * Global Commands
1228  */
1229 EXPORT_SYMBOL(scst_suspend_activity);
1230 EXPORT_SYMBOL(scst_resume_activity);
1231
1232 EXPORT_SYMBOL(scst_add_threads);
1233 EXPORT_SYMBOL(scst_del_threads);
1234
1235 #if defined(DEBUG) || defined(TRACING)
1236 EXPORT_SYMBOL(scst_proc_log_entry_read);
1237 EXPORT_SYMBOL(scst_proc_log_entry_write);
1238 #endif
1239
1240 EXPORT_SYMBOL(__scst_get_buf);
1241 EXPORT_SYMBOL(scst_check_mem);
1242
1243 /*
1244  * Other Commands
1245  */
1246 EXPORT_SYMBOL(scst_get_cdb_info);
1247 EXPORT_SYMBOL(scst_cmd_get_tgt_specific_lock);
1248 EXPORT_SYMBOL(scst_cmd_set_tgt_specific_lock);
1249
1250 #ifdef DEBUG
1251 EXPORT_SYMBOL(scst_random);
1252 #endif
1253
1254 module_init(init_scst);
1255 module_exit(exit_scst);
1256
1257 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1258 MODULE_LICENSE("GPL");
1259 MODULE_DESCRIPTION("SCSI target core");
1260 MODULE_VERSION(SCST_VERSION_STRING);