20e8c8e649b44e0c6d059f9b06527c3818587544
[mirror/scst/.git] / scst / src / scst.c
1 /*
2  *  scst.c
3  *  
4  *  Copyright (C) 2004-2006 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <asm/unistd.h>
28 #include <asm/string.h>
29
30 #include "scst_debug.h"
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #include "scst_debug.c"
36
37 #if defined(DEBUG) || defined(TRACING)
38 unsigned long trace_flag = SCST_DEFAULT_LOG_FLAGS;
39 #endif
40
41 /*
42  * All targets, devices and dev_types management is done under
43  * this mutex.
44  */
45 DECLARE_MUTEX(scst_mutex);
46
47 int scst_num_cpus;
48 DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ);
49 LIST_HEAD(scst_dev_wait_sess_list);
50
51 LIST_HEAD(scst_template_list);
52 LIST_HEAD(scst_dev_list);
53 LIST_HEAD(scst_dev_type_list);
54
55 kmem_cache_t *scst_mgmt_cachep;
56 mempool_t *scst_mgmt_mempool;
57 kmem_cache_t *scst_ua_cachep;
58 mempool_t *scst_ua_mempool;
59 kmem_cache_t *scst_tgtd_cachep;
60 kmem_cache_t *scst_sess_cachep;
61 kmem_cache_t *scst_acgd_cachep;
62
63 LIST_HEAD(scst_acg_list);
64 struct scst_acg *scst_default_acg;
65
66 kmem_cache_t *scst_cmd_cachep;
67
68 unsigned long scst_flags;
69 atomic_t scst_cmd_count = ATOMIC_INIT(0);
70 spinlock_t scst_list_lock = SPIN_LOCK_UNLOCKED;
71 LIST_HEAD(scst_active_cmd_list);
72 LIST_HEAD(scst_init_cmd_list);
73 LIST_HEAD(scst_cmd_list);
74 DECLARE_WAIT_QUEUE_HEAD(scst_list_waitQ);
75
76 spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED;
77 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
78
79 struct tasklet_struct scst_tasklets[NR_CPUS];
80
81 struct scst_sgv_pools scst_sgv;
82
83 DECLARE_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn, 0);
84
85 unsigned long scst_max_cmd_mem;
86
87 LIST_HEAD(scst_mgmt_cmd_list);
88 LIST_HEAD(scst_active_mgmt_cmd_list);
89 LIST_HEAD(scst_delayed_mgmt_cmd_list);
90 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ);
91
92 DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ);
93 spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED;
94 LIST_HEAD(scst_sess_mgmt_list);
95
96 struct semaphore *scst_shutdown_mutex = NULL;
97
98 int scst_threads;
99 atomic_t scst_threads_count = ATOMIC_INIT(0);
100 int scst_shut_threads_count;
101 int scst_thread_num;
102 static int suspend_count;
103
104 int scst_virt_dev_last_id = 1; /* protected by scst_mutex */
105
106 /* 
107  * This buffer and lock are intended to avoid memory allocation, which
108  * could fail in improper places.
109  */
110 spinlock_t scst_temp_UA_lock = SPIN_LOCK_UNLOCKED;
111 uint8_t scst_temp_UA[SCSI_SENSE_BUFFERSIZE];
112
113 module_param_named(scst_threads, scst_threads, int, 0);
114 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
115
116 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
117 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
118         "the SCST commands at any given time in Mb");
119
120 int scst_register_target_template(struct scst_tgt_template *vtt)
121 {
122         int res = 0;
123         struct scst_tgt_template *t;
124
125         TRACE_ENTRY();
126
127         INIT_LIST_HEAD(&vtt->tgt_list);
128
129         if (!vtt->detect) {
130                 PRINT_ERROR_PR("Target driver %s doesn't have a "
131                         "detect() method.", vtt->name);
132                 res = -EINVAL;
133                 goto out;
134         }
135         
136         if (!vtt->release) {
137                 PRINT_ERROR_PR("Target driver %s doesn't have a "
138                         "release() method.", vtt->name);
139                 res = -EINVAL;
140                 goto out;
141         }
142
143         if (!vtt->xmit_response) {
144                 PRINT_ERROR_PR("Target driver %s doesn't have a "
145                         "xmit_response() method.", vtt->name);
146                 res = -EINVAL;
147                 goto out;
148         }
149
150         if (!vtt->rdy_to_xfer) {
151                 PRINT_ERROR_PR("Target driver %s doesn't have a "
152                         "rdy_to_xfer() method.", vtt->name);
153                 res = -EINVAL;
154                 goto out;
155         }
156         
157         if (!vtt->on_free_cmd) {
158                 PRINT_ERROR_PR("Target driver %s doesn't have a "
159                         "on_free_cmd() method.", vtt->name);
160                 res = -EINVAL;
161                 goto out;
162         }
163
164         if (!vtt->no_proc_entry) {
165                 res = scst_build_proc_target_dir_entries(vtt);
166                 if (res < 0) {
167                         goto out;
168                 }
169         }
170
171         if (down_interruptible(&scst_mutex) != 0)
172                 goto out;
173         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
174                 if (strcmp(t->name, vtt->name) == 0) {
175                         PRINT_ERROR_PR("Target driver %s already registered",
176                                 vtt->name);
177                         up(&scst_mutex);
178                         goto out_cleanup;
179                 }
180         }
181         /* That's OK to drop it. The race doesn't matter */
182         up(&scst_mutex);
183
184         TRACE_DBG("%s", "Calling target driver's detect()");
185         res = vtt->detect(vtt);
186         TRACE_DBG("Target driver's detect() returned %d", res);
187         if (res < 0) {
188                 PRINT_ERROR_PR("%s", "The detect() routine failed");
189                 res = -EINVAL;
190                 goto out_cleanup;
191         }
192
193         down(&scst_mutex);
194         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
195         up(&scst_mutex);
196
197         res = 0;
198
199 out:
200         TRACE_EXIT_RES(res);
201         return res;
202
203 out_cleanup:
204         scst_cleanup_proc_target_dir_entries(vtt);
205         goto out;
206 }
207
208 void scst_unregister_target_template(struct scst_tgt_template *vtt)
209 {
210         struct scst_tgt *tgt;
211
212         TRACE_ENTRY();
213
214         if (down_interruptible(&scst_mutex) != 0)
215                 goto out;
216
217 restart:
218         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
219                 up(&scst_mutex);
220                 scst_unregister(tgt);
221                 down(&scst_mutex);
222                 goto restart;
223         }
224         list_del(&vtt->scst_template_list_entry);
225         up(&scst_mutex);
226
227         scst_cleanup_proc_target_dir_entries(vtt);
228
229 out:
230         TRACE_EXIT();
231         return;
232 }
233
234 struct scst_tgt *scst_register(struct scst_tgt_template *vtt)
235 {
236         struct scst_tgt *tgt;
237
238         TRACE_ENTRY();
239
240         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
241         TRACE_MEM("kzalloc(GFP_KERNEL) for tgt (%zd): %p",
242               sizeof(*tgt), tgt);
243         if (tgt == NULL) {
244                 TRACE(TRACE_OUT_OF_MEM, "%s", "kzalloc() failed");
245                 goto out;
246         }
247
248         INIT_LIST_HEAD(&tgt->sess_list);
249         init_waitqueue_head(&tgt->unreg_waitQ);
250         tgt->tgtt = vtt;
251         tgt->sg_tablesize = vtt->sg_tablesize;
252         spin_lock_init(&tgt->tgt_lock);
253         INIT_LIST_HEAD(&tgt->retry_cmd_list);
254         atomic_set(&tgt->finished_cmds, 0);
255         init_timer(&tgt->retry_timer);
256         tgt->retry_timer.data = (unsigned long)tgt;
257         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
258
259         down(&scst_mutex);
260
261         if (scst_build_proc_target_entries(tgt) < 0) {
262                 tgt = NULL;
263                 goto out_free;
264         }
265
266         list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
267
268         up(&scst_mutex);
269
270 out:
271         TRACE_EXIT();
272         return tgt;
273
274 out_free:
275         TRACE_MEM("kfree() for tgt %p", tgt);
276         kfree(tgt);
277         goto out;
278 }
279
280 static inline int test_sess_list(struct scst_tgt *tgt)
281 {
282         int res;
283         down(&scst_mutex);
284         res = list_empty(&tgt->sess_list);
285         up(&scst_mutex);
286         return res;
287 }
288
289 void scst_unregister(struct scst_tgt *tgt)
290 {
291         struct scst_session *sess;
292
293         TRACE_ENTRY();
294
295         TRACE_DBG("%s", "Calling target driver's release()");
296         tgt->tgtt->release(tgt);
297         TRACE_DBG("%s", "Target driver's release() returned");
298
299         down(&scst_mutex);
300         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
301                 BUG_ON(!sess->shutting_down);
302         }
303         up(&scst_mutex);
304
305         TRACE_DBG("%s", "Waiting for sessions shutdown");
306         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
307         TRACE_DBG("%s", "wait_event() returned");
308
309         down(&scst_mutex);
310         list_del(&tgt->tgt_list_entry);
311         up(&scst_mutex);
312
313         scst_cleanup_proc_target_entries(tgt);
314
315         del_timer_sync(&tgt->retry_timer);
316
317         TRACE_MEM("kfree for tgt: %p", tgt);
318         kfree(tgt);
319
320         TRACE_EXIT();
321         return;
322 }
323
324 /* scst_mutex supposed to be held */
325 void __scst_suspend_activity(void)
326 {
327         TRACE_ENTRY();
328
329         suspend_count++;
330         if (suspend_count > 1)
331                 goto out;
332
333         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
334         smp_mb__after_set_bit();
335
336         TRACE_DBG("Waiting for all %d active commands to complete",
337               atomic_read(&scst_cmd_count));
338         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
339         TRACE_DBG("%s", "wait_event() returned");
340
341 out:
342         TRACE_EXIT();
343         return;
344 }
345
346 void scst_suspend_activity(void)
347 {
348         down(&scst_mutex);
349         __scst_suspend_activity();
350 }
351
352 /* scst_mutex supposed to be held */
353 void __scst_resume_activity(void)
354 {
355         struct scst_session *sess, *tsess;
356
357         TRACE_ENTRY();
358
359         suspend_count--;
360         if (suspend_count > 0)
361                 goto out;
362
363         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
364         smp_mb__after_clear_bit();
365
366         spin_lock_irq(&scst_list_lock);
367         list_for_each_entry_safe(sess, tsess, &scst_dev_wait_sess_list,
368                             dev_wait_sess_list_entry) 
369         {
370                 sess->waiting = 0;
371                 list_del(&sess->dev_wait_sess_list_entry);
372         }
373         spin_unlock_irq(&scst_list_lock);
374
375         wake_up_all(&scst_list_waitQ);
376         wake_up_all(&scst_mgmt_cmd_list_waitQ);
377
378 out:
379         TRACE_EXIT();
380         return;
381 }
382
383 void scst_resume_activity(void)
384 {
385         __scst_resume_activity();
386         up(&scst_mutex);
387 }
388
389 /* Called under scst_mutex */
390 static int scst_register_device(struct scsi_device *scsidp)
391 {
392         int res = 0;
393         struct scst_device *dev;
394         struct scst_dev_type *dt;
395
396         TRACE_ENTRY();
397
398         dev = scst_alloc_device(GFP_KERNEL);
399         if (dev == NULL) {
400                 res = -ENOMEM;
401                 goto out;
402         }
403         
404         dev->rq_disk = alloc_disk(1);
405         if (dev->rq_disk == NULL) {
406                 res = -ENOMEM;
407                 scst_free_device(dev);
408                 goto out;
409         }
410         dev->rq_disk->major = SCST_MAJOR;
411
412         dev->scsi_dev = scsidp;
413
414         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
415         
416         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
417                 if (dt->type == scsidp->type) {
418                         res = scst_assign_dev_handler(dev, dt);
419                         if (res != 0)
420                                 goto out_free;
421                         break;
422                 }
423         }
424
425 out:
426         if (res == 0) {
427                 PRINT_INFO_PR("Attached SCSI target mid-level at "
428                     "scsi%d, channel %d, id %d, lun %d, type %d", 
429                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
430                     scsidp->lun, scsidp->type);
431         } 
432         else {
433                 PRINT_ERROR_PR("Failed to attach SCSI target mid-level "
434                     "at scsi%d, channel %d, id %d, lun %d, type %d", 
435                     scsidp->host->host_no, scsidp->channel, scsidp->id, 
436                     scsidp->lun, scsidp->type);
437         }
438
439         TRACE_EXIT_RES(res);
440         return res;
441
442 out_free:
443         put_disk(dev->rq_disk);
444
445         list_del(&dev->dev_list_entry);
446         scst_assign_dev_handler(dev, NULL);
447         goto out;
448 }
449
450 /* Called under scst_mutex */
451 static void scst_unregister_device(struct scsi_device *scsidp)
452 {
453         struct scst_device *d, *dev = NULL;
454         struct scst_acg_dev *acg_dev, *aa;
455
456         TRACE_ENTRY();
457         
458         __scst_suspend_activity();
459
460         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
461                 if (d->scsi_dev == scsidp) {
462                         dev = d;
463                         TRACE_DBG("Target device %p found", dev);
464                         break;
465                 }
466         }
467
468         if (dev == NULL) {
469                 PRINT_ERROR_PR("%s", "Target device not found");
470                 goto out_unblock;
471         }
472
473         list_del(&dev->dev_list_entry);
474         
475         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
476                                  dev_acg_dev_list_entry) 
477         {
478                 scst_acg_remove_dev(acg_dev->acg, dev);
479         }
480
481         scst_assign_dev_handler(dev, NULL);
482
483         put_disk(dev->rq_disk);
484         scst_free_device(dev);
485
486 out_unblock:
487         __scst_resume_activity();
488
489         PRINT_INFO_PR("Detached SCSI target mid-level from scsi%d, channel %d, "
490              "id %d, lun %d, type %d", scsidp->host->host_no, scsidp->channel, 
491              scsidp->id, scsidp->lun, scsidp->type);
492
493         TRACE_EXIT();
494         return;
495 }
496
497 int scst_register_virtual_device(struct scst_dev_type *dev_handler, 
498         const char *dev_name)
499 {
500         int res = -EINVAL, rc;
501         struct scst_device *dev = NULL;
502
503         TRACE_ENTRY();
504         
505         if (dev_handler == NULL) {
506                 PRINT_ERROR_PR("%s: valid device handler must be supplied", 
507                         __FUNCTION__);
508                 res = -EINVAL;
509                 goto out;
510         }
511         
512         if (dev_name == NULL) {
513                 PRINT_ERROR_PR("%s: device name must be non-NULL", __FUNCTION__);
514                 res = -EINVAL;
515                 goto out;
516         }
517
518         dev = scst_alloc_device(GFP_KERNEL);
519         if (dev == NULL) {
520                 res = -ENOMEM;
521                 goto out;
522         }
523
524         dev->scsi_dev = NULL;
525         dev->virt_name = dev_name;
526
527         if (down_interruptible(&scst_mutex) != 0) {
528                 res = -EINTR;
529                 goto out_free_dev;
530         }
531
532         dev->virt_id = scst_virt_dev_last_id++;
533
534         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
535
536         res = dev->virt_id;
537
538         rc = scst_assign_dev_handler(dev, dev_handler);
539         if (rc != 0) {
540                 res = rc;
541                 goto out_free_del;
542         }
543         
544         up(&scst_mutex);
545
546 out:
547         if (res > 0) {
548                 PRINT_INFO_PR("Attached SCSI target mid-level to virtual "
549                     "device %s (id %d)", dev_name, dev->virt_id);
550         } 
551         else {
552                 PRINT_INFO_PR("Failed to attach SCSI target mid-level to "
553                     "virtual device %s", dev_name);
554         }
555
556         TRACE_EXIT_RES(res);
557         return res;
558
559 out_free_del:
560         list_del(&dev->dev_list_entry);
561         up(&scst_mutex);
562
563 out_free_dev:
564         scst_free_device(dev);
565         goto out;
566 }
567
568 void scst_unregister_virtual_device(int id)
569 {
570         struct scst_device *d, *dev = NULL;
571         struct scst_acg_dev *acg_dev, *aa;
572
573         TRACE_ENTRY();
574
575         if (down_interruptible(&scst_mutex) != 0)
576                 goto out;
577
578         __scst_suspend_activity();
579
580         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
581                 if (d->virt_id == id) {
582                         dev = d;
583                         TRACE_DBG("Target device %p found", dev);
584                         break;
585                 }
586         }
587
588         if (dev == NULL) {
589                 PRINT_ERROR_PR("%s", "Target device not found");
590                 goto out_unblock;
591         }
592
593         list_del(&dev->dev_list_entry);
594         
595         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
596                                  dev_acg_dev_list_entry) 
597         {
598                 scst_acg_remove_dev(acg_dev->acg, dev);
599         }
600
601         scst_assign_dev_handler(dev, NULL);
602
603         PRINT_INFO_PR("Detached SCSI target mid-level from virtual device %s "
604                 "(id %d)", dev->virt_name, dev->virt_id);
605
606         scst_free_device(dev);
607
608 out_unblock:
609         __scst_resume_activity();
610
611         up(&scst_mutex);
612
613 out:
614         TRACE_EXIT();
615         return;
616 }
617
618 int scst_register_dev_driver(struct scst_dev_type *dev_type)
619 {
620         struct scst_dev_type *dt;
621         struct scst_device *dev;
622         int res = 0;
623         int exist;
624
625         TRACE_ENTRY();
626
627         if (dev_type->parse == NULL) {
628                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
629                         "parse() method.", dev_type->name);
630                 res = -EINVAL;
631                 goto out;
632         }
633
634         if (down_interruptible(&scst_mutex) != 0) {
635                 res = -EINTR;
636                 goto out;
637         }
638
639         exist = 0;
640         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
641                 if (dt->type == dev_type->type) {
642                         TRACE_DBG("Device type handler for type %d "
643                                 "already exist", dt->type);
644                         exist = 1;
645                         break;
646                 }
647         }
648         
649         res = scst_build_proc_dev_handler_dir_entries(dev_type);
650         if (res < 0) {
651                 goto out_up;
652         }
653         
654         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
655         
656         if (exist)
657                 goto out_up;
658
659         __scst_suspend_activity();
660         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
661                 if (dev->scsi_dev == NULL)
662                         continue;
663                 if (dev->scsi_dev->type == dev_type->type)
664                         scst_assign_dev_handler(dev, dev_type);
665         }
666         __scst_resume_activity();
667
668 out_up:
669         up(&scst_mutex);
670
671         if (res == 0) {
672                 PRINT_INFO_PR("Device handler %s for type %d loaded "
673                         "successfully", dev_type->name, dev_type->type);
674         }
675
676 out:
677         TRACE_EXIT_RES(res);
678         return res;
679 }
680
681 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
682 {
683         struct scst_device *dev;
684
685         TRACE_ENTRY();
686
687         if (down_interruptible(&scst_mutex) != 0)
688                 goto out;
689
690         __scst_suspend_activity();
691         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
692                 if (dev->handler == dev_type) {
693                         scst_assign_dev_handler(dev, NULL);
694                         TRACE_DBG("Dev handler removed from device %p", dev);
695                 }
696         }
697         __scst_resume_activity();
698
699         list_del(&dev_type->dev_type_list_entry);
700
701         up(&scst_mutex);
702
703         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
704
705         PRINT_INFO_PR("Device handler %s for type %d unloaded",
706                    dev_type->name, dev_type->type);
707
708 out:
709         TRACE_EXIT();
710         return;
711 }
712
713 int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type)
714 {
715         int res = 0;
716
717         TRACE_ENTRY();
718
719         if (dev_type->parse == NULL) {
720                 PRINT_ERROR_PR("scst dev_type driver %s doesn't have a "
721                         "parse() method.", dev_type->name);
722                 res = -EINVAL;
723                 goto out;
724         }
725
726         res = scst_build_proc_dev_handler_dir_entries(dev_type);
727         if (res < 0) {
728                 goto out;
729         }
730         
731         PRINT_INFO_PR("Device handler %s for type %d loaded "
732                 "successfully", dev_type->name, dev_type->type);
733
734 out:
735         TRACE_EXIT_RES(res);
736         return res;
737 }
738
739 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
740 {
741         TRACE_ENTRY();
742
743         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
744
745         PRINT_INFO_PR("Device handler %s for type %d unloaded",
746                    dev_type->name, dev_type->type);
747
748         TRACE_EXIT();
749         return;
750 }
751
752 /* scst_mutex supposed to be held */
753 int scst_assign_dev_handler(struct scst_device *dev, 
754         struct scst_dev_type *handler)
755 {
756         int res = 0;
757         struct scst_tgt_dev *tgt_dev;
758         LIST_HEAD(attached_tgt_devs);
759         
760         TRACE_ENTRY();
761         
762         if (dev->handler == handler)
763                 goto out;
764         
765         __scst_suspend_activity();
766
767         if (dev->handler && dev->handler->detach_tgt) {
768                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
769                         dev_tgt_dev_list_entry) 
770                 {
771                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
772                                 tgt_dev);
773                         dev->handler->detach_tgt(tgt_dev);
774                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
775                 }
776         }
777
778         if (dev->handler && dev->handler->detach) {
779                 TRACE_DBG("%s", "Calling dev handler's detach()");
780                 dev->handler->detach(dev);
781                 TRACE_DBG("%s", "Old handler's detach() returned");
782         }
783
784         dev->handler = handler;
785
786         if (handler && handler->attach) {
787                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
788                 res = handler->attach(dev);
789                 TRACE_DBG("New dev handler's attach() returned %d", res);
790                 if (res != 0) {
791                         PRINT_ERROR_PR("New device handler's %s attach() "
792                                 "failed: %d", handler->name, res);
793                 }
794                 goto out_resume;
795         }
796         
797         if (handler && handler->attach_tgt) {
798                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
799                         dev_tgt_dev_list_entry) 
800                 {
801                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
802                                 tgt_dev);
803                         res = handler->attach_tgt(tgt_dev);
804                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
805                         if (res != 0) {
806                                 PRINT_ERROR_PR("Device handler's %s attach_tgt() "
807                                     "failed: %d", handler->name, res);
808                                 goto out_err_detach_tgt;
809                         }
810                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
811                                 &attached_tgt_devs);
812                 }
813         }
814         
815 out_resume:
816         if (res != 0)
817                 dev->handler = NULL;
818         __scst_resume_activity();
819         
820 out:
821         TRACE_EXIT_RES(res);
822         return res;
823
824 out_err_detach_tgt:
825         if (handler && handler->detach_tgt) {
826                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
827                                  extra_tgt_dev_list_entry) 
828                 {
829                         TRACE_DBG("Calling handler's detach_tgt(%p)",
830                                 tgt_dev);
831                         handler->detach_tgt(tgt_dev);
832                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
833                 }
834         }
835         if (handler && handler->detach) {
836                 TRACE_DBG("%s", "Calling handler's detach()");
837                 handler->detach(dev);
838                 TRACE_DBG("%s", "Hhandler's detach() returned");
839         }
840         goto out_resume;
841 }
842
843 int scst_add_threads(int num)
844 {
845         int res = 0, i;
846         
847         TRACE_ENTRY();
848         
849         for (i = 0; i < num; i++) {
850                 res = kernel_thread(scst_cmd_thread, 0, SCST_THREAD_FLAGS);
851                 if (res < 0) {
852                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
853                         goto out_error;
854                 }
855                 atomic_inc(&scst_threads_count);
856         }
857
858         res = 0;
859
860 out:
861         TRACE_EXIT_RES(res);
862         return res;
863
864 out_error:
865         if (i > 0)
866                 scst_del_threads(i-1);
867         goto out;
868 }
869
870 void scst_del_threads(int num)
871 {
872         TRACE_ENTRY();
873         
874         spin_lock_irq(&scst_list_lock);
875         scst_shut_threads_count += num;
876         spin_unlock_irq(&scst_list_lock);
877         
878         wake_up_nr(&scst_list_waitQ, num);
879
880         TRACE_EXIT();
881         return;
882 }
883
884 void scst_get(void)
885 {
886         scst_inc_cmd_count();
887 }
888
889 void scst_put(void)
890 {
891         scst_dec_cmd_count();
892 }
893
894 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
895 static int scst_add(struct class_device *cdev)
896 #else
897 static int scst_add(struct class_device *cdev, struct class_interface *intf)
898 #endif
899 {
900         struct scsi_device *scsidp;
901         int res = 0;
902
903         TRACE_ENTRY();
904         
905         scsidp = to_scsi_device(cdev->dev);
906         
907         down(&scst_mutex);
908         res = scst_register_device(scsidp);
909         up(&scst_mutex);
910
911         TRACE_EXIT();
912         return res;
913 }
914
915 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
916 static void scst_remove(struct class_device *cdev)
917 #else
918 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
919 #endif
920 {
921         struct scsi_device *scsidp;
922
923         TRACE_ENTRY();
924
925         scsidp = to_scsi_device(cdev->dev);
926
927         down(&scst_mutex);
928         scst_unregister_device(scsidp);
929         up(&scst_mutex);
930
931         TRACE_EXIT();
932         return;
933 }
934
935 static struct class_interface scst_interface = {
936         .add = scst_add,
937         .remove = scst_remove,
938 };
939
940 static inline int get_cpus_count(void)
941 {
942 #ifdef CONFIG_SMP
943         return cpus_weight(cpu_online_map);
944 #else
945         return 1;
946 #endif
947 }
948
949 static int __init init_scst(void)
950 {
951         int res = 0, i;
952         struct scst_cmd *cmd;
953         struct scsi_request *req;
954
955         TRACE_ENTRY();
956
957         BUILD_BUG_ON(sizeof(cmd->sense_buffer) != sizeof(req->sr_sense_buffer));
958
959         scst_num_cpus = get_cpus_count();
960         
961         /* ToDo: register_cpu_notifier() */
962         
963         if (scst_threads == 0)
964                 scst_threads = scst_num_cpus;
965                 
966         if (scst_threads < scst_num_cpus) {
967                 PRINT_ERROR_PR("%s", "scst_threads can not be less than "
968                         "CPUs count");
969                 res = -EFAULT;
970                 goto out;
971         }
972         
973 #define INIT_CACHEP(p, s, t, o) do {                                    \
974                 p = kmem_cache_create(s, sizeof(struct t), 0,           \
975                                       SCST_SLAB_FLAGS, NULL, NULL);     \
976                 TRACE_MEM("Slab create: %s at %p size %zd", s, p,       \
977                           sizeof(struct t));                            \
978                 if (p == NULL) { res = -ENOMEM; goto o; }               \
979         } while (0)
980           
981         INIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING, 
982                     scst_mgmt_cmd, out);
983         INIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING, 
984                     scst_tgt_dev_UA, out_destroy_mgmt_cache);
985         INIT_CACHEP(scst_cmd_cachep,  SCST_CMD_CACHE_STRING, 
986                     scst_cmd, out_destroy_ua_cache);
987         INIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING,
988                     scst_session, out_destroy_cmd_cache);
989         INIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING,
990                     scst_tgt_dev, out_destroy_sess_cache);
991         INIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING,
992                     scst_acg_dev, out_destroy_tgt_cache);
993
994         scst_mgmt_mempool = mempool_create(10, mempool_alloc_slab,
995                 mempool_free_slab, scst_mgmt_cachep);
996         if (scst_mgmt_mempool == NULL) {
997                 res = -ENOMEM;
998                 goto out_destroy_acg_cache;
999         }
1000
1001         scst_ua_mempool = mempool_create(25, mempool_alloc_slab,
1002                 mempool_free_slab, scst_ua_cachep);
1003         if (scst_ua_mempool == NULL) {
1004                 res = -ENOMEM;
1005                 goto out_destroy_mgmt_mempool;
1006         }
1007
1008         res = scst_sgv_pools_init(&scst_sgv);
1009         if (res != 0)
1010                 goto out_destroy_ua_mempool;
1011
1012         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1013         if (scst_default_acg == NULL) {
1014                 res = -ENOMEM;
1015                 goto out_destroy_sgv_pool;
1016         }
1017         
1018         res = scsi_register_interface(&scst_interface);
1019         if (res != 0)
1020                 goto out_free_acg;
1021
1022         scst_scsi_op_list_init();
1023
1024         res = scst_proc_init_module();
1025         if (res != 0)
1026                 goto out_unreg_interface;
1027
1028         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1029                 scst_threads);
1030
1031         for (i = 0; i < scst_threads; i++) {
1032                 res = kernel_thread(scst_cmd_thread, NULL, SCST_THREAD_FLAGS);
1033                 if (res < 0) {
1034                         PRINT_ERROR_PR("kernel_thread() failed: %d", res);
1035                         goto out_thread_free;
1036                 }
1037                 atomic_inc(&scst_threads_count);
1038         }
1039
1040         for (i = 0; i < sizeof(scst_tasklets) / sizeof(scst_tasklets[0]); i++)
1041                 tasklet_init(&scst_tasklets[i], (void *)scst_cmd_tasklet, 0);
1042
1043         res = kernel_thread(scst_mgmt_cmd_thread, NULL, SCST_THREAD_FLAGS);
1044         if (res < 0) {
1045                 PRINT_ERROR_PR("kernel_thread() for mcmd failed: %d", res);
1046                 goto out_thread_free;
1047         }
1048         atomic_inc(&scst_threads_count);
1049
1050         res = kernel_thread(scst_mgmt_thread, NULL, SCST_THREAD_FLAGS);
1051         if (res < 0) {
1052                 PRINT_ERROR_PR("kernel_thread() for mgmt failed: %d", res);
1053                 goto out_thread_free;
1054         }
1055         atomic_inc(&scst_threads_count);
1056
1057         if (scst_max_cmd_mem == 0) {
1058                 struct sysinfo si;
1059                 si_meminfo(&si);
1060 #if BITS_PER_LONG == 32
1061                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1062                                         (uint64_t)1 << 30);
1063 #else
1064                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1065 #endif
1066         } else
1067                 scst_max_cmd_mem <<= 20;
1068
1069         scst_cur_max_cmd_mem = scst_max_cmd_mem;
1070
1071         PRINT_INFO_PR("SCST version %s loaded successfully (max mem for "
1072                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1073
1074 out:
1075         TRACE_EXIT_RES(res);
1076         return res;
1077
1078 out_thread_free:
1079         if (atomic_read(&scst_threads_count)) {
1080                 DECLARE_MUTEX_LOCKED(shm);
1081                 scst_shutdown_mutex = &shm;
1082                 smp_mb();
1083                 set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1084
1085                 wake_up_all(&scst_list_waitQ);
1086                 wake_up_all(&scst_mgmt_cmd_list_waitQ);
1087                 wake_up_all(&scst_mgmt_waitQ);
1088
1089                 TRACE_DBG("Waiting for %d threads to complete",
1090                         atomic_read(&scst_threads_count));
1091                 down(&shm);
1092         }
1093         
1094         scst_proc_cleanup_module();
1095
1096 out_unreg_interface:
1097         scsi_unregister_interface(&scst_interface);
1098
1099 out_free_acg:
1100         scst_destroy_acg(scst_default_acg);
1101
1102 out_destroy_sgv_pool:
1103         scst_sgv_pools_deinit(&scst_sgv);
1104
1105 out_destroy_ua_mempool:
1106         mempool_destroy(scst_ua_mempool);
1107
1108 out_destroy_mgmt_mempool:
1109         mempool_destroy(scst_mgmt_mempool);
1110
1111 out_destroy_acg_cache:
1112         kmem_cache_destroy(scst_acgd_cachep);
1113
1114 out_destroy_tgt_cache:
1115         kmem_cache_destroy(scst_tgtd_cachep);
1116
1117 out_destroy_sess_cache:
1118         kmem_cache_destroy(scst_sess_cachep);
1119
1120 out_destroy_cmd_cache:
1121         kmem_cache_destroy(scst_cmd_cachep);
1122
1123 out_destroy_ua_cache:
1124         kmem_cache_destroy(scst_ua_cachep);
1125
1126 out_destroy_mgmt_cache:
1127         kmem_cache_destroy(scst_mgmt_cachep);
1128         goto out;
1129 }
1130
1131 static void __exit exit_scst(void)
1132 {
1133         DECLARE_MUTEX_LOCKED(shm);
1134
1135         TRACE_ENTRY();
1136         
1137         /* ToDo: unregister_cpu_notifier() */
1138
1139         scst_shutdown_mutex = &shm;
1140         smp_mb();
1141         set_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
1142
1143         wake_up_all(&scst_list_waitQ);
1144         wake_up_all(&scst_mgmt_cmd_list_waitQ);
1145         wake_up_all(&scst_mgmt_waitQ);
1146
1147         if (atomic_read(&scst_threads_count)) {
1148                 TRACE_DBG("Waiting for %d threads to complete",
1149                       atomic_read(&scst_threads_count));
1150                 down(&shm);
1151         
1152                 /* 
1153                  * Wait for one sec. to allow the thread(s) actually exit,
1154                  * otherwise we can get Oops. Any better way?
1155                  */
1156                 {
1157                         unsigned long t = jiffies;
1158                         TRACE_DBG("%s", "Waiting 1 sec...");
1159                         while((jiffies - t) < HZ)
1160                                 schedule();
1161                 }
1162         }
1163
1164         if (test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
1165                 cancel_delayed_work(&scst_cmd_mem_work);
1166                 flush_scheduled_work();
1167         }
1168
1169         scst_proc_cleanup_module();
1170         scsi_unregister_interface(&scst_interface);
1171         scst_destroy_acg(scst_default_acg);
1172
1173         scst_sgv_pools_deinit(&scst_sgv);
1174
1175 #define DEINIT_CACHEP(p, s) do {                        \
1176                 if (kmem_cache_destroy(p)) {            \
1177                         PRINT_INFO_PR("kmem_cache_destroy of %s returned an "\
1178                                 "error", s);            \
1179                 }                                       \
1180                 p = NULL;                               \
1181         } while (0)
1182
1183         mempool_destroy(scst_mgmt_mempool);
1184         mempool_destroy(scst_ua_mempool);
1185
1186         DEINIT_CACHEP(scst_mgmt_cachep, SCST_MGMT_CMD_CACHE_STRING);
1187         DEINIT_CACHEP(scst_ua_cachep, SCST_UA_CACHE_STRING);
1188         DEINIT_CACHEP(scst_cmd_cachep, SCST_CMD_CACHE_STRING);
1189         DEINIT_CACHEP(scst_sess_cachep, SCST_SESSION_CACHE_STRING);
1190         DEINIT_CACHEP(scst_tgtd_cachep, SCST_TGT_DEV_CACHE_STRING);
1191         DEINIT_CACHEP(scst_acgd_cachep, SCST_ACG_DEV_CACHE_STRING);
1192
1193         PRINT_INFO_PR("%s", "SCST unloaded");
1194
1195         TRACE_EXIT();
1196         return;
1197 }
1198
1199 /*
1200  * Device Handler Side (i.e. scst_fileio)
1201  */
1202 EXPORT_SYMBOL(scst_register_dev_driver);
1203 EXPORT_SYMBOL(scst_unregister_dev_driver);
1204 EXPORT_SYMBOL(scst_register);
1205 EXPORT_SYMBOL(scst_unregister);
1206
1207 EXPORT_SYMBOL(scst_register_virtual_device);
1208 EXPORT_SYMBOL(scst_unregister_virtual_device);
1209 EXPORT_SYMBOL(scst_register_virtual_dev_driver);
1210 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
1211
1212 EXPORT_SYMBOL(scst_set_busy);
1213 EXPORT_SYMBOL(scst_set_cmd_error_status);
1214 EXPORT_SYMBOL(scst_set_cmd_error);
1215 EXPORT_SYMBOL(scst_set_resp_data_len);
1216
1217 /*
1218  * Target Driver Side (i.e. HBA)
1219  */
1220 EXPORT_SYMBOL(scst_register_session);
1221 EXPORT_SYMBOL(scst_unregister_session);
1222
1223 EXPORT_SYMBOL(scst_register_target_template);
1224 EXPORT_SYMBOL(scst_unregister_target_template);
1225
1226 EXPORT_SYMBOL(scst_cmd_init_done);
1227 EXPORT_SYMBOL(scst_tgt_cmd_done);
1228 EXPORT_SYMBOL(scst_rx_cmd);
1229 EXPORT_SYMBOL(scst_rx_data);
1230 EXPORT_SYMBOL(scst_rx_mgmt_fn_tag);
1231 EXPORT_SYMBOL(scst_rx_mgmt_fn_lun);
1232
1233 EXPORT_SYMBOL(scst_find_cmd);
1234 EXPORT_SYMBOL(scst_find_cmd_by_tag);
1235
1236 /*
1237  * Global Commands
1238  */
1239 EXPORT_SYMBOL(scst_suspend_activity);
1240 EXPORT_SYMBOL(scst_resume_activity);
1241
1242 EXPORT_SYMBOL(scst_add_threads);
1243 EXPORT_SYMBOL(scst_del_threads);
1244
1245 #if defined(DEBUG) || defined(TRACING)
1246 EXPORT_SYMBOL(scst_proc_log_entry_read);
1247 EXPORT_SYMBOL(scst_proc_log_entry_write);
1248 #endif
1249
1250 EXPORT_SYMBOL(__scst_get_buf);
1251 EXPORT_SYMBOL(scst_check_mem);
1252 EXPORT_SYMBOL(scst_get);
1253 EXPORT_SYMBOL(scst_put);
1254
1255 /*
1256  * Other Commands
1257  */
1258 EXPORT_SYMBOL(scst_get_cdb_info);
1259 EXPORT_SYMBOL(scst_cmd_get_tgt_specific_lock);
1260 EXPORT_SYMBOL(scst_cmd_set_tgt_specific_lock);
1261
1262 #ifdef DEBUG
1263 EXPORT_SYMBOL(scst_random);
1264 #endif
1265
1266 module_init(init_scst);
1267 module_exit(exit_scst);
1268
1269 MODULE_AUTHOR("Vladislav Bolkhovitin & Leonid Stoljar");
1270 MODULE_LICENSE("GPL");
1271 MODULE_DESCRIPTION("SCSI target core");
1272 MODULE_VERSION(SCST_VERSION_STRING);