f42eacd58c7f40c4882312fd6deb0124420ae500
[mirror/scst/.git] / scst / src / scst_main.c
1 /*
2  *  scst_main.c
3  *
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  *
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/module.h>
19
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/list.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30
31 #include "scst.h"
32 #include "scst_priv.h"
33 #include "scst_mem.h"
34
35 #if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
36 #warning "HIGHMEM kernel configurations are fully supported, but not \
37         recommended for performance reasons. Consider change VMSPLIT \
38         option or use 64-bit configuration instead. See README file for \
39         details."
40 #endif
41
42 #ifdef SCST_HIGHMEM
43 #error "SCST_HIGHMEM configuration isn't supported and broken, because there \
44         is no real point to support it, at least it definitely doesn't worth \
45         the effort. Better use no-HIGHMEM kernel with VMSPLIT option \
46         or in 64-bit configuration instead. See README file for details."
47 #endif
48
49 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
50 #warning "Patch scst_exec_req_fifo-<kernel-version>.patch was not applied on \
51         your kernel and STRICT_SERIALIZING isn't defined. Pass-through dev \
52         handlers will not be supported."
53 #endif
54
55 /**
56  ** SCST global variables. They are all uninitialized to have their layout in
57  ** memory be exactly as specified. Otherwise compiler puts zero-initialized
58  ** variable separately from nonzero-initialized ones.
59  **/
60
61 /*
62  * All targets, devices and dev_types management is done under this mutex.
63  *
64  * It must NOT be used in any works (schedule_work(), etc.), because
65  * otherwise a deadlock (double lock, actually) is possible, e.g., with
66  * scst_user detach_tgt(), which is called under scst_mutex and calls
67  * flush_scheduled_work().
68  */
69 struct mutex scst_mutex;
70
71 struct list_head scst_template_list;
72 struct list_head scst_dev_list;
73 struct list_head scst_dev_type_list;
74
75 spinlock_t scst_main_lock;
76
77 struct kmem_cache *scst_mgmt_cachep;
78 mempool_t *scst_mgmt_mempool;
79 struct kmem_cache *scst_mgmt_stub_cachep;
80 mempool_t *scst_mgmt_stub_mempool;
81 struct kmem_cache *scst_ua_cachep;
82 mempool_t *scst_ua_mempool;
83 struct kmem_cache *scst_sense_cachep;
84 mempool_t *scst_sense_mempool;
85 struct kmem_cache *scst_tgtd_cachep;
86 struct kmem_cache *scst_sess_cachep;
87 struct kmem_cache *scst_acgd_cachep;
88
89 struct list_head scst_acg_list;
90 struct scst_acg *scst_default_acg;
91
92 spinlock_t scst_init_lock;
93 wait_queue_head_t scst_init_cmd_list_waitQ;
94 struct list_head scst_init_cmd_list;
95 unsigned int scst_init_poll_cnt;
96
97 struct kmem_cache *scst_cmd_cachep;
98
99 #if defined(DEBUG) || defined(TRACING)
100 unsigned long scst_trace_flag;
101 #endif
102
103 unsigned long scst_flags;
104 atomic_t scst_cmd_count;
105
106 spinlock_t scst_cmd_mem_lock;
107 unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem;
108 unsigned long scst_max_cmd_mem;
109
110 struct scst_cmd_lists scst_main_cmd_lists;
111
112 struct scst_tasklet scst_tasklets[NR_CPUS];
113
114 spinlock_t scst_mcmd_lock;
115 struct list_head scst_active_mgmt_cmd_list;
116 struct list_head scst_delayed_mgmt_cmd_list;
117 wait_queue_head_t scst_mgmt_cmd_list_waitQ;
118
119 wait_queue_head_t scst_mgmt_waitQ;
120 spinlock_t scst_mgmt_lock;
121 struct list_head scst_sess_init_list;
122 struct list_head scst_sess_shut_list;
123
124 wait_queue_head_t scst_dev_cmd_waitQ;
125
126 struct mutex scst_suspend_mutex;
127 struct list_head scst_cmd_lists_list;
128
129 static int scst_threads;
130 struct scst_threads_info_t scst_threads_info;
131
132 static int suspend_count;
133
134 static int scst_virt_dev_last_id; /* protected by scst_mutex */
135
136 /*
137  * This buffer and lock are intended to avoid memory allocation, which
138  * could fail in improper places.
139  */
140 spinlock_t scst_temp_UA_lock;
141 uint8_t scst_temp_UA[SCST_SENSE_BUFFERSIZE];
142
143 module_param_named(scst_threads, scst_threads, int, 0);
144 MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
145
146 module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, long, 0);
147 MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
148         "the SCST commands at any given time in MB");
149
150 struct scst_dev_type scst_null_devtype = {
151         .name = "none",
152 };
153
154 int __scst_register_target_template(struct scst_tgt_template *vtt,
155         const char *version)
156 {
157         int res = 0;
158         struct scst_tgt_template *t;
159         static DEFINE_MUTEX(m);
160
161         TRACE_ENTRY();
162
163         INIT_LIST_HEAD(&vtt->tgt_list);
164
165         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
166                 PRINT_ERROR("Incorrect version of target %s", vtt->name);
167                 res = -EINVAL;
168                 goto out_err;
169         }
170
171         if (!vtt->detect) {
172                 PRINT_ERROR("Target driver %s doesn't have a "
173                         "detect() method.", vtt->name);
174                 res = -EINVAL;
175                 goto out_err;
176         }
177
178         if (!vtt->release) {
179                 PRINT_ERROR("Target driver %s doesn't have a "
180                         "release() method.", vtt->name);
181                 res = -EINVAL;
182                 goto out_err;
183         }
184
185         if (!vtt->xmit_response) {
186                 PRINT_ERROR("Target driver %s doesn't have a "
187                         "xmit_response() method.", vtt->name);
188                 res = -EINVAL;
189                 goto out_err;
190         }
191
192         if (vtt->threads_num < 0) {
193                 PRINT_ERROR("Wrong threads_num value %d for "
194                         "target \"%s\"", vtt->threads_num,
195                         vtt->name);
196                 res = -EINVAL;
197                 goto out_err;
198         }
199
200         if (!vtt->no_proc_entry) {
201                 res = scst_build_proc_target_dir_entries(vtt);
202                 if (res < 0)
203                         goto out_err;
204         }
205
206         if (mutex_lock_interruptible(&m) != 0)
207                 goto out_err;
208
209         if (mutex_lock_interruptible(&scst_mutex) != 0)
210                 goto out_m_up;
211         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
212                 if (strcmp(t->name, vtt->name) == 0) {
213                         PRINT_ERROR("Target driver %s already registered",
214                                 vtt->name);
215                         mutex_unlock(&scst_mutex);
216                         goto out_cleanup;
217                 }
218         }
219         mutex_unlock(&scst_mutex);
220
221         TRACE_DBG("%s", "Calling target driver's detect()");
222         res = vtt->detect(vtt);
223         TRACE_DBG("Target driver's detect() returned %d", res);
224         if (res < 0) {
225                 PRINT_ERROR("%s", "The detect() routine failed");
226                 res = -EINVAL;
227                 goto out_cleanup;
228         }
229
230         mutex_lock(&scst_mutex);
231         list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
232         mutex_unlock(&scst_mutex);
233
234         res = 0;
235
236         PRINT_INFO("Target template %s registered successfully", vtt->name);
237
238         mutex_unlock(&m);
239
240 out:
241         TRACE_EXIT_RES(res);
242         return res;
243
244 out_cleanup:
245         scst_cleanup_proc_target_dir_entries(vtt);
246
247 out_m_up:
248         mutex_unlock(&m);
249
250 out_err:
251         PRINT_ERROR("Failed to register target template %s", vtt->name);
252         goto out;
253 }
254 EXPORT_SYMBOL(__scst_register_target_template);
255
256 void scst_unregister_target_template(struct scst_tgt_template *vtt)
257 {
258         struct scst_tgt *tgt;
259         struct scst_tgt_template *t;
260         int found = 0;
261
262         TRACE_ENTRY();
263
264         mutex_lock(&scst_mutex);
265
266         list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
267                 if (strcmp(t->name, vtt->name) == 0) {
268                         found = 1;
269                         break;
270                 }
271         }
272         if (!found) {
273                 PRINT_ERROR("Target driver %s isn't registered", vtt->name);
274                 goto out_up;
275         }
276
277 restart:
278         list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
279                 mutex_unlock(&scst_mutex);
280                 scst_unregister(tgt);
281                 mutex_lock(&scst_mutex);
282                 goto restart;
283         }
284         list_del(&vtt->scst_template_list_entry);
285
286         PRINT_INFO("Target template %s unregistered successfully", vtt->name);
287
288 out_up:
289         mutex_unlock(&scst_mutex);
290
291         scst_cleanup_proc_target_dir_entries(vtt);
292
293         TRACE_EXIT();
294         return;
295 }
296 EXPORT_SYMBOL(scst_unregister_target_template);
297
298 struct scst_tgt *scst_register(struct scst_tgt_template *vtt,
299         const char *target_name)
300 {
301         struct scst_tgt *tgt;
302
303         TRACE_ENTRY();
304
305         tgt = kzalloc(sizeof(*tgt), GFP_KERNEL);
306         if (tgt == NULL) {
307                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of tgt failed");
308                 goto out_err;
309         }
310
311         INIT_LIST_HEAD(&tgt->sess_list);
312         init_waitqueue_head(&tgt->unreg_waitQ);
313         tgt->tgtt = vtt;
314         tgt->sg_tablesize = vtt->sg_tablesize;
315         spin_lock_init(&tgt->tgt_lock);
316         INIT_LIST_HEAD(&tgt->retry_cmd_list);
317         atomic_set(&tgt->finished_cmds, 0);
318         init_timer(&tgt->retry_timer);
319         tgt->retry_timer.data = (unsigned long)tgt;
320         tgt->retry_timer.function = scst_tgt_retry_timer_fn;
321
322         scst_suspend_activity();
323         mutex_lock(&scst_mutex);
324
325         if (target_name != NULL) {
326                 int len = strlen(target_name) + 1 +
327                         strlen(SCST_DEFAULT_ACG_NAME) + 1;
328
329                 tgt->default_group_name = kmalloc(len, GFP_KERNEL);
330                 if (tgt->default_group_name == NULL) {
331                         TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of default "
332                                 "group name failed");
333                         goto out_free_err;
334                 }
335                 sprintf(tgt->default_group_name, "%s_%s", SCST_DEFAULT_ACG_NAME,
336                         target_name);
337         }
338
339         if (scst_build_proc_target_entries(tgt) < 0)
340                 goto out_free_name;
341         else
342                 list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
343
344         mutex_unlock(&scst_mutex);
345         scst_resume_activity();
346
347         PRINT_INFO("Target %s (%p) for template %s registered successfully",
348                 target_name, tgt, vtt->name);
349
350 out:
351         TRACE_EXIT();
352         return tgt;
353
354 out_free_name:
355         kfree(tgt->default_group_name);
356
357 out_free_err:
358         mutex_unlock(&scst_mutex);
359         scst_resume_activity();
360
361         kfree(tgt);
362         tgt = NULL;
363
364 out_err:
365         PRINT_ERROR("Failed to register target %s for template %s",
366                 target_name, vtt->name);
367         goto out;
368 }
369 EXPORT_SYMBOL(scst_register);
370
371 static inline int test_sess_list(struct scst_tgt *tgt)
372 {
373         int res;
374         mutex_lock(&scst_mutex);
375         res = list_empty(&tgt->sess_list);
376         mutex_unlock(&scst_mutex);
377         return res;
378 }
379
380 void scst_unregister(struct scst_tgt *tgt)
381 {
382         struct scst_session *sess;
383         struct scst_tgt_template *vtt = tgt->tgtt;
384
385         TRACE_ENTRY();
386
387         TRACE_DBG("%s", "Calling target driver's release()");
388         tgt->tgtt->release(tgt);
389         TRACE_DBG("%s", "Target driver's release() returned");
390
391         mutex_lock(&scst_mutex);
392         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
393                 sBUG_ON(sess->shut_phase == SCST_SESS_SPH_READY);
394         }
395         mutex_unlock(&scst_mutex);
396
397         TRACE_DBG("%s", "Waiting for sessions shutdown");
398         wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
399         TRACE_DBG("%s", "wait_event() returned");
400
401         scst_suspend_activity();
402         mutex_lock(&scst_mutex);
403
404         list_del(&tgt->tgt_list_entry);
405
406         scst_cleanup_proc_target_entries(tgt);
407
408         kfree(tgt->default_group_name);
409
410         mutex_unlock(&scst_mutex);
411         scst_resume_activity();
412
413         del_timer_sync(&tgt->retry_timer);
414
415         PRINT_INFO("Target %p for template %s unregistered successfully",
416                 tgt, vtt->name);
417
418         kfree(tgt);
419
420         TRACE_EXIT();
421         return;
422 }
423 EXPORT_SYMBOL(scst_unregister);
424
425 void scst_suspend_activity(void)
426 {
427         TRACE_ENTRY();
428
429         mutex_lock(&scst_suspend_mutex);
430
431         TRACE_MGMT_DBG("suspend_count %d", suspend_count);
432         suspend_count++;
433         if (suspend_count > 1)
434                 goto out_up;
435
436         set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
437         set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
438         smp_mb__after_set_bit();
439
440         TRACE_MGMT_DBG("Waiting for %d active commands to complete",
441               atomic_read(&scst_cmd_count));
442         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
443         TRACE_MGMT_DBG("%s", "wait_event() returned");
444
445         clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
446         smp_mb__after_clear_bit();
447
448         TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
449               atomic_read(&scst_cmd_count));
450         wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0);
451         TRACE_MGMT_DBG("%s", "wait_event() returned");
452
453 out_up:
454         mutex_unlock(&scst_suspend_mutex);
455
456         TRACE_EXIT();
457         return;
458 }
459 EXPORT_SYMBOL(scst_suspend_activity);
460
461 void scst_resume_activity(void)
462 {
463         struct scst_cmd_lists *l;
464
465         TRACE_ENTRY();
466
467         mutex_lock(&scst_suspend_mutex);
468
469         suspend_count--;
470         TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
471         if (suspend_count > 0)
472                 goto out_up;
473
474         clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
475         smp_mb__after_clear_bit();
476
477         list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) {
478                 wake_up_all(&l->cmd_list_waitQ);
479         }
480         wake_up_all(&scst_init_cmd_list_waitQ);
481
482         spin_lock_irq(&scst_mcmd_lock);
483         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
484                 struct scst_mgmt_cmd *m;
485                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
486                                 mgmt_cmd_list_entry);
487                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
488                         "mgmt cmd list", m);
489                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
490         }
491         spin_unlock_irq(&scst_mcmd_lock);
492         wake_up_all(&scst_mgmt_cmd_list_waitQ);
493
494 out_up:
495         mutex_unlock(&scst_suspend_mutex);
496
497         TRACE_EXIT();
498         return;
499 }
500 EXPORT_SYMBOL(scst_resume_activity);
501
502 static int scst_register_device(struct scsi_device *scsidp)
503 {
504         int res = 0;
505         struct scst_device *dev;
506         struct scst_dev_type *dt;
507
508         TRACE_ENTRY();
509
510         scst_suspend_activity();
511         mutex_lock(&scst_mutex);
512
513         res = scst_alloc_device(GFP_KERNEL, &dev);
514         if (res != 0)
515                 goto out_up;
516
517         dev->type = scsidp->type;
518
519         dev->rq_disk = alloc_disk(1);
520         if (dev->rq_disk == NULL) {
521                 res = -ENOMEM;
522                 goto out_free_dev;
523         }
524         dev->rq_disk->major = SCST_MAJOR;
525
526         dev->scsi_dev = scsidp;
527
528         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
529
530         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
531                 if (dt->type == scsidp->type) {
532                         res = scst_assign_dev_handler(dev, dt);
533                         if (res != 0)
534                                 goto out_free;
535                         break;
536                 }
537         }
538
539 out_up:
540         mutex_unlock(&scst_mutex);
541         scst_resume_activity();
542
543         if (res == 0) {
544                 PRINT_INFO("Attached SCSI target mid-level at "
545                     "scsi%d, channel %d, id %d, lun %d, type %d",
546                     scsidp->host->host_no, scsidp->channel, scsidp->id,
547                     scsidp->lun, scsidp->type);
548         } else {
549                 PRINT_ERROR("Failed to attach SCSI target mid-level "
550                     "at scsi%d, channel %d, id %d, lun %d, type %d",
551                     scsidp->host->host_no, scsidp->channel, scsidp->id,
552                     scsidp->lun, scsidp->type);
553         }
554
555         TRACE_EXIT_RES(res);
556         return res;
557
558 out_free:
559         list_del(&dev->dev_list_entry);
560         put_disk(dev->rq_disk);
561
562 out_free_dev:
563         scst_free_device(dev);
564         goto out_up;
565 }
566
567 static void scst_unregister_device(struct scsi_device *scsidp)
568 {
569         struct scst_device *d, *dev = NULL;
570         struct scst_acg_dev *acg_dev, *aa;
571
572         TRACE_ENTRY();
573
574         scst_suspend_activity();
575         mutex_lock(&scst_mutex);
576
577         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
578                 if (d->scsi_dev == scsidp) {
579                         dev = d;
580                         TRACE_DBG("Target device %p found", dev);
581                         break;
582                 }
583         }
584         if (dev == NULL) {
585                 PRINT_ERROR("%s", "Target device not found");
586                 goto out_unblock;
587         }
588
589         list_del(&dev->dev_list_entry);
590
591         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
592                                  dev_acg_dev_list_entry)
593         {
594                 scst_acg_remove_dev(acg_dev->acg, dev);
595         }
596
597         scst_assign_dev_handler(dev, &scst_null_devtype);
598
599         put_disk(dev->rq_disk);
600         scst_free_device(dev);
601
602         PRINT_INFO("Detached SCSI target mid-level from scsi%d, channel %d, "
603                 "id %d, lun %d, type %d", scsidp->host->host_no,
604                 scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
605
606 out_unblock:
607         mutex_unlock(&scst_mutex);
608         scst_resume_activity();
609
610         TRACE_EXIT();
611         return;
612 }
613
614 static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
615 {
616         int res = 0;
617
618         if (dev_handler->parse == NULL) {
619                 PRINT_ERROR("scst dev_type driver %s doesn't have a "
620                         "parse() method.", dev_handler->name);
621                 res = -EINVAL;
622                 goto out;
623         }
624
625         if (dev_handler->exec == NULL) {
626 #ifdef ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
627                 dev_handler->exec_atomic = 1;
628 #else
629                 dev_handler->exec_atomic = 0;
630 #endif
631         }
632
633         if (dev_handler->dev_done == NULL)
634                 dev_handler->dev_done_atomic = 1;
635
636 out:
637         TRACE_EXIT_RES(res);
638         return res;
639 }
640
641 int scst_register_virtual_device(struct scst_dev_type *dev_handler,
642         const char *dev_name)
643 {
644         int res, rc;
645         struct scst_device *dev = NULL;
646
647         TRACE_ENTRY();
648
649         if (dev_handler == NULL) {
650                 PRINT_ERROR("%s: valid device handler must be supplied",
651                             __func__);
652                 res = -EINVAL;
653                 goto out;
654         }
655
656         if (dev_name == NULL) {
657                 PRINT_ERROR("%s: device name must be non-NULL", __func__);
658                 res = -EINVAL;
659                 goto out;
660         }
661
662         res = scst_dev_handler_check(dev_handler);
663         if (res != 0)
664                 goto out;
665
666         scst_suspend_activity();
667         if (mutex_lock_interruptible(&scst_mutex) != 0) {
668                 res = -EINTR;
669                 goto out_resume;
670         }
671
672         res = scst_alloc_device(GFP_KERNEL, &dev);
673         if (res != 0)
674                 goto out_up;
675
676         dev->type = dev_handler->type;
677         dev->scsi_dev = NULL;
678         dev->virt_name = dev_name;
679         dev->virt_id = scst_virt_dev_last_id++;
680
681         list_add_tail(&dev->dev_list_entry, &scst_dev_list);
682
683         res = dev->virt_id;
684
685         rc = scst_assign_dev_handler(dev, dev_handler);
686         if (rc != 0) {
687                 res = rc;
688                 goto out_free_del;
689         }
690
691 out_up:
692         mutex_unlock(&scst_mutex);
693
694 out_resume:
695         scst_resume_activity();
696
697 out:
698         if (res > 0) {
699                 PRINT_INFO("Attached SCSI target mid-level to virtual "
700                     "device %s (id %d)", dev_name, dev->virt_id);
701         } else {
702                 PRINT_INFO("Failed to attach SCSI target mid-level to "
703                     "virtual device %s", dev_name);
704         }
705
706         TRACE_EXIT_RES(res);
707         return res;
708
709 out_free_del:
710         list_del(&dev->dev_list_entry);
711         scst_free_device(dev);
712         goto out_up;
713 }
714 EXPORT_SYMBOL(scst_register_virtual_device);
715
716 void scst_unregister_virtual_device(int id)
717 {
718         struct scst_device *d, *dev = NULL;
719         struct scst_acg_dev *acg_dev, *aa;
720
721         TRACE_ENTRY();
722
723         scst_suspend_activity();
724         mutex_lock(&scst_mutex);
725
726         list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
727                 if (d->virt_id == id) {
728                         dev = d;
729                         TRACE_DBG("Target device %p (id %d) found", dev, id);
730                         break;
731                 }
732         }
733         if (dev == NULL) {
734                 PRINT_ERROR("Target virtual device (id %d) not found", id);
735                 goto out_unblock;
736         }
737
738         list_del(&dev->dev_list_entry);
739
740         list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
741                                  dev_acg_dev_list_entry)
742         {
743                 scst_acg_remove_dev(acg_dev->acg, dev);
744         }
745
746         scst_assign_dev_handler(dev, &scst_null_devtype);
747
748         PRINT_INFO("Detached SCSI target mid-level from virtual device %s "
749                 "(id %d)", dev->virt_name, dev->virt_id);
750
751         scst_free_device(dev);
752
753 out_unblock:
754         mutex_unlock(&scst_mutex);
755         scst_resume_activity();
756
757         TRACE_EXIT();
758         return;
759 }
760 EXPORT_SYMBOL(scst_unregister_virtual_device);
761
762 int __scst_register_dev_driver(struct scst_dev_type *dev_type,
763         const char *version)
764 {
765         struct scst_dev_type *dt;
766         struct scst_device *dev;
767         int res;
768         int exist;
769
770         TRACE_ENTRY();
771
772         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
773                 PRINT_ERROR("Incorrect version of dev handler %s",
774                         dev_type->name);
775                 res = -EINVAL;
776                 goto out_error;
777         }
778
779         res = scst_dev_handler_check(dev_type);
780         if (res != 0)
781                 goto out_error;
782
783 #if !defined(SCSI_EXEC_REQ_FIFO_DEFINED) && !defined(STRICT_SERIALIZING)
784         if (dev_type->exec == NULL) {
785                 PRINT_ERROR("Pass-through dev handlers (handler \"%s\") not "
786                         "supported. Consider applying on your kernel patch "
787                         "scst_exec_req_fifo-<kernel-version>.patch or define "
788                         "STRICT_SERIALIZING", dev_type->name);
789                 res = -EINVAL;
790                 goto out;
791         }
792 #endif
793
794         scst_suspend_activity();
795         if (mutex_lock_interruptible(&scst_mutex) != 0) {
796                 res = -EINTR;
797                 goto out_err_res;
798         }
799
800         exist = 0;
801         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
802                 if (strcmp(dt->name, dev_type->name) == 0) {
803                         PRINT_ERROR("Device type handler \"%s\" already "
804                                 "exist", dt->name);
805                         exist = 1;
806                         break;
807                 }
808         }
809         if (exist)
810                 goto out_up;
811
812         res = scst_build_proc_dev_handler_dir_entries(dev_type);
813         if (res < 0)
814                 goto out_up;
815
816         list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
817
818         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
819                 if ((dev->scsi_dev == NULL) || (dev->handler != &scst_null_devtype))
820                         continue;
821                 if (dev->scsi_dev->type == dev_type->type)
822                         scst_assign_dev_handler(dev, dev_type);
823         }
824
825         mutex_unlock(&scst_mutex);
826         scst_resume_activity();
827
828         if (res == 0) {
829                 PRINT_INFO("Device handler \"%s\" for type %d registered "
830                         "successfully", dev_type->name, dev_type->type);
831         }
832
833 out:
834         TRACE_EXIT_RES(res);
835         return res;
836
837 out_up:
838         mutex_unlock(&scst_mutex);
839
840 out_err_res:
841         scst_resume_activity();
842
843 out_error:
844         PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
845                 dev_type->name, dev_type->type);
846         goto out;
847 }
848 EXPORT_SYMBOL(__scst_register_dev_driver);
849
850 void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
851 {
852         struct scst_device *dev;
853         struct scst_dev_type *dt;
854         int found = 0;
855
856         TRACE_ENTRY();
857
858         scst_suspend_activity();
859         mutex_lock(&scst_mutex);
860
861         list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
862                 if (strcmp(dt->name, dev_type->name) == 0) {
863                         found = 1;
864                         break;
865                 }
866         }
867         if (!found) {
868                 PRINT_ERROR("Dev handler \"%s\" isn't registered",
869                         dev_type->name);
870                 goto out_up;
871         }
872
873         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
874                 if (dev->handler == dev_type) {
875                         scst_assign_dev_handler(dev, &scst_null_devtype);
876                         TRACE_DBG("Dev handler removed from device %p", dev);
877                 }
878         }
879
880         list_del(&dev_type->dev_type_list_entry);
881
882         mutex_unlock(&scst_mutex);
883         scst_resume_activity();
884
885         scst_cleanup_proc_dev_handler_dir_entries(dev_type);
886
887         PRINT_INFO("Device handler \"%s\" for type %d unloaded",
888                    dev_type->name, dev_type->type);
889
890 out:
891         TRACE_EXIT();
892         return;
893
894 out_up:
895         mutex_unlock(&scst_mutex);
896         scst_resume_activity();
897         goto out;
898 }
899 EXPORT_SYMBOL(scst_unregister_dev_driver);
900
901 int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
902         const char *version)
903 {
904         int res;
905
906         TRACE_ENTRY();
907
908         if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
909                 PRINT_ERROR("Incorrect version of virtual dev handler %s",
910                         dev_type->name);
911                 res = -EINVAL;
912                 goto out_err;
913         }
914
915         res = scst_dev_handler_check(dev_type);
916         if (res != 0)
917                 goto out_err;
918
919         if (!dev_type->no_proc) {
920                 res = scst_build_proc_dev_handler_dir_entries(dev_type);
921                 if (res < 0)
922                         goto out_err;
923         }
924
925         if (dev_type->type != -1) {
926                 PRINT_INFO("Virtual device handler %s for type %d "
927                         "registered successfully", dev_type->name,
928                         dev_type->type);
929         } else {
930                 PRINT_INFO("Virtual device handler \"%s\" registered "
931                         "successfully", dev_type->name);
932         }
933
934 out:
935         TRACE_EXIT_RES(res);
936         return res;
937
938 out_err:
939         PRINT_ERROR("Failed to register virtual device handler \"%s\"",
940                 dev_type->name);
941         goto out;
942 }
943 EXPORT_SYMBOL(__scst_register_virtual_dev_driver);
944
945 void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
946 {
947         TRACE_ENTRY();
948
949         if (!dev_type->no_proc)
950                 scst_cleanup_proc_dev_handler_dir_entries(dev_type);
951
952         PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
953
954         TRACE_EXIT();
955         return;
956 }
957 EXPORT_SYMBOL(scst_unregister_virtual_dev_driver);
958
959 /* Called under scst_mutex */
960 int scst_add_dev_threads(struct scst_device *dev, int num)
961 {
962         int i, res = 0;
963         int n = 0;
964         struct scst_cmd_thread_t *thr;
965         char nm[12];
966
967         TRACE_ENTRY();
968
969         list_for_each_entry(thr, &dev->threads_list, thread_list_entry) {
970                 n++;
971         }
972
973         for (i = 0; i < num; i++) {
974                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
975                 if (!thr) {
976                         res = -ENOMEM;
977                         PRINT_ERROR("Failed to allocate thr %d", res);
978                         goto out;
979                 }
980                 strncpy(nm, dev->handler->name, ARRAY_SIZE(nm)-1);
981                 nm[ARRAY_SIZE(nm)-1] = '\0';
982                 thr->cmd_thread = kthread_run(scst_cmd_thread,
983                         &dev->cmd_lists, "%sd%d_%d", nm, dev->dev_num, n++);
984                 if (IS_ERR(thr->cmd_thread)) {
985                         res = PTR_ERR(thr->cmd_thread);
986                         PRINT_ERROR("kthread_create() failed: %d", res);
987                         kfree(thr);
988                         goto out;
989                 }
990                 list_add(&thr->thread_list_entry, &dev->threads_list);
991         }
992
993 out:
994         TRACE_EXIT_RES(res);
995         return res;
996 }
997
998 /* Called under scst_mutex and suspended activity */
999 static int scst_create_dev_threads(struct scst_device *dev)
1000 {
1001         int res = 0;
1002         int threads_num;
1003
1004         TRACE_ENTRY();
1005
1006         if (dev->handler->threads_num <= 0)
1007                 goto out;
1008
1009         threads_num = dev->handler->threads_num;
1010
1011         spin_lock_init(&dev->cmd_lists.cmd_list_lock);
1012         INIT_LIST_HEAD(&dev->cmd_lists.active_cmd_list);
1013         init_waitqueue_head(&dev->cmd_lists.cmd_list_waitQ);
1014
1015         res = scst_add_dev_threads(dev, threads_num);
1016         if (res != 0)
1017                 goto out;
1018
1019         mutex_lock(&scst_suspend_mutex);
1020         list_add_tail(&dev->cmd_lists.lists_list_entry,
1021                 &scst_cmd_lists_list);
1022         mutex_unlock(&scst_suspend_mutex);
1023
1024         dev->p_cmd_lists = &dev->cmd_lists;
1025
1026 out:
1027         TRACE_EXIT_RES(res);
1028         return res;
1029 }
1030
1031 /* Called under scst_mutex */
1032 void scst_del_dev_threads(struct scst_device *dev, int num)
1033 {
1034         struct scst_cmd_thread_t *ct, *tmp;
1035         int i = 0;
1036
1037         TRACE_ENTRY();
1038
1039         list_for_each_entry_safe(ct, tmp, &dev->threads_list,
1040                                 thread_list_entry) {
1041                 int rc = kthread_stop(ct->cmd_thread);
1042                 if (rc < 0)
1043                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
1044                 list_del(&ct->thread_list_entry);
1045                 kfree(ct);
1046                 if ((num > 0) && (++i >= num))
1047                         break;
1048         }
1049
1050         TRACE_EXIT();
1051         return;
1052 }
1053
1054 /* Called under scst_mutex and suspended activity */
1055 static void scst_stop_dev_threads(struct scst_device *dev)
1056 {
1057         TRACE_ENTRY();
1058
1059         if (list_empty(&dev->threads_list))
1060                 goto out;
1061
1062         scst_del_dev_threads(dev, -1);
1063
1064         if (dev->p_cmd_lists == &dev->cmd_lists) {
1065                 mutex_lock(&scst_suspend_mutex);
1066                 list_del(&dev->cmd_lists.lists_list_entry);
1067                 mutex_unlock(&scst_suspend_mutex);
1068         }
1069
1070 out:
1071         TRACE_EXIT();
1072         return;
1073 }
1074
1075 /* The activity supposed to be suspended and scst_mutex held */
1076 int scst_assign_dev_handler(struct scst_device *dev,
1077         struct scst_dev_type *handler)
1078 {
1079         int res = 0;
1080         struct scst_tgt_dev *tgt_dev;
1081         LIST_HEAD(attached_tgt_devs);
1082
1083         TRACE_ENTRY();
1084
1085         sBUG_ON(handler == NULL);
1086
1087         if (dev->handler == handler)
1088                 goto out;
1089
1090         if (dev->handler && dev->handler->detach_tgt) {
1091                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1092                                 dev_tgt_dev_list_entry) {
1093                         TRACE_DBG("Calling dev handler's detach_tgt(%p)",
1094                                 tgt_dev);
1095                         dev->handler->detach_tgt(tgt_dev);
1096                         TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
1097                 }
1098         }
1099
1100         if (dev->handler && dev->handler->detach) {
1101                 TRACE_DBG("%s", "Calling dev handler's detach()");
1102                 dev->handler->detach(dev);
1103                 TRACE_DBG("%s", "Old handler's detach() returned");
1104         }
1105
1106         scst_stop_dev_threads(dev);
1107
1108         dev->handler = handler;
1109
1110         if (handler) {
1111                 res = scst_create_dev_threads(dev);
1112                 if (res != 0)
1113                         goto out_null;
1114         }
1115
1116         if (handler && handler->attach) {
1117                 TRACE_DBG("Calling new dev handler's attach(%p)", dev);
1118                 res = handler->attach(dev);
1119                 TRACE_DBG("New dev handler's attach() returned %d", res);
1120                 if (res != 0) {
1121                         PRINT_ERROR("New device handler's %s attach() "
1122                                 "failed: %d", handler->name, res);
1123                 }
1124                 goto out_thr_null;
1125         }
1126
1127         if (handler && handler->attach_tgt) {
1128                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1129                                 dev_tgt_dev_list_entry) {
1130                         TRACE_DBG("Calling dev handler's attach_tgt(%p)",
1131                                 tgt_dev);
1132                         res = handler->attach_tgt(tgt_dev);
1133                         TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
1134                         if (res != 0) {
1135                                 PRINT_ERROR("Device handler's %s attach_tgt() "
1136                                     "failed: %d", handler->name, res);
1137                                 goto out_err_detach_tgt;
1138                         }
1139                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
1140                                 &attached_tgt_devs);
1141                 }
1142         }
1143
1144 out_thr_null:
1145         if (res != 0)
1146                 scst_stop_dev_threads(dev);
1147
1148 out_null:
1149         if (res != 0)
1150                 dev->handler = &scst_null_devtype;
1151
1152 out:
1153         TRACE_EXIT_RES(res);
1154         return res;
1155
1156 out_err_detach_tgt:
1157         if (handler && handler->detach_tgt) {
1158                 list_for_each_entry(tgt_dev, &attached_tgt_devs,
1159                                  extra_tgt_dev_list_entry)
1160                 {
1161                         TRACE_DBG("Calling handler's detach_tgt(%p)",
1162                                 tgt_dev);
1163                         handler->detach_tgt(tgt_dev);
1164                         TRACE_DBG("%s", "Handler's detach_tgt() returned");
1165                 }
1166         }
1167         if (handler && handler->detach) {
1168                 TRACE_DBG("%s", "Calling handler's detach()");
1169                 handler->detach(dev);
1170                 TRACE_DBG("%s", "Handler's detach() returned");
1171         }
1172         goto out_null;
1173 }
1174
1175 int scst_cmd_threads_count(void)
1176 {
1177         int i;
1178
1179         /* Just to lower the race window, when user can get just changed value */
1180         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1181         i = scst_threads_info.nr_cmd_threads;
1182         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1183         return i;
1184 }
1185
1186 static void scst_threads_info_init(void)
1187 {
1188         memset(&scst_threads_info, 0, sizeof(scst_threads_info));
1189         mutex_init(&scst_threads_info.cmd_threads_mutex);
1190         INIT_LIST_HEAD(&scst_threads_info.cmd_threads_list);
1191 }
1192
1193 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1194 void __scst_del_cmd_threads(int num)
1195 {
1196         struct scst_cmd_thread_t *ct, *tmp;
1197         int i;
1198
1199         TRACE_ENTRY();
1200
1201         i = scst_threads_info.nr_cmd_threads;
1202         if (num <= 0 || num > i) {
1203                 PRINT_ERROR("can not del %d cmd threads from %d", num, i);
1204                 return;
1205         }
1206
1207         list_for_each_entry_safe(ct, tmp, &scst_threads_info.cmd_threads_list,
1208                                 thread_list_entry) {
1209                 int res;
1210
1211                 res = kthread_stop(ct->cmd_thread);
1212                 if (res < 0)
1213                         TRACE_MGMT_DBG("kthread_stop() failed: %d", res);
1214                 list_del(&ct->thread_list_entry);
1215                 kfree(ct);
1216                 scst_threads_info.nr_cmd_threads--;
1217                 --num;
1218                 if (num == 0)
1219                         break;
1220         }
1221
1222         TRACE_EXIT();
1223         return;
1224 }
1225
1226 /* scst_threads_info.cmd_threads_mutex supposed to be held */
1227 int __scst_add_cmd_threads(int num)
1228 {
1229         int res = 0, i;
1230         static int scst_thread_num;
1231
1232         TRACE_ENTRY();
1233
1234         for (i = 0; i < num; i++) {
1235                 struct scst_cmd_thread_t *thr;
1236
1237                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
1238                 if (!thr) {
1239                         res = -ENOMEM;
1240                         PRINT_ERROR("fail to allocate thr %d", res);
1241                         goto out_error;
1242                 }
1243                 thr->cmd_thread = kthread_run(scst_cmd_thread,
1244                         &scst_main_cmd_lists, "scsi_tgt%d",
1245                         scst_thread_num++);
1246                 if (IS_ERR(thr->cmd_thread)) {
1247                         res = PTR_ERR(thr->cmd_thread);
1248                         PRINT_ERROR("kthread_create() failed: %d", res);
1249                         kfree(thr);
1250                         goto out_error;
1251                 }
1252                 list_add(&thr->thread_list_entry,
1253                         &scst_threads_info.cmd_threads_list);
1254                 scst_threads_info.nr_cmd_threads++;
1255         }
1256         res = 0;
1257
1258 out:
1259         TRACE_EXIT_RES(res);
1260         return res;
1261
1262 out_error:
1263         if (i > 0)
1264                 __scst_del_cmd_threads(i - 1);
1265         goto out;
1266 }
1267
1268 int scst_add_cmd_threads(int num)
1269 {
1270         int res;
1271
1272         TRACE_ENTRY();
1273
1274         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1275         res = __scst_add_cmd_threads(num);
1276         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1277
1278         TRACE_EXIT_RES(res);
1279         return res;
1280 }
1281 EXPORT_SYMBOL(scst_add_cmd_threads);
1282
1283 void scst_del_cmd_threads(int num)
1284 {
1285         TRACE_ENTRY();
1286
1287         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1288         __scst_del_cmd_threads(num);
1289         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1290
1291         TRACE_EXIT();
1292         return;
1293 }
1294 EXPORT_SYMBOL(scst_del_cmd_threads);
1295
1296 static void scst_stop_all_threads(void)
1297 {
1298         TRACE_ENTRY();
1299
1300         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1301         __scst_del_cmd_threads(scst_threads_info.nr_cmd_threads);
1302         if (scst_threads_info.mgmt_cmd_thread)
1303                 kthread_stop(scst_threads_info.mgmt_cmd_thread);
1304         if (scst_threads_info.mgmt_thread)
1305                 kthread_stop(scst_threads_info.mgmt_thread);
1306         if (scst_threads_info.init_cmd_thread)
1307                 kthread_stop(scst_threads_info.init_cmd_thread);
1308         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1309
1310         TRACE_EXIT();
1311         return;
1312 }
1313
1314 static int scst_start_all_threads(int num)
1315 {
1316         int res;
1317
1318         TRACE_ENTRY();
1319
1320         mutex_lock(&scst_threads_info.cmd_threads_mutex);
1321         res = __scst_add_cmd_threads(num);
1322         if (res < 0)
1323                 goto out;
1324
1325         scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread,
1326                 NULL, "scsi_tgt_init");
1327         if (IS_ERR(scst_threads_info.init_cmd_thread)) {
1328                 res = PTR_ERR(scst_threads_info.init_cmd_thread);
1329                 PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
1330                 scst_threads_info.init_cmd_thread = NULL;
1331                 goto out;
1332         }
1333
1334         scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread,
1335                 NULL, "scsi_tgt_mc");
1336         if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) {
1337                 res = PTR_ERR(scst_threads_info.mgmt_cmd_thread);
1338                 PRINT_ERROR("kthread_create() for mcmd failed: %d", res);
1339                 scst_threads_info.mgmt_cmd_thread = NULL;
1340                 goto out;
1341         }
1342
1343         scst_threads_info.mgmt_thread = kthread_run(scst_mgmt_thread,
1344                 NULL, "scsi_tgt_mgmt");
1345         if (IS_ERR(scst_threads_info.mgmt_thread)) {
1346                 res = PTR_ERR(scst_threads_info.mgmt_thread);
1347                 PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
1348                 scst_threads_info.mgmt_thread = NULL;
1349                 goto out;
1350         }
1351
1352 out:
1353         mutex_unlock(&scst_threads_info.cmd_threads_mutex);
1354         TRACE_EXIT_RES(res);
1355         return res;
1356 }
1357
1358 void scst_get(void)
1359 {
1360         __scst_get(0);
1361 }
1362 EXPORT_SYMBOL(scst_get);
1363
1364 void scst_put(void)
1365 {
1366         __scst_put();
1367 }
1368 EXPORT_SYMBOL(scst_put);
1369
1370 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 15)
1371 static int scst_add(struct class_device *cdev)
1372 #else
1373 static int scst_add(struct class_device *cdev, struct class_interface *intf)
1374 #endif
1375 {
1376         struct scsi_device *scsidp;
1377         int res = 0;
1378
1379         TRACE_ENTRY();
1380
1381         scsidp = to_scsi_device(cdev->dev);
1382         res = scst_register_device(scsidp);
1383
1384         TRACE_EXIT();
1385         return res;
1386 }
1387
1388 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 15)
1389 static void scst_remove(struct class_device *cdev)
1390 #else
1391 static void scst_remove(struct class_device *cdev, struct class_interface *intf)
1392 #endif
1393 {
1394         struct scsi_device *scsidp;
1395
1396         TRACE_ENTRY();
1397
1398         scsidp = to_scsi_device(cdev->dev);
1399         scst_unregister_device(scsidp);
1400
1401         TRACE_EXIT();
1402         return;
1403 }
1404
1405 static struct class_interface scst_interface = {
1406         .add = scst_add,
1407         .remove = scst_remove,
1408 };
1409
1410 static void __init scst_print_config(void)
1411 {
1412         char buf[128];
1413         int i, j;
1414
1415         i = snprintf(buf, sizeof(buf), "Enabled features: ");
1416         j = i;
1417
1418 #ifdef STRICT_SERIALIZING
1419         i += snprintf(&buf[i], sizeof(buf) - i, "Strict serializing");
1420 #endif
1421
1422 #ifdef EXTRACHECKS
1423         i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
1424                 (j == i) ? "" : ", ");
1425 #endif
1426
1427 #ifdef TRACING
1428         i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
1429                 (j == i) ? "" : ", ");
1430 #endif
1431
1432 #ifdef DEBUG
1433         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
1434                 (j == i) ? "" : ", ");
1435 #endif
1436
1437 #ifdef DEBUG_TM
1438         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
1439                 (j == i) ? "" : ", ");
1440 #endif
1441
1442 #ifdef DEBUG_RETRY
1443         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
1444                 (j == i) ? "" : ", ");
1445 #endif
1446
1447 #ifdef DEBUG_OOM
1448         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
1449                 (j == i) ? "" : ", ");
1450 #endif
1451
1452 #ifdef DEBUG_SN
1453         i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
1454                 (j == i) ? "" : ", ");
1455 #endif
1456
1457 #ifdef USE_EXPECTED_VALUES
1458         i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
1459                 (j == i) ? "" : ", ");
1460 #endif
1461
1462 #ifdef ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1463         i += snprintf(&buf[i], sizeof(buf) - i, "%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
1464                 (j == i) ? "" : ", ");
1465 #endif
1466
1467 #ifdef SCST_STRICT_SECURITY
1468         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
1469                 (j == i) ? "" : ", ");
1470 #endif
1471
1472 #ifdef SCST_HIGHMEM
1473         i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_HIGHMEM",
1474                 (j == i) ? "" : ", ");
1475 #endif
1476
1477         if (j != i)
1478                 PRINT_INFO("%s", buf);
1479 }
1480
1481 static int __init init_scst(void)
1482 {
1483         int res = 0, i;
1484         int scst_num_cpus;
1485
1486         TRACE_ENTRY();
1487
1488 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1489         {
1490                 struct scsi_request *req;
1491                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE !=
1492                         sizeof(req->sr_sense_buffer));
1493         }
1494 #else
1495         {
1496                 struct scsi_sense_hdr *shdr;
1497                 BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
1498         }
1499 #endif
1500         {
1501                 struct scst_tgt_dev *t;
1502                 struct scst_cmd *c;
1503                 BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
1504                 BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
1505         }
1506
1507         BUILD_BUG_ON(SCST_DATA_UNKNOWN != DMA_BIDIRECTIONAL);
1508         BUILD_BUG_ON(SCST_DATA_WRITE != DMA_TO_DEVICE);
1509         BUILD_BUG_ON(SCST_DATA_READ != DMA_FROM_DEVICE);
1510         BUILD_BUG_ON(SCST_DATA_NONE != DMA_NONE);
1511
1512         mutex_init(&scst_mutex);
1513         INIT_LIST_HEAD(&scst_template_list);
1514         INIT_LIST_HEAD(&scst_dev_list);
1515         INIT_LIST_HEAD(&scst_dev_type_list);
1516         spin_lock_init(&scst_main_lock);
1517         INIT_LIST_HEAD(&scst_acg_list);
1518         spin_lock_init(&scst_init_lock);
1519         init_waitqueue_head(&scst_init_cmd_list_waitQ);
1520         INIT_LIST_HEAD(&scst_init_cmd_list);
1521 #if defined(DEBUG) || defined(TRACING)
1522         scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
1523 #endif
1524         atomic_set(&scst_cmd_count, 0);
1525         spin_lock_init(&scst_cmd_mem_lock);
1526         spin_lock_init(&scst_mcmd_lock);
1527         INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
1528         INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
1529         init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
1530         init_waitqueue_head(&scst_mgmt_waitQ);
1531         spin_lock_init(&scst_mgmt_lock);
1532         INIT_LIST_HEAD(&scst_sess_init_list);
1533         INIT_LIST_HEAD(&scst_sess_shut_list);
1534         init_waitqueue_head(&scst_dev_cmd_waitQ);
1535         mutex_init(&scst_suspend_mutex);
1536         INIT_LIST_HEAD(&scst_cmd_lists_list);
1537         scst_virt_dev_last_id = 1;
1538         spin_lock_init(&scst_temp_UA_lock);
1539
1540         spin_lock_init(&scst_main_cmd_lists.cmd_list_lock);
1541         INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list);
1542         init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ);
1543         list_add_tail(&scst_main_cmd_lists.lists_list_entry,
1544                 &scst_cmd_lists_list);
1545
1546         scst_num_cpus = num_online_cpus();
1547
1548         /* ToDo: register_cpu_notifier() */
1549
1550         if (scst_threads == 0)
1551                 scst_threads = scst_num_cpus;
1552
1553         if (scst_threads < 1) {
1554                 PRINT_ERROR("%s", "scst_threads can not be less than 1");
1555                 scst_threads = scst_num_cpus;
1556         }
1557
1558         scst_threads_info_init();
1559
1560 #define INIT_CACHEP(p, s, o) do {                                       \
1561                 p = KMEM_CACHE(s, SCST_SLAB_FLAGS);                     \
1562                 TRACE_MEM("Slab create: %s at %p size %zd", #s, p,      \
1563                           sizeof(struct s));                            \
1564                 if (p == NULL) {                                        \
1565                         res = -ENOMEM;                                  \
1566                         goto o;                                         \
1567                 }                                                       \
1568         } while (0)
1569
1570         INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out);
1571         INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
1572                         out_destroy_mgmt_cache);
1573         INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
1574                         out_destroy_mgmt_stub_cache);
1575         {
1576                 struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
1577                 INIT_CACHEP(scst_sense_cachep, scst_sense, out_destroy_ua_cache);
1578         }
1579         INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_sense_cache);
1580         INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
1581         INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
1582         INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
1583
1584         scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
1585                 mempool_free_slab, scst_mgmt_cachep);
1586         if (scst_mgmt_mempool == NULL) {
1587                 res = -ENOMEM;
1588                 goto out_destroy_acg_cache;
1589         }
1590
1591         scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
1592                 mempool_free_slab, scst_mgmt_stub_cachep);
1593         if (scst_mgmt_stub_mempool == NULL) {
1594                 res = -ENOMEM;
1595                 goto out_destroy_mgmt_mempool;
1596         }
1597
1598         scst_ua_mempool = mempool_create(64, mempool_alloc_slab,
1599                 mempool_free_slab, scst_ua_cachep);
1600         if (scst_ua_mempool == NULL) {
1601                 res = -ENOMEM;
1602                 goto out_destroy_mgmt_stub_mempool;
1603         }
1604
1605         /* Loosing sense may have fatal consequences, so let's have a big pool */
1606         scst_sense_mempool = mempool_create(128, mempool_alloc_slab,
1607                 mempool_free_slab, scst_sense_cachep);
1608         if (scst_sense_mempool == NULL) {
1609                 res = -ENOMEM;
1610                 goto out_destroy_ua_mempool;
1611         }
1612
1613         if (scst_max_cmd_mem == 0) {
1614                 struct sysinfo si;
1615                 si_meminfo(&si);
1616 #if BITS_PER_LONG == 32
1617                 scst_max_cmd_mem = min(((uint64_t)si.totalram << PAGE_SHIFT) >> 2,
1618                                         (uint64_t)1 << 30);
1619 #else
1620                 scst_max_cmd_mem = (si.totalram << PAGE_SHIFT) >> 2;
1621 #endif
1622         } else
1623                 scst_max_cmd_mem <<= 20;
1624
1625         res = scst_sgv_pools_init(scst_max_cmd_mem, 0);
1626         if (res != 0)
1627                 goto out_destroy_sense_mempool;
1628
1629         scst_default_acg = scst_alloc_add_acg(SCST_DEFAULT_ACG_NAME);
1630         if (scst_default_acg == NULL) {
1631                 res = -ENOMEM;
1632                 goto out_destroy_sgv_pool;
1633         }
1634
1635         res = scsi_register_interface(&scst_interface);
1636         if (res != 0)
1637                 goto out_free_acg;
1638
1639         scst_scsi_op_list_init();
1640
1641         for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
1642                 spin_lock_init(&scst_tasklets[i].tasklet_lock);
1643                 INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
1644                 tasklet_init(&scst_tasklets[i].tasklet, (void *)scst_cmd_tasklet,
1645                         (unsigned long)&scst_tasklets[i]);
1646         }
1647
1648         TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
1649                 scst_threads);
1650
1651         res = scst_start_all_threads(scst_threads);
1652         if (res < 0)
1653                 goto out_thread_free;
1654
1655         res = scst_proc_init_module();
1656         if (res != 0)
1657                 goto out_thread_free;
1658
1659
1660         PRINT_INFO("SCST version %s loaded successfully (max mem for "
1661                 "commands %ld Mb)", SCST_VERSION_STRING, scst_max_cmd_mem >> 20);
1662
1663         scst_print_config();
1664
1665 out:
1666         TRACE_EXIT_RES(res);
1667         return res;
1668
1669 out_thread_free:
1670         scst_stop_all_threads();
1671
1672         scsi_unregister_interface(&scst_interface);
1673
1674 out_free_acg:
1675         scst_destroy_acg(scst_default_acg);
1676
1677 out_destroy_sgv_pool:
1678         scst_sgv_pools_deinit();
1679
1680 out_destroy_sense_mempool:
1681         mempool_destroy(scst_sense_mempool);
1682
1683 out_destroy_ua_mempool:
1684         mempool_destroy(scst_ua_mempool);
1685
1686 out_destroy_mgmt_stub_mempool:
1687         mempool_destroy(scst_mgmt_stub_mempool);
1688
1689 out_destroy_mgmt_mempool:
1690         mempool_destroy(scst_mgmt_mempool);
1691
1692 out_destroy_acg_cache:
1693         kmem_cache_destroy(scst_acgd_cachep);
1694
1695 out_destroy_tgt_cache:
1696         kmem_cache_destroy(scst_tgtd_cachep);
1697
1698 out_destroy_sess_cache:
1699         kmem_cache_destroy(scst_sess_cachep);
1700
1701 out_destroy_cmd_cache:
1702         kmem_cache_destroy(scst_cmd_cachep);
1703
1704 out_destroy_sense_cache:
1705         kmem_cache_destroy(scst_sense_cachep);
1706
1707 out_destroy_ua_cache:
1708         kmem_cache_destroy(scst_ua_cachep);
1709
1710 out_destroy_mgmt_stub_cache:
1711         kmem_cache_destroy(scst_mgmt_stub_cachep);
1712
1713 out_destroy_mgmt_cache:
1714         kmem_cache_destroy(scst_mgmt_cachep);
1715         goto out;
1716 }
1717
1718 static void __exit exit_scst(void)
1719 {
1720 #ifdef CONFIG_LOCKDEP
1721         static /* To hide the lockdep's warning about non-static key */
1722 #endif
1723         DECLARE_MUTEX_LOCKED(shm);
1724
1725         TRACE_ENTRY();
1726
1727         /* ToDo: unregister_cpu_notifier() */
1728
1729         scst_proc_cleanup_module();
1730
1731         scst_stop_all_threads();
1732
1733         scsi_unregister_interface(&scst_interface);
1734         scst_destroy_acg(scst_default_acg);
1735
1736         scst_sgv_pools_deinit();
1737
1738 #define DEINIT_CACHEP(p) do {           \
1739                 kmem_cache_destroy(p);  \
1740                 p = NULL;               \
1741         } while (0)
1742
1743         mempool_destroy(scst_mgmt_mempool);
1744         mempool_destroy(scst_mgmt_stub_mempool);
1745         mempool_destroy(scst_ua_mempool);
1746         mempool_destroy(scst_sense_mempool);
1747
1748         DEINIT_CACHEP(scst_mgmt_cachep);
1749         DEINIT_CACHEP(scst_mgmt_stub_cachep);
1750         DEINIT_CACHEP(scst_ua_cachep);
1751         DEINIT_CACHEP(scst_sense_cachep);
1752         DEINIT_CACHEP(scst_cmd_cachep);
1753         DEINIT_CACHEP(scst_sess_cachep);
1754         DEINIT_CACHEP(scst_tgtd_cachep);
1755         DEINIT_CACHEP(scst_acgd_cachep);
1756
1757         PRINT_INFO("%s", "SCST unloaded");
1758
1759         TRACE_EXIT();
1760         return;
1761 }
1762
1763
1764 module_init(init_scst);
1765 module_exit(exit_scst);
1766
1767 MODULE_AUTHOR("Vladislav Bolkhovitin");
1768 MODULE_LICENSE("GPL");
1769 MODULE_DESCRIPTION("SCSI target core");
1770 MODULE_VERSION(SCST_VERSION_STRING);