- ISCSI sending response timeout increased to 30 seconds
[mirror/scst/.git] / scst / src / scst_mem.c
1 /*
2  *  scst_mem.c
3  *
4  *  Copyright (C) 2006 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 Krzysztof Blaszkowski <kb@sysmikro.com.pl>
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/mm.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29
30 #ifdef SCST_HIGHMEM
31 #include <linux/highmem.h>
32 #endif
33
34 #include "scst.h"
35 #include "scst_priv.h"
36 #include "scst_mem.h"
37
38 #define PURGE_INTERVAL          (60 * HZ)
39 #define PURGE_TIME_AFTER        PURGE_INTERVAL
40 #define SHRINK_TIME_AFTER       (1 * HZ)
41
42 static struct scst_sgv_pools_manager sgv_pools_mgr;
43
44 void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev)
45 {
46         tgt_dev->gfp_mask = __GFP_NOWARN;
47         tgt_dev->pool = &sgv_pools_mgr.default_set.norm;
48 }
49
50 void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev)
51 {
52         TRACE_MEM("%s", "Use clustering");
53         tgt_dev->gfp_mask = __GFP_NOWARN;
54         tgt_dev->pool = &sgv_pools_mgr.default_set.norm_clust;
55 }
56
57 void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev)
58 {
59         TRACE_MEM("%s", "Use ISA DMA memory");
60         tgt_dev->gfp_mask = __GFP_NOWARN | GFP_DMA;
61         tgt_dev->pool = &sgv_pools_mgr.default_set.dma;
62 }
63
64 #ifdef SCST_HIGHMEM
65 void scst_sgv_pool_use_highmem(struct scst_tgt_dev *tgt_dev)
66 {
67         TRACE_MEM("%s", "Use HIGHMEM");
68         tgt_dev->gfp_mask = __GFP_NOWARN | __GFP_HIGHMEM;
69         tgt_dev->pool = &sgv_pools_mgr.default_set.highmem;
70 }
71 #endif
72
73 static int scst_check_clustering(struct scatterlist *sg, int cur, int hint)
74 {
75         int res = -1;
76         int i = hint;
77         unsigned long pfn_cur = page_to_pfn(sg_page(&sg[cur]));
78         int len_cur = sg[cur].length;
79         unsigned long pfn_cur_next = pfn_cur + (len_cur >> PAGE_SHIFT);
80         int full_page_cur = (len_cur & (PAGE_SIZE - 1)) == 0;
81         unsigned long pfn, pfn_next, full_page;
82
83 #ifdef SCST_HIGHMEM
84         if (page >= highmem_start_page) {
85                 TRACE_MEM("%s", "HIGHMEM page allocated, no clustering")
86                 goto out;
87         }
88 #endif
89
90 #if 0
91         TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d",
92                 pfn_cur, pfn_cur_next, len_cur, full_page_cur);
93 #endif
94
95         /* check the hint first */
96         if (i >= 0) {
97                 pfn = page_to_pfn(sg_page(&sg[i]));
98                 pfn_next = pfn + (sg[i].length >> PAGE_SHIFT);
99                 full_page = (sg[i].length & (PAGE_SIZE - 1)) == 0;
100
101                 if ((pfn == pfn_cur_next) && full_page_cur)
102                         goto out_head;
103
104                 if ((pfn_next == pfn_cur) && full_page)
105                         goto out_tail;
106         }
107
108         /* ToDo: implement more intelligent search */
109         for (i = cur - 1; i >= 0; i--) {
110                 pfn = page_to_pfn(sg_page(&sg[i]));
111                 pfn_next = pfn + (sg[i].length >> PAGE_SHIFT);
112                 full_page = (sg[i].length & (PAGE_SIZE - 1)) == 0;
113
114                 if ((pfn == pfn_cur_next) && full_page_cur)
115                         goto out_head;
116
117                 if ((pfn_next == pfn_cur) && full_page)
118                         goto out_tail;
119         }
120
121 out:
122         return res;
123
124 out_tail:
125         TRACE_MEM("SG segment %d will be tail merged with segment %d", cur, i);
126         sg[i].length += len_cur;
127         sg_clear(&sg[cur]);
128         res = i;
129         goto out;
130
131 out_head:
132         TRACE_MEM("SG segment %d will be head merged with segment %d", cur, i);
133         sg_assign_page(&sg[i], sg_page(&sg[cur]));
134         sg[i].length += len_cur;
135         sg_clear(&sg[cur]);
136         res = i;
137         goto out;
138 }
139
140 static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count,
141         void *priv)
142 {
143         int i;
144
145         TRACE_MEM("sg=%p, sg_count=%d", sg, sg_count);
146
147         for (i = 0; i < sg_count; i++) {
148                 struct page *p = sg_page(&sg[i]);
149                 int len = sg[i].length;
150                 int pages =
151                         (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
152
153                 TRACE_MEM("page %lx, len %d, pages %d",
154                         (unsigned long)p, len, pages);
155
156                 while (pages > 0) {
157                         int order = 0;
158
159 /*
160  * __free_pages() doesn't like freeing pages with not that order with
161  * which they were allocated, so disable this small optimization.
162  */
163 #if 0
164                         if (len > 0) {
165                                 while (((1 << order) << PAGE_SHIFT) < len)
166                                         order++;
167                                 len = 0;
168                         }
169 #endif
170                         TRACE_MEM("free_pages(): order %d, page %lx",
171                                 order, (unsigned long)p);
172
173                         __free_pages(p, order);
174
175                         pages -= 1 << order;
176                         p += 1 << order;
177                 }
178         }
179 }
180
181 static struct page *scst_alloc_sys_pages(struct scatterlist *sg,
182         gfp_t gfp_mask, void *priv)
183 {
184         struct page *page = alloc_pages(gfp_mask, 0);
185
186         sg_set_page(sg, page, PAGE_SIZE, 0);
187         TRACE_MEM("page=%p, sg=%p, priv=%p", page, sg, priv);
188         if (page == NULL) {
189                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of "
190                         "sg page failed");
191         }
192         return page;
193 }
194
195 static int scst_alloc_sg_entries(struct scatterlist *sg, int pages,
196         gfp_t gfp_mask, int clustered, struct trans_tbl_ent *trans_tbl,
197         const struct sgv_pool_alloc_fns *alloc_fns, void *priv)
198 {
199         int sg_count = 0;
200         int pg, i, j;
201         int merged = -1;
202
203         TRACE_MEM("pages=%d, clustered=%d", pages, clustered);
204
205 #if 0
206         gfp_mask |= __GFP_COLD;
207 #endif
208 #ifdef SCST_STRICT_SECURITY
209         gfp_mask |= __GFP_ZERO;
210 #endif
211
212         for (pg = 0; pg < pages; pg++) {
213                 void *rc;
214 #ifdef DEBUG_OOM
215                 if (((gfp_mask & __GFP_NOFAIL) != __GFP_NOFAIL) &&
216                     ((scst_random() % 10000) == 55))
217                         rc = NULL;
218                 else
219 #endif
220                         rc = alloc_fns->alloc_pages_fn(&sg[sg_count], gfp_mask,
221                                 priv);
222                 if (rc == NULL)
223                         goto out_no_mem;
224                 if (clustered) {
225                         merged = scst_check_clustering(sg, sg_count, merged);
226                         if (merged == -1)
227                                 sg_count++;
228                 } else
229                         sg_count++;
230                 TRACE_MEM("pg=%d, merged=%d, sg_count=%d", pg, merged,
231                         sg_count);
232         }
233
234         if (clustered && (trans_tbl != NULL)) {
235                 pg = 0;
236                 for (i = 0; i < pages; i++) {
237                         int n = (sg[i].length >> PAGE_SHIFT) +
238                                 ((sg[i].length & ~PAGE_MASK) != 0);
239                         trans_tbl[i].pg_count = pg;
240                         for (j = 0; j < n; j++)
241                                 trans_tbl[pg++].sg_num = i+1;
242                         TRACE_MEM("i=%d, n=%d, pg_count=%d", i, n,
243                                 trans_tbl[i].pg_count);
244                 }
245         }
246
247 out:
248         TRACE_MEM("sg_count=%d", sg_count);
249         return sg_count;
250
251 out_no_mem:
252         alloc_fns->free_pages_fn(sg, sg_count, priv);
253         sg_count = 0;
254         goto out;
255 }
256
257 static int sgv_alloc_arrays(struct sgv_pool_obj *obj,
258         int pages_to_alloc, int order, unsigned long gfp_mask)
259 {
260         int sz, tsz = 0;
261         int res = 0;
262
263         TRACE_ENTRY();
264
265         sz = pages_to_alloc * sizeof(obj->sg_entries[0]);
266
267         obj->sg_entries = kmalloc(sz, gfp_mask);
268         if (unlikely(obj->sg_entries == NULL)) {
269                 TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool_obj "
270                         "SG vector failed (size %d)", sz);
271                 res = -ENOMEM;
272                 goto out;
273         }
274
275         sg_init_table(obj->sg_entries, pages_to_alloc);
276
277         if (obj->owner_pool->clustered) {
278                 if (order <= sgv_pools_mgr.sgv_max_trans_order) {
279                         obj->trans_tbl = (struct trans_tbl_ent *)obj->sg_entries_data;
280                         /*
281                          * No need to clear trans_tbl, if needed, it will be
282                          * fully rewritten in scst_alloc_sg_entries()
283                          */
284                 } else {
285                         tsz = pages_to_alloc * sizeof(obj->trans_tbl[0]);
286                         obj->trans_tbl = kzalloc(tsz, gfp_mask);
287                         if (unlikely(obj->trans_tbl == NULL)) {
288                                 TRACE(TRACE_OUT_OF_MEM, "Allocation of trans_tbl "
289                                         "failed (size %d)", tsz);
290                                 res = -ENOMEM;
291                                 goto out_free;
292                         }
293                 }
294         }
295
296         TRACE_MEM("pages_to_alloc %d, order %d, sz %d, tsz %d, obj %p, "
297                 "sg_entries %p, trans_tbl %p", pages_to_alloc, order,
298                 sz, tsz, obj, obj->sg_entries, obj->trans_tbl);
299
300 out:
301         TRACE_EXIT_RES(res);
302         return res;
303
304 out_free:
305         kfree(obj->sg_entries);
306         obj->sg_entries = NULL;
307         goto out;
308 }
309
310 static void sgv_dtor_and_free(struct sgv_pool_obj *obj)
311 {
312         if (obj->sg_count != 0) {
313                 obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries,
314                         obj->sg_count, obj->allocator_priv);
315         }
316         if (obj->sg_entries != obj->sg_entries_data) {
317                 if (obj->trans_tbl != (struct trans_tbl_ent *)obj->sg_entries_data) {
318                         /* kfree() handles NULL parameter */
319                         kfree(obj->trans_tbl);
320                         obj->trans_tbl = NULL;
321                 }
322                 kfree(obj->sg_entries);
323         }
324
325         kmem_cache_free(obj->owner_pool->caches[obj->order_or_pages], obj);
326         return;
327 }
328
329 static struct sgv_pool_obj *sgv_pool_cached_get(struct sgv_pool *pool,
330         int order, unsigned long gfp_mask)
331 {
332         struct sgv_pool_obj *obj;
333         int pages = 1 << order;
334
335         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
336         if (likely(!list_empty(&pool->recycling_lists[order]))) {
337                 obj = list_entry(pool->recycling_lists[order].next,
338                          struct sgv_pool_obj,
339                         recycle_entry.recycling_list_entry);
340
341                 list_del(&obj->recycle_entry.sorted_recycling_list_entry);
342                 list_del(&obj->recycle_entry.recycling_list_entry);
343
344                 sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages;
345                 sgv_pools_mgr.mgr.throttle.active_pages_total += pages;
346
347                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
348
349                 EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
350                 goto out;
351         }
352
353         pool->acc.cached_entries++;
354         pool->acc.cached_pages += pages;
355
356         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
357
358         obj = kmem_cache_alloc(pool->caches[order],
359                 gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA));
360         if (likely(obj)) {
361                 memset(obj, 0, sizeof(*obj));
362                 obj->order_or_pages = order;
363                 obj->owner_pool = pool;
364         } else {
365                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
366                 pool->acc.cached_entries--;
367                 pool->acc.cached_pages -= pages;
368                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
369         }
370
371 out:
372         return obj;
373 }
374
375 static void sgv_pool_cached_put(struct sgv_pool_obj *sgv)
376 {
377         struct sgv_pool *owner = sgv->owner_pool;
378         struct list_head *entry;
379         struct list_head *list = &owner->recycling_lists[sgv->order_or_pages];
380         int sched = 0;
381         int pages = 1 << sgv->order_or_pages;
382
383         EXTRACHECKS_BUG_ON(sgv->order_or_pages < 0);
384
385         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
386
387         TRACE_MEM("sgv %p, order %d, sg_count %d", sgv, sgv->order_or_pages,
388                 sgv->sg_count);
389
390         if (owner->clustered) {
391                 /* Make objects with less entries more preferred */
392                 __list_for_each(entry, list) {
393                         struct sgv_pool_obj *tmp = list_entry(entry,
394                                 struct sgv_pool_obj,
395                                 recycle_entry.recycling_list_entry);
396                         TRACE_DBG("tmp %p, order %d, sg_count %d", tmp,
397                                 tmp->order_or_pages, tmp->sg_count);
398                         if (sgv->sg_count <= tmp->sg_count)
399                                 break;
400                 }
401                 entry = entry->prev;
402         } else
403                 entry = list;
404
405         TRACE_DBG("Adding in %p (list %p)", entry, list);
406         list_add(&sgv->recycle_entry.recycling_list_entry, entry);
407
408         list_add_tail(&sgv->recycle_entry.sorted_recycling_list_entry,
409                 &sgv_pools_mgr.mgr.sorted_recycling_list);
410
411         sgv->recycle_entry.time_stamp = jiffies;
412
413         sgv_pools_mgr.mgr.throttle.inactive_pages_total += pages;
414         sgv_pools_mgr.mgr.throttle.active_pages_total -= pages;
415
416         if (!sgv_pools_mgr.mgr.pitbool_running) {
417                 sgv_pools_mgr.mgr.pitbool_running = 1;
418                 sched = 1;
419         }
420
421         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
422
423         if (sched)
424                 schedule_delayed_work(&sgv_pools_mgr.mgr.apit_pool,
425                         PURGE_INTERVAL);
426 }
427
428 /* Must be called under pool_mgr_lock held */
429 static void __sgv_pool_cached_purge(struct sgv_pool_obj *e)
430 {
431         int pages = 1 << e->order_or_pages;
432
433         list_del(&e->recycle_entry.sorted_recycling_list_entry);
434         list_del(&e->recycle_entry.recycling_list_entry);
435         e->owner_pool->acc.cached_entries--;
436         e->owner_pool->acc.cached_pages -= pages;
437         sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages;
438
439         return;
440 }
441
442 /* Must be called under pool_mgr_lock held */
443 static int sgv_pool_cached_purge(struct sgv_pool_obj *e, int t,
444         unsigned long rt)
445 {
446         EXTRACHECKS_BUG_ON(t == 0);
447
448         if (time_after(rt, (e->recycle_entry.time_stamp + t))) {
449                 __sgv_pool_cached_purge(e);
450                 return 0;
451         }
452         return 1;
453 }
454
455 /* Called under pool_mgr_lock held, but drops/reaquires it inside */
456 static int sgv_pool_oom_free_objs(int pgs)
457 {
458         TRACE_MEM("Shrinking pools about %d pages", pgs);
459         while ((sgv_pools_mgr.mgr.throttle.inactive_pages_total >
460                         sgv_pools_mgr.mgr.throttle.lo_wmk) &&
461               (pgs > 0)) {
462                 struct sgv_pool_obj *e;
463
464                 sBUG_ON(list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list));
465
466                 e = list_entry(sgv_pools_mgr.mgr.sorted_recycling_list.next,
467                                struct sgv_pool_obj,
468                                recycle_entry.sorted_recycling_list_entry);
469
470                 __sgv_pool_cached_purge(e);
471                 pgs -= 1 << e->order_or_pages;
472
473                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
474                 sgv_dtor_and_free(e);
475                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
476         }
477
478         TRACE_MEM("Pages remaining %d ", pgs);
479         return pgs;
480 }
481
482 static int sgv_pool_hiwmk_check(int pages_to_alloc)
483 {
484         int res = 0;
485         int pages = pages_to_alloc;
486
487         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
488
489         pages += sgv_pools_mgr.mgr.throttle.active_pages_total;
490         pages += sgv_pools_mgr.mgr.throttle.inactive_pages_total;
491
492         if (unlikely((u32)pages > sgv_pools_mgr.mgr.throttle.hi_wmk)) {
493                 pages -= sgv_pools_mgr.mgr.throttle.hi_wmk;
494                 sgv_pools_mgr.mgr.throttle.releases_on_hiwmk++;
495
496                 pages = sgv_pool_oom_free_objs(pages);
497                 if (pages > 0) {
498                         TRACE(TRACE_OUT_OF_MEM, "Requested amount of "
499                             "memory (%d pages) for being executed "
500                             "commands together with the already "
501                             "allocated memory exceeds the allowed "
502                             "maximum %d. Should you increase "
503                             "scst_max_cmd_mem?", pages_to_alloc,
504                            sgv_pools_mgr.mgr.throttle.hi_wmk);
505                         sgv_pools_mgr.mgr.throttle.releases_failed++;
506                         res = -ENOMEM;
507                         goto out_unlock;
508                 }
509         }
510
511         sgv_pools_mgr.mgr.throttle.active_pages_total += pages_to_alloc;
512
513 out_unlock:
514         TRACE_MEM("pages_to_alloc %d, new active %d", pages_to_alloc,
515                 sgv_pools_mgr.mgr.throttle.active_pages_total);
516
517         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
518         return res;
519 }
520
521 static void sgv_pool_hiwmk_uncheck(int pages)
522 {
523         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
524         sgv_pools_mgr.mgr.throttle.active_pages_total -= pages;
525         TRACE_MEM("pages %d, new active %d", pages,
526                 sgv_pools_mgr.mgr.throttle.active_pages_total);
527         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
528         return;
529 }
530
531 static bool scst_check_allowed_mem(struct scst_mem_lim *mem_lim, int pages)
532 {
533         int alloced;
534         bool res = true;
535
536         alloced = atomic_add_return(pages, &mem_lim->alloced_pages);
537         if (unlikely(alloced > mem_lim->max_allowed_pages)) {
538                 TRACE(TRACE_OUT_OF_MEM, "Requested amount of memory "
539                         "(%d pages) for being executed commands on a device "
540                         "together with the already allocated memory exceeds "
541                         "the allowed maximum %d. Should you increase "
542                         "scst_max_dev_cmd_mem?", pages,
543                         mem_lim->max_allowed_pages);
544                 atomic_sub(pages, &mem_lim->alloced_pages);
545                 res = false;
546         }
547
548         TRACE_MEM("mem_lim %p, pages %d, res %d, new alloced %d", mem_lim,
549                 pages, res, atomic_read(&mem_lim->alloced_pages));
550
551         return res;
552 }
553
554 static void scst_uncheck_allowed_mem(struct scst_mem_lim *mem_lim, int pages)
555 {
556         atomic_sub(pages, &mem_lim->alloced_pages);
557
558         TRACE_MEM("mem_lim %p, pages %d, new alloced %d", mem_lim,
559                 pages, atomic_read(&mem_lim->alloced_pages));
560         return;
561 }
562
563 struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
564         unsigned long gfp_mask, int flags, int *count,
565         struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv)
566 {
567         struct sgv_pool_obj *obj;
568         int order, pages, cnt;
569         struct scatterlist *res = NULL;
570         int pages_to_alloc;
571         struct kmem_cache *cache;
572         int no_cached = flags & SCST_POOL_ALLOC_NO_CACHED;
573         bool allowed_mem_checked = false, hiwmk_checked = false;
574
575         TRACE_ENTRY();
576
577         if (unlikely(size == 0))
578                 goto out;
579
580         sBUG_ON((gfp_mask & __GFP_NOFAIL) == __GFP_NOFAIL);
581
582         pages = ((size + PAGE_SIZE - 1) >> PAGE_SHIFT);
583         order = get_order(size);
584
585         TRACE_MEM("size=%d, pages=%d, order=%d, flags=%x, *sgv %p", size, pages,
586                 order, flags, *sgv);
587
588         if (*sgv != NULL) {
589                 obj = *sgv;
590                 pages_to_alloc = (1 << order);
591                 cache = pool->caches[obj->order_or_pages];
592
593                 TRACE_MEM("Supplied sgv_obj %p, sgv_order %d", obj,
594                         obj->order_or_pages);
595
596                 EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
597                 EXTRACHECKS_BUG_ON(obj->sg_count != 0);
598
599                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
600                         goto out_fail_free_sg_entries;
601                 allowed_mem_checked = true;
602
603                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
604                         goto out_fail_free_sg_entries;
605                 hiwmk_checked = true;
606         } else if ((order < SGV_POOL_ELEMENTS) && !no_cached) {
607                 pages_to_alloc = (1 << order);
608                 cache = pool->caches[order];
609
610                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
611                         goto out_fail;
612                 allowed_mem_checked = true;
613
614                 obj = sgv_pool_cached_get(pool, order, gfp_mask);
615                 if (unlikely(obj == NULL)) {
616                         TRACE(TRACE_OUT_OF_MEM, "Allocation of "
617                                 "sgv_pool_obj failed (size %d)", size);
618                         goto out_fail;
619                 }
620
621                 if (obj->sg_count != 0) {
622                         TRACE_MEM("Cached sgv_obj %p", obj);
623                         EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
624                         atomic_inc(&pool->cache_acc[order].hit_alloc);
625                         goto success;
626                 }
627
628                 if (flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS) {
629                         if (!(flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL))
630                                 goto out_fail_free;
631                 }
632
633                 TRACE_MEM("Brand new sgv_obj %p", obj);
634
635                 if (order <= sgv_pools_mgr.sgv_max_local_order) {
636                         obj->sg_entries = obj->sg_entries_data;
637                         sg_init_table(obj->sg_entries, pages_to_alloc);
638                         TRACE_MEM("sg_entries %p", obj->sg_entries);
639                         if (pool->clustered) {
640                                 obj->trans_tbl = (struct trans_tbl_ent *)
641                                         (obj->sg_entries + pages_to_alloc);
642                                 TRACE_MEM("trans_tbl %p", obj->trans_tbl);
643                                 /*
644                                  * No need to clear trans_tbl, if needed, it will
645                                  * be fully rewritten in scst_alloc_sg_entries()
646                                  */
647                         }
648                 } else {
649                         if (unlikely(sgv_alloc_arrays(obj, pages_to_alloc,
650                                         order, gfp_mask) != 0))
651                                 goto out_fail_free;
652                 }
653
654                 if ((flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS) &&
655                     (flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL))
656                         goto out_return;
657
658                 obj->allocator_priv = priv;
659
660                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
661                         goto out_fail_free_sg_entries;
662                 hiwmk_checked = true;
663         } else {
664                 int sz;
665
666                 pages_to_alloc = pages;
667
668                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
669                         goto out_fail;
670                 allowed_mem_checked = true;
671
672                 if (flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS)
673                         goto out_return2;
674
675                 cache = NULL;
676                 sz = sizeof(*obj) + pages*sizeof(obj->sg_entries[0]);
677
678                 obj = kmalloc(sz, gfp_mask);
679                 if (unlikely(obj == NULL)) {
680                         TRACE(TRACE_OUT_OF_MEM, "Allocation of "
681                                 "sgv_pool_obj failed (size %d)", size);
682                         goto out_fail;
683                 }
684                 memset(obj, 0, sizeof(*obj));
685
686                 obj->owner_pool = pool;
687                 obj->order_or_pages = -pages_to_alloc;
688                 obj->allocator_priv = priv;
689
690                 obj->sg_entries = obj->sg_entries_data;
691                 sg_init_table(obj->sg_entries, pages);
692
693                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
694                         goto out_fail_free_sg_entries;
695                 hiwmk_checked = true;
696
697                 TRACE_MEM("Big or no_cached sgv_obj %p (size %d)", obj, sz);
698         }
699
700         obj->sg_count = scst_alloc_sg_entries(obj->sg_entries,
701                 pages_to_alloc, gfp_mask, pool->clustered, obj->trans_tbl,
702                 &pool->alloc_fns, priv);
703         if (unlikely(obj->sg_count <= 0)) {
704                 obj->sg_count = 0;
705                 if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache)
706                         goto out_return1;
707                 else
708                         goto out_fail_free_sg_entries;
709         }
710
711         if (cache) {
712                 atomic_add(pages_to_alloc - obj->sg_count,
713                         &pool->cache_acc[order].merged);
714         } else {
715                 if (no_cached) {
716                         atomic_add(pages_to_alloc,
717                                 &pool->acc.other_pages);
718                         atomic_add(pages_to_alloc - obj->sg_count,
719                                 &pool->acc.other_merged);
720                 } else {
721                         atomic_add(pages_to_alloc,
722                                 &pool->acc.big_pages);
723                         atomic_add(pages_to_alloc - obj->sg_count,
724                                 &pool->acc.big_merged);
725                 }
726         }
727
728 success:
729         if (cache) {
730                 int sg;
731                 atomic_inc(&pool->cache_acc[order].total_alloc);
732                 if (pool->clustered)
733                         cnt = obj->trans_tbl[pages-1].sg_num;
734                 else
735                         cnt = pages;
736                 sg = cnt-1;
737                 obj->orig_sg = sg;
738                 obj->orig_length = obj->sg_entries[sg].length;
739                 if (pool->clustered) {
740                         obj->sg_entries[sg].length =
741                                 (pages - obj->trans_tbl[sg].pg_count) << PAGE_SHIFT;
742                 }
743         } else {
744                 cnt = obj->sg_count;
745                 if (no_cached)
746                         atomic_inc(&pool->acc.other_alloc);
747                 else
748                         atomic_inc(&pool->acc.big_alloc);
749         }
750
751         *count = cnt;
752         res = obj->sg_entries;
753         *sgv = obj;
754
755         if (size & ~PAGE_MASK)
756                 obj->sg_entries[cnt-1].length -= PAGE_SIZE - (size & ~PAGE_MASK);
757
758         TRACE_MEM("sgv_obj=%p, sg_entries %p (size=%d, pages=%d, sg_count=%d, "
759                 "count=%d, last_len=%d)", obj, obj->sg_entries, size, pages,
760                 obj->sg_count, *count, obj->sg_entries[obj->orig_sg].length);
761
762 out:
763         TRACE_EXIT_HRES(res);
764         return res;
765
766 out_return:
767         obj->allocator_priv = priv;
768         obj->owner_pool = pool;
769
770 out_return1:
771         *sgv = obj;
772         TRACE_MEM("Returning failed sgv_obj %p (count %d)", obj, *count);
773
774 out_return2:
775         *count = pages_to_alloc;
776         res = NULL;
777         goto out_uncheck;
778
779 out_fail_free_sg_entries:
780         if (obj->sg_entries != obj->sg_entries_data) {
781                 if (obj->trans_tbl != (struct trans_tbl_ent *)obj->sg_entries_data) {
782                         /* kfree() handles NULL parameter */
783                         kfree(obj->trans_tbl);
784                         obj->trans_tbl = NULL;
785                 }
786                 kfree(obj->sg_entries);
787                 obj->sg_entries = NULL;
788         }
789
790 out_fail_free:
791         if (cache)
792                 sgv_pool_cached_put(obj);
793         else
794                 kfree(obj);
795
796 out_fail:
797         res = NULL;
798         *count = 0;
799         *sgv = NULL;
800         TRACE_MEM("%s", "Allocation failed");
801
802 out_uncheck:
803         if (hiwmk_checked)
804                 sgv_pool_hiwmk_uncheck(pages_to_alloc);
805         if (allowed_mem_checked)
806                 scst_uncheck_allowed_mem(mem_lim, pages_to_alloc);
807         goto out;
808 }
809 EXPORT_SYMBOL(sgv_pool_alloc);
810
811 void *sgv_get_priv(struct sgv_pool_obj *sgv)
812 {
813         return sgv->allocator_priv;
814 }
815 EXPORT_SYMBOL(sgv_get_priv);
816
817 void sgv_pool_free(struct sgv_pool_obj *sgv, struct scst_mem_lim *mem_lim)
818 {
819         int pages;
820
821         TRACE_MEM("Freeing sgv_obj %p, order %d, sg_entries %p, "
822                 "sg_count %d, allocator_priv %p", sgv, sgv->order_or_pages,
823                 sgv->sg_entries, sgv->sg_count, sgv->allocator_priv);
824
825         if (sgv->order_or_pages >= 0) {
826                 sgv->sg_entries[sgv->orig_sg].length = sgv->orig_length;
827                 pages = (sgv->sg_count != 0) ? 1 << sgv->order_or_pages : 0;
828                 sgv_pool_cached_put(sgv);
829         } else {
830                 sgv->owner_pool->alloc_fns.free_pages_fn(sgv->sg_entries,
831                         sgv->sg_count, sgv->allocator_priv);
832                 pages = (sgv->sg_count != 0) ? -sgv->order_or_pages : 0;
833                 kfree(sgv);
834                 sgv_pool_hiwmk_uncheck(pages);
835         }
836
837         scst_uncheck_allowed_mem(mem_lim, pages);
838
839         return;
840 }
841 EXPORT_SYMBOL(sgv_pool_free);
842
843 struct scatterlist *scst_alloc(int size, unsigned long gfp_mask, int *count)
844 {
845         struct scatterlist *res;
846         int pages = (size >> PAGE_SHIFT) + ((size & ~PAGE_MASK) != 0);
847         struct sgv_pool_alloc_fns sys_alloc_fns = {
848                 scst_alloc_sys_pages, scst_free_sys_sg_entries };
849         int no_fail = ((gfp_mask & __GFP_NOFAIL) == __GFP_NOFAIL);
850
851         TRACE_ENTRY();
852
853         atomic_inc(&sgv_pools_mgr.sgv_other_total_alloc);
854
855         if (unlikely(!no_fail)) {
856                 if (unlikely(sgv_pool_hiwmk_check(pages) != 0)) {
857                         res = NULL;
858                         goto out;
859                 }
860         }
861
862         res = kmalloc(pages*sizeof(*res), gfp_mask);
863         if (res == NULL) {
864                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate sg for %d pages",
865                         pages);
866                 goto out_uncheck;
867         }
868
869         sg_init_table(res, pages);
870
871         /*
872          * If we allow use clustering here, we will have troubles in
873          * scst_free() to figure out how many pages are in the SG vector.
874          * So, always don't use clustering.
875          */
876         *count = scst_alloc_sg_entries(res, pages, gfp_mask, 0, NULL,
877                         &sys_alloc_fns, NULL);
878         if (*count <= 0)
879                 goto out_free;
880
881 out:
882         TRACE_MEM("Alloced sg %p (count %d)", res, *count);
883
884         TRACE_EXIT_HRES(res);
885         return res;
886
887 out_free:
888         kfree(res);
889         res = NULL;
890
891 out_uncheck:
892         if (!no_fail)
893                 sgv_pool_hiwmk_uncheck(pages);
894         goto out;
895 }
896 EXPORT_SYMBOL(scst_alloc);
897
898 void scst_free(struct scatterlist *sg, int count)
899 {
900         TRACE_MEM("Freeing sg=%p", sg);
901
902         sgv_pool_hiwmk_uncheck(count);
903
904         scst_free_sys_sg_entries(sg, count, NULL);
905         kfree(sg);
906         return;
907 }
908 EXPORT_SYMBOL(scst_free);
909
910 static void sgv_pool_cached_init(struct sgv_pool *pool)
911 {
912         int i;
913         for (i = 0; i < SGV_POOL_ELEMENTS; i++)
914                 INIT_LIST_HEAD(&pool->recycling_lists[i]);
915 }
916
917 int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered)
918 {
919         int res = -ENOMEM;
920         int i;
921         struct sgv_pool_obj *obj;
922
923         TRACE_ENTRY();
924
925         memset(pool, 0, sizeof(*pool));
926
927         atomic_set(&pool->acc.other_alloc, 0);
928         atomic_set(&pool->acc.big_alloc, 0);
929         atomic_set(&pool->acc.other_pages, 0);
930         atomic_set(&pool->acc.big_pages, 0);
931         atomic_set(&pool->acc.other_merged, 0);
932         atomic_set(&pool->acc.big_merged, 0);
933
934         pool->clustered = clustered;
935         pool->alloc_fns.alloc_pages_fn = scst_alloc_sys_pages;
936         pool->alloc_fns.free_pages_fn = scst_free_sys_sg_entries;
937
938         TRACE_MEM("name %s, sizeof(*obj)=%zd, clustered=%d", name, sizeof(*obj),
939                 clustered);
940
941         strncpy(pool->name, name, sizeof(pool->name)-1);
942         pool->name[sizeof(pool->name)-1] = '\0';
943
944         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
945                 int size;
946
947                 atomic_set(&pool->cache_acc[i].total_alloc, 0);
948                 atomic_set(&pool->cache_acc[i].hit_alloc, 0);
949                 atomic_set(&pool->cache_acc[i].merged, 0);
950
951                 if (i <= sgv_pools_mgr.sgv_max_local_order) {
952                         size = sizeof(*obj) + (1 << i) *
953                                 (sizeof(obj->sg_entries[0]) +
954                                  (clustered ? sizeof(obj->trans_tbl[0]) : 0));
955                 } else if (i <= sgv_pools_mgr.sgv_max_trans_order) {
956                         /* sgv ie sg_entries is allocated outside object but ttbl
957                         is embedded still */
958                         size = sizeof(*obj) + (1 << i) *
959                                 ((clustered ? sizeof(obj->trans_tbl[0]) : 0));
960                 } else {
961                         size = sizeof(*obj);
962
963                         /* both sgv and ttbl are kallocated() */
964                 }
965
966                 TRACE_MEM("pages=%d, size=%d", 1 << i, size);
967
968                 scnprintf(pool->cache_names[i], sizeof(pool->cache_names[i]),
969                         "%s-%luK", name, (PAGE_SIZE >> 10) << i);
970                 pool->caches[i] = kmem_cache_create(pool->cache_names[i],
971                         size, 0, SCST_SLAB_FLAGS, NULL
972 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
973                         , NULL);
974 #else
975                         );
976 #endif
977                 if (pool->caches[i] == NULL) {
978                         TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool cache "
979                                 "%s(%d) failed", name, i);
980                         goto out_free;
981                 }
982         }
983
984         sgv_pool_cached_init(pool);
985
986         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
987         list_add_tail(&pool->sgv_pool_list_entry,
988                 &sgv_pools_mgr.scst_sgv_pool_list);
989         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
990
991         res = 0;
992
993 out:
994         TRACE_EXIT_RES(res);
995         return res;
996
997 out_free:
998         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
999                 if (pool->caches[i]) {
1000                         kmem_cache_destroy(pool->caches[i]);
1001                         pool->caches[i] = NULL;
1002                 } else
1003                         break;
1004         }
1005         goto out;
1006 }
1007
1008 static void sgv_pool_evaluate_local_order(struct scst_sgv_pools_manager *pmgr)
1009 {
1010         int space4sgv_ttbl = PAGE_SIZE - sizeof(struct sgv_pool_obj);
1011
1012         pmgr->sgv_max_local_order = get_order(
1013                 (((space4sgv_ttbl /
1014                   (sizeof(struct trans_tbl_ent) + sizeof(struct scatterlist))) *
1015                         PAGE_SIZE) & PAGE_MASK)) - 1;
1016
1017         pmgr->sgv_max_trans_order = get_order(
1018                 (((space4sgv_ttbl /
1019                   (sizeof(struct trans_tbl_ent))) * PAGE_SIZE) & PAGE_MASK)) - 1;
1020
1021         TRACE_MEM("sgv_max_local_order %d, sgv_max_trans_order %d",
1022                 pmgr->sgv_max_local_order, pmgr->sgv_max_trans_order);
1023         TRACE_MEM("max object size with embedded sgv & ttbl %zd",
1024                 (1 << pmgr->sgv_max_local_order) *
1025                 (sizeof(struct trans_tbl_ent) + sizeof(struct scatterlist))
1026                 + sizeof(struct sgv_pool_obj));
1027         TRACE_MEM("max object size with embedded sgv (!clustered) %zd",
1028                 (1 << pmgr->sgv_max_local_order) *
1029                 (sizeof(struct scatterlist))
1030                 + sizeof(struct sgv_pool_obj));
1031         TRACE_MEM("max object size with embedded ttbl %zd",
1032                 (1 << pmgr->sgv_max_trans_order) * sizeof(struct trans_tbl_ent) +
1033                 sizeof(struct sgv_pool_obj));
1034 }
1035
1036 void sgv_pool_deinit(struct sgv_pool *pool)
1037 {
1038         int i;
1039
1040         TRACE_ENTRY();
1041
1042         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1043         list_del(&pool->sgv_pool_list_entry);
1044         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1045
1046         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1047                 struct sgv_pool_obj *e;
1048
1049                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1050                 while (!list_empty(&pool->recycling_lists[i])) {
1051                         e = list_entry(pool->recycling_lists[i].next,
1052                                  struct sgv_pool_obj,
1053                                 recycle_entry.recycling_list_entry);
1054
1055                         __sgv_pool_cached_purge(e);
1056                         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1057
1058                         EXTRACHECKS_BUG_ON(e->owner_pool != pool);
1059                         sgv_dtor_and_free(e);
1060
1061                         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1062                 }
1063                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1064
1065                 if (pool->caches[i])
1066                         kmem_cache_destroy(pool->caches[i]);
1067                 pool->caches[i] = NULL;
1068         }
1069
1070         TRACE_EXIT();
1071 }
1072
1073 void sgv_pool_set_allocator(struct sgv_pool *pool,
1074         struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *),
1075         void (*free_pages_fn)(struct scatterlist *, int, void *))
1076 {
1077         pool->alloc_fns.alloc_pages_fn = alloc_pages_fn;
1078         pool->alloc_fns.free_pages_fn = free_pages_fn;
1079         return;
1080 }
1081 EXPORT_SYMBOL(sgv_pool_set_allocator);
1082
1083 struct sgv_pool *sgv_pool_create(const char *name, int clustered)
1084 {
1085         struct sgv_pool *pool;
1086         int rc;
1087
1088         TRACE_ENTRY();
1089
1090         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
1091         if (pool == NULL) {
1092                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of sgv_pool failed");
1093                 goto out;
1094         }
1095
1096         rc = sgv_pool_init(pool, name, clustered);
1097         if (rc != 0)
1098                 goto out_free;
1099
1100 out:
1101         TRACE_EXIT_RES(pool != NULL);
1102         return pool;
1103
1104 out_free:
1105         kfree(pool);
1106         pool = NULL;
1107         goto out;
1108 }
1109 EXPORT_SYMBOL(sgv_pool_create);
1110
1111 void sgv_pool_destroy(struct sgv_pool *pool)
1112 {
1113         TRACE_ENTRY();
1114
1115         sgv_pool_deinit(pool);
1116         kfree(pool);
1117
1118         TRACE_EXIT();
1119 }
1120 EXPORT_SYMBOL(sgv_pool_destroy);
1121
1122 static int sgv_pool_cached_shrinker(int nr, gfp_t gfpm)
1123 {
1124         TRACE_ENTRY();
1125
1126         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1127
1128         if (nr > 0) {
1129                 struct sgv_pool_obj *e;
1130                 unsigned long rt = jiffies;
1131
1132                 while (!list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list)) {
1133                         e = list_entry(
1134                                 sgv_pools_mgr.mgr.sorted_recycling_list.next,
1135                                 struct sgv_pool_obj,
1136                                 recycle_entry.sorted_recycling_list_entry);
1137
1138                         if (sgv_pool_cached_purge(e, SHRINK_TIME_AFTER, rt) == 0) {
1139                                 nr -= 1 << e->order_or_pages;
1140                                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1141                                 sgv_dtor_and_free(e);
1142                                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1143                         } else
1144                                 break;
1145
1146                         if (nr <= 0)
1147                                 break;
1148                 }
1149         }
1150
1151         nr = sgv_pools_mgr.mgr.throttle.inactive_pages_total;
1152
1153         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1154
1155         TRACE_EXIT();
1156         return nr;
1157 }
1158
1159 static void sgv_pool_cached_pitbool(void *p)
1160 {
1161         u32 total_pages;
1162         struct sgv_pool_obj *e;
1163         unsigned long rt = jiffies;
1164
1165         TRACE_ENTRY();
1166
1167         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1168
1169         sgv_pools_mgr.mgr.pitbool_running = 0;
1170
1171         while (!list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list)) {
1172                 e = list_entry(sgv_pools_mgr.mgr.sorted_recycling_list.next,
1173                         struct sgv_pool_obj,
1174                         recycle_entry.sorted_recycling_list_entry);
1175
1176                 if (sgv_pool_cached_purge(e, PURGE_TIME_AFTER, rt) == 0) {
1177                         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1178                         sgv_dtor_and_free(e);
1179                         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1180                 } else
1181                         break;
1182         }
1183
1184         total_pages = sgv_pools_mgr.mgr.throttle.inactive_pages_total;
1185
1186         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1187
1188         if (total_pages) {
1189                 schedule_delayed_work(&sgv_pools_mgr.mgr.apit_pool,
1190                         PURGE_INTERVAL);
1191         }
1192
1193         TRACE_EXIT();
1194         return;
1195 }
1196
1197 /* Both parameters in pages */
1198 int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
1199 {
1200         int res;
1201         struct scst_sgv_pools_manager *pools = &sgv_pools_mgr;
1202
1203         TRACE_ENTRY();
1204
1205         memset(pools, 0, sizeof(*pools));
1206
1207         sgv_pools_mgr.mgr.throttle.hi_wmk = mem_hwmark;
1208         sgv_pools_mgr.mgr.throttle.lo_wmk = mem_lwmark;
1209
1210         sgv_pool_evaluate_local_order(&sgv_pools_mgr);
1211
1212         atomic_set(&pools->sgv_other_total_alloc, 0);
1213         INIT_LIST_HEAD(&pools->scst_sgv_pool_list);
1214         mutex_init(&pools->scst_sgv_pool_mutex);
1215
1216         INIT_LIST_HEAD(&pools->mgr.sorted_recycling_list);
1217         spin_lock_init(&pools->mgr.pool_mgr_lock);
1218
1219         res = sgv_pool_init(&pools->default_set.norm, "sgv", 0);
1220         if (res != 0)
1221                 goto out;
1222
1223         res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust", 1);
1224         if (res != 0)
1225                 goto out_free_clust;
1226
1227         res = sgv_pool_init(&pools->default_set.dma, "sgv-dma", 0);
1228         if (res != 0)
1229                 goto out_free_norm;
1230
1231 #ifdef SCST_HIGHMEM
1232         res = sgv_pool_init(&pools->default_set.highmem, "sgv-high", 0);
1233         if (res != 0)
1234                 goto out_free_dma;
1235 #endif
1236
1237 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20))
1238         INIT_DELAYED_WORK(&pools->mgr.apit_pool,
1239                 (void (*)(struct work_struct *))sgv_pool_cached_pitbool);
1240 #else
1241         INIT_WORK(&pools->mgr.apit_pool, sgv_pool_cached_pitbool, NULL);
1242 #endif
1243
1244 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
1245         pools->mgr.sgv_shrinker = set_shrinker(DEFAULT_SEEKS,
1246                 sgv_pool_cached_shrinker);
1247 #else
1248         pools->mgr.sgv_shrinker.shrink = sgv_pool_cached_shrinker;
1249         pools->mgr.sgv_shrinker.seeks = DEFAULT_SEEKS;
1250         register_shrinker(&pools->mgr.sgv_shrinker);
1251 #endif
1252
1253 out:
1254         TRACE_EXIT_RES(res);
1255         return res;
1256
1257 #ifdef SCST_HIGHMEM
1258 out_free_dma:
1259         sgv_pool_deinit(&pools->default_set.dma);
1260 #endif
1261
1262 out_free_norm:
1263         sgv_pool_deinit(&pools->default_set.norm);
1264
1265 out_free_clust:
1266         sgv_pool_deinit(&pools->default_set.norm_clust);
1267         goto out;
1268 }
1269
1270 void scst_sgv_pools_deinit(void)
1271 {
1272         struct scst_sgv_pools_manager *pools = &sgv_pools_mgr;
1273
1274         TRACE_ENTRY();
1275
1276 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
1277         remove_shrinker(pools->mgr.sgv_shrinker);
1278 #else
1279         unregister_shrinker(&pools->mgr.sgv_shrinker);
1280 #endif
1281
1282         cancel_delayed_work(&pools->mgr.apit_pool);
1283
1284 #ifdef SCST_HIGHMEM
1285         sgv_pool_deinit(&pools->default_set.highmem);
1286 #endif
1287         sgv_pool_deinit(&pools->default_set.dma);
1288         sgv_pool_deinit(&pools->default_set.norm);
1289         sgv_pool_deinit(&pools->default_set.norm_clust);
1290
1291         flush_scheduled_work();
1292
1293         TRACE_EXIT();
1294         return;
1295 }
1296
1297 static void scst_do_sgv_read(struct seq_file *seq, const struct sgv_pool *pool)
1298 {
1299         int i, total = 0, hit = 0, merged = 0, allocated = 0;
1300         int oa, om;
1301
1302         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1303                 int t;
1304
1305                 hit += atomic_read(&pool->cache_acc[i].hit_alloc);
1306                 total += atomic_read(&pool->cache_acc[i].total_alloc);
1307
1308                 t = atomic_read(&pool->cache_acc[i].total_alloc) -
1309                         atomic_read(&pool->cache_acc[i].hit_alloc);
1310                 allocated += t * (1 << i);
1311                 merged += atomic_read(&pool->cache_acc[i].merged);
1312         }
1313
1314         seq_printf(seq, "\n%-30s %-11d %-11d %-11d %d/%d (P/O)\n", pool->name,
1315                 hit, total, (allocated != 0) ? merged*100/allocated : 0,
1316                 pool->acc.cached_pages, pool->acc.cached_entries);
1317
1318         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1319                 int t = atomic_read(&pool->cache_acc[i].total_alloc) -
1320                         atomic_read(&pool->cache_acc[i].hit_alloc);
1321                 allocated = t * (1 << i);
1322                 merged = atomic_read(&pool->cache_acc[i].merged);
1323
1324                 seq_printf(seq, "  %-28s %-11d %-11d %d\n",
1325                         pool->cache_names[i],
1326                         atomic_read(&pool->cache_acc[i].hit_alloc),
1327                         atomic_read(&pool->cache_acc[i].total_alloc),
1328                         (allocated != 0) ? merged*100/allocated : 0);
1329         }
1330
1331         allocated = atomic_read(&pool->acc.big_pages);
1332         merged = atomic_read(&pool->acc.big_merged);
1333         oa = atomic_read(&pool->acc.other_pages);
1334         om = atomic_read(&pool->acc.other_merged);
1335
1336         seq_printf(seq, "  %-40s %d/%-9d %d/%d\n", "big/other",
1337                    atomic_read(&pool->acc.big_alloc),
1338                    atomic_read(&pool->acc.other_alloc),
1339                    (allocated != 0) ? merged*100/allocated : 0,
1340                    (oa != 0) ? om/oa : 0);
1341
1342         return;
1343 }
1344
1345 int sgv_pool_procinfo_show(struct seq_file *seq, void *v)
1346 {
1347         struct sgv_pool *pool;
1348
1349         TRACE_ENTRY();
1350
1351         seq_printf(seq, "%-42s %d/%d\n%-42s %d/%d\n%-42s %d/%d\n\n",
1352                 "Inactive/active pages",
1353                 sgv_pools_mgr.mgr.throttle.inactive_pages_total,
1354                 sgv_pools_mgr.mgr.throttle.active_pages_total,
1355                 "Hi/lo watermarks [pages]", sgv_pools_mgr.mgr.throttle.hi_wmk,
1356                 sgv_pools_mgr.mgr.throttle.lo_wmk, "Hi watermark releases/failures",
1357                 sgv_pools_mgr.mgr.throttle.releases_on_hiwmk,
1358                 sgv_pools_mgr.mgr.throttle.releases_failed);
1359
1360         seq_printf(seq, "%-30s %-11s %-11s %-11s %-11s", "Name", "Hit", "Total",
1361                 "% merged", "Cached");
1362
1363         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1364         list_for_each_entry(pool, &sgv_pools_mgr.scst_sgv_pool_list,
1365                         sgv_pool_list_entry) {
1366                 scst_do_sgv_read(seq, pool);
1367         }
1368         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1369
1370         seq_printf(seq, "\n%-42s %-11d\n", "other",
1371                 atomic_read(&sgv_pools_mgr.sgv_other_total_alloc));
1372
1373         TRACE_EXIT();
1374         return 0;
1375 }