dc0e05bcd4eb875894c30d04d78ee7134acfcc49
[mirror/scst/.git] / scst / src / scst_mem.c
1 /*
2  *  scst_mem.c
3  *
4  *  Copyright (C) 2006 - 2009 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2007 Krzysztof Blaszkowski <kb@sysmikro.com.pl>
6  *  Copyright (C) 2007 - 2009 ID7 Ltd.
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/mm.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29
30 #include "scst.h"
31 #include "scst_priv.h"
32 #include "scst_mem.h"
33
34 #define PURGE_INTERVAL          (60 * HZ)
35 #define PURGE_TIME_AFTER        PURGE_INTERVAL
36 #define SHRINK_TIME_AFTER       (1 * HZ)
37
38 static struct scst_sgv_pools_manager sgv_pools_mgr;
39
40 static inline bool sgv_pool_clustered(const struct sgv_pool *pool)
41 {
42         return pool->clustering_type != sgv_no_clustering;
43 }
44
45 void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev)
46 {
47         tgt_dev->gfp_mask = __GFP_NOWARN;
48         tgt_dev->pool = &sgv_pools_mgr.default_set.norm;
49         clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
50 }
51
52 void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev)
53 {
54         TRACE_MEM("%s", "Use clustering");
55         tgt_dev->gfp_mask = __GFP_NOWARN;
56         tgt_dev->pool = &sgv_pools_mgr.default_set.norm_clust;
57         set_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
58 }
59
60 void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev)
61 {
62         TRACE_MEM("%s", "Use ISA DMA memory");
63         tgt_dev->gfp_mask = __GFP_NOWARN | GFP_DMA;
64         tgt_dev->pool = &sgv_pools_mgr.default_set.dma;
65         clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
66 }
67
68 static int sgv_check_full_clustering(struct scatterlist *sg, int cur, int hint)
69 {
70         int res = -1;
71         int i = hint;
72         unsigned long pfn_cur = page_to_pfn(sg_page(&sg[cur]));
73         int len_cur = sg[cur].length;
74         unsigned long pfn_cur_next = pfn_cur + (len_cur >> PAGE_SHIFT);
75         int full_page_cur = (len_cur & (PAGE_SIZE - 1)) == 0;
76         unsigned long pfn, pfn_next;
77         bool full_page;
78
79 #if 0
80         TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d",
81                 pfn_cur, pfn_cur_next, len_cur, full_page_cur);
82 #endif
83
84         /* check the hint first */
85         if (i >= 0) {
86                 pfn = page_to_pfn(sg_page(&sg[i]));
87                 pfn_next = pfn + (sg[i].length >> PAGE_SHIFT);
88                 full_page = (sg[i].length & (PAGE_SIZE - 1)) == 0;
89
90                 if ((pfn == pfn_cur_next) && full_page_cur)
91                         goto out_head;
92
93                 if ((pfn_next == pfn_cur) && full_page)
94                         goto out_tail;
95         }
96
97         /* ToDo: implement more intelligent search */
98         for (i = cur - 1; i >= 0; i--) {
99                 pfn = page_to_pfn(sg_page(&sg[i]));
100                 pfn_next = pfn + (sg[i].length >> PAGE_SHIFT);
101                 full_page = (sg[i].length & (PAGE_SIZE - 1)) == 0;
102
103                 if ((pfn == pfn_cur_next) && full_page_cur)
104                         goto out_head;
105
106                 if ((pfn_next == pfn_cur) && full_page)
107                         goto out_tail;
108         }
109
110 out:
111         return res;
112
113 out_tail:
114         TRACE_MEM("SG segment %d will be tail merged with segment %d", cur, i);
115         sg[i].length += len_cur;
116         sg_clear(&sg[cur]);
117         res = i;
118         goto out;
119
120 out_head:
121         TRACE_MEM("SG segment %d will be head merged with segment %d", cur, i);
122         sg_assign_page(&sg[i], sg_page(&sg[cur]));
123         sg[i].length += len_cur;
124         sg_clear(&sg[cur]);
125         res = i;
126         goto out;
127 }
128
129 static int sgv_check_tail_clustering(struct scatterlist *sg, int cur, int hint)
130 {
131         int res = -1;
132         unsigned long pfn_cur = page_to_pfn(sg_page(&sg[cur]));
133         int len_cur = sg[cur].length;
134         int prev;
135         unsigned long pfn_prev;
136         bool full_page;
137
138 #ifdef SCST_HIGHMEM
139         if (page >= highmem_start_page) {
140                 TRACE_MEM("%s", "HIGHMEM page allocated, no clustering")
141                 goto out;
142         }
143 #endif
144
145 #if 0
146         TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d",
147                 pfn_cur, pfn_cur_next, len_cur, full_page_cur);
148 #endif
149
150         if (cur == 0)
151                 goto out;
152
153         prev = cur - 1;
154         pfn_prev = page_to_pfn(sg_page(&sg[prev])) +
155                         (sg[prev].length >> PAGE_SHIFT);
156         full_page = (sg[prev].length & (PAGE_SIZE - 1)) == 0;
157
158         if ((pfn_prev == pfn_cur) && full_page) {
159                 TRACE_MEM("SG segment %d will be tail merged with segment %d",
160                         cur, prev);
161                 sg[prev].length += len_cur;
162                 sg_clear(&sg[cur]);
163                 res = prev;
164         }
165
166 out:
167         return res;
168 }
169
170 static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count,
171         void *priv)
172 {
173         int i;
174
175         TRACE_MEM("sg=%p, sg_count=%d", sg, sg_count);
176
177         for (i = 0; i < sg_count; i++) {
178                 struct page *p = sg_page(&sg[i]);
179                 int len = sg[i].length;
180                 int pages =
181                         (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
182
183                 TRACE_MEM("page %lx, len %d, pages %d",
184                         (unsigned long)p, len, pages);
185
186                 while (pages > 0) {
187                         int order = 0;
188
189 /*
190  * __free_pages() doesn't like freeing pages with not that order with
191  * which they were allocated, so disable this small optimization.
192  */
193 #if 0
194                         if (len > 0) {
195                                 while (((1 << order) << PAGE_SHIFT) < len)
196                                         order++;
197                                 len = 0;
198                         }
199 #endif
200                         TRACE_MEM("free_pages(): order %d, page %lx",
201                                 order, (unsigned long)p);
202
203                         __free_pages(p, order);
204
205                         pages -= 1 << order;
206                         p += 1 << order;
207                 }
208         }
209 }
210
211 static struct page *scst_alloc_sys_pages(struct scatterlist *sg,
212         gfp_t gfp_mask, void *priv)
213 {
214         struct page *page = alloc_pages(gfp_mask, 0);
215
216         sg_set_page(sg, page, PAGE_SIZE, 0);
217         TRACE_MEM("page=%p, sg=%p, priv=%p", page, sg, priv);
218         if (page == NULL) {
219                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of "
220                         "sg page failed");
221         }
222         return page;
223 }
224
225 static int scst_alloc_sg_entries(struct scatterlist *sg, int pages,
226         gfp_t gfp_mask, enum sgv_clustering_types clustering_type,
227         struct trans_tbl_ent *trans_tbl,
228         const struct sgv_pool_alloc_fns *alloc_fns, void *priv)
229 {
230         int sg_count = 0;
231         int pg, i, j;
232         int merged = -1;
233
234         TRACE_MEM("pages=%d, clustering_type=%d", pages, clustering_type);
235
236 #if 0
237         gfp_mask |= __GFP_COLD;
238 #endif
239 #ifdef CONFIG_SCST_STRICT_SECURITY
240         gfp_mask |= __GFP_ZERO;
241 #endif
242
243         for (pg = 0; pg < pages; pg++) {
244                 void *rc;
245 #ifdef CONFIG_SCST_DEBUG_OOM
246                 if (((gfp_mask & __GFP_NOFAIL) != __GFP_NOFAIL) &&
247                     ((scst_random() % 10000) == 55))
248                         rc = NULL;
249                 else
250 #endif
251                         rc = alloc_fns->alloc_pages_fn(&sg[sg_count], gfp_mask,
252                                 priv);
253                 if (rc == NULL)
254                         goto out_no_mem;
255
256                 if (clustering_type == sgv_full_clustering)
257                         merged = sgv_check_full_clustering(sg, sg_count, merged);
258                 else if (clustering_type == sgv_tail_clustering)
259                         merged = sgv_check_tail_clustering(sg, sg_count, merged);
260                 else
261                         merged = -1;
262
263                 if (merged == -1)
264                         sg_count++;
265
266                 TRACE_MEM("pg=%d, merged=%d, sg_count=%d", pg, merged,
267                         sg_count);
268         }
269
270         if ((clustering_type != sgv_no_clustering) && (trans_tbl != NULL)) {
271                 pg = 0;
272                 for (i = 0; i < pages; i++) {
273                         int n = (sg[i].length >> PAGE_SHIFT) +
274                                 ((sg[i].length & ~PAGE_MASK) != 0);
275                         trans_tbl[i].pg_count = pg;
276                         for (j = 0; j < n; j++)
277                                 trans_tbl[pg++].sg_num = i+1;
278                         TRACE_MEM("i=%d, n=%d, pg_count=%d", i, n,
279                                 trans_tbl[i].pg_count);
280                 }
281         }
282
283 out:
284         TRACE_MEM("sg_count=%d", sg_count);
285         return sg_count;
286
287 out_no_mem:
288         alloc_fns->free_pages_fn(sg, sg_count, priv);
289         sg_count = 0;
290         goto out;
291 }
292
293 static int sgv_alloc_arrays(struct sgv_pool_obj *obj,
294         int pages_to_alloc, int order, gfp_t gfp_mask)
295 {
296         int sz, tsz = 0;
297         int res = 0;
298
299         TRACE_ENTRY();
300
301         sz = pages_to_alloc * sizeof(obj->sg_entries[0]);
302
303         obj->sg_entries = kmalloc(sz, gfp_mask);
304         if (unlikely(obj->sg_entries == NULL)) {
305                 TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool_obj "
306                         "SG vector failed (size %d)", sz);
307                 res = -ENOMEM;
308                 goto out;
309         }
310
311         sg_init_table(obj->sg_entries, pages_to_alloc);
312
313         if (sgv_pool_clustered(obj->owner_pool)) {
314                 if (order <= sgv_pools_mgr.sgv_max_trans_order) {
315                         obj->trans_tbl =
316                                 (struct trans_tbl_ent *)obj->sg_entries_data;
317                         /*
318                          * No need to clear trans_tbl, if needed, it will be
319                          * fully rewritten in scst_alloc_sg_entries()
320                          */
321                 } else {
322                         tsz = pages_to_alloc * sizeof(obj->trans_tbl[0]);
323                         obj->trans_tbl = kzalloc(tsz, gfp_mask);
324                         if (unlikely(obj->trans_tbl == NULL)) {
325                                 TRACE(TRACE_OUT_OF_MEM, "Allocation of "
326                                         "trans_tbl failed (size %d)", tsz);
327                                 res = -ENOMEM;
328                                 goto out_free;
329                         }
330                 }
331         }
332
333         TRACE_MEM("pages_to_alloc %d, order %d, sz %d, tsz %d, obj %p, "
334                 "sg_entries %p, trans_tbl %p", pages_to_alloc, order,
335                 sz, tsz, obj, obj->sg_entries, obj->trans_tbl);
336
337 out:
338         TRACE_EXIT_RES(res);
339         return res;
340
341 out_free:
342         kfree(obj->sg_entries);
343         obj->sg_entries = NULL;
344         goto out;
345 }
346
347 static void sgv_dtor_and_free(struct sgv_pool_obj *obj)
348 {
349         if (obj->sg_count != 0) {
350                 obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries,
351                         obj->sg_count, obj->allocator_priv);
352         }
353         if (obj->sg_entries != obj->sg_entries_data) {
354                 if (obj->trans_tbl !=
355                     (struct trans_tbl_ent *)obj->sg_entries_data) {
356                         /* kfree() handles NULL parameter */
357                         kfree(obj->trans_tbl);
358                         obj->trans_tbl = NULL;
359                 }
360                 kfree(obj->sg_entries);
361         }
362
363         kmem_cache_free(obj->owner_pool->caches[obj->order_or_pages], obj);
364         return;
365 }
366
367 static struct sgv_pool_obj *sgv_pool_cached_get(struct sgv_pool *pool,
368         int order, gfp_t gfp_mask)
369 {
370         struct sgv_pool_obj *obj;
371         int pages = 1 << order;
372
373         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
374         if (likely(!list_empty(&pool->recycling_lists[order]))) {
375                 obj = list_entry(pool->recycling_lists[order].next,
376                          struct sgv_pool_obj,
377                         recycle_entry.recycling_list_entry);
378
379                 list_del(&obj->recycle_entry.sorted_recycling_list_entry);
380                 list_del(&obj->recycle_entry.recycling_list_entry);
381
382                 sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages;
383                 sgv_pools_mgr.mgr.throttle.active_pages_total += pages;
384
385                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
386
387                 EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
388                 goto out;
389         }
390
391         pool->acc.cached_entries++;
392         pool->acc.cached_pages += pages;
393
394         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
395
396         obj = kmem_cache_alloc(pool->caches[order],
397                 gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA));
398         if (likely(obj)) {
399                 memset(obj, 0, sizeof(*obj));
400                 obj->order_or_pages = order;
401                 obj->owner_pool = pool;
402         } else {
403                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
404                 pool->acc.cached_entries--;
405                 pool->acc.cached_pages -= pages;
406                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
407         }
408
409 out:
410         return obj;
411 }
412
413 static void sgv_pool_cached_put(struct sgv_pool_obj *sgv)
414 {
415         struct sgv_pool *owner = sgv->owner_pool;
416         struct list_head *entry;
417         struct list_head *list = &owner->recycling_lists[sgv->order_or_pages];
418         int sched = 0;
419         int pages = 1 << sgv->order_or_pages;
420
421         EXTRACHECKS_BUG_ON(sgv->order_or_pages < 0);
422
423         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
424
425         TRACE_MEM("sgv %p, order %d, sg_count %d", sgv, sgv->order_or_pages,
426                 sgv->sg_count);
427
428         if (sgv_pool_clustered(owner)) {
429                 /* Make objects with less entries more preferred */
430                 __list_for_each(entry, list) {
431                         struct sgv_pool_obj *tmp = list_entry(entry,
432                                 struct sgv_pool_obj,
433                                 recycle_entry.recycling_list_entry);
434                         TRACE_DBG("tmp %p, order %d, sg_count %d", tmp,
435                                 tmp->order_or_pages, tmp->sg_count);
436                         if (sgv->sg_count <= tmp->sg_count)
437                                 break;
438                 }
439                 entry = entry->prev;
440         } else
441                 entry = list;
442
443         TRACE_DBG("Adding in %p (list %p)", entry, list);
444         list_add(&sgv->recycle_entry.recycling_list_entry, entry);
445
446         list_add_tail(&sgv->recycle_entry.sorted_recycling_list_entry,
447                 &sgv_pools_mgr.mgr.sorted_recycling_list);
448
449         sgv->recycle_entry.time_stamp = jiffies;
450
451         sgv_pools_mgr.mgr.throttle.inactive_pages_total += pages;
452         sgv_pools_mgr.mgr.throttle.active_pages_total -= pages;
453
454         if (!sgv_pools_mgr.mgr.pitbool_running) {
455                 sgv_pools_mgr.mgr.pitbool_running = 1;
456                 sched = 1;
457         }
458
459         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
460
461         if (sched)
462                 schedule_delayed_work(&sgv_pools_mgr.mgr.apit_pool,
463                         PURGE_INTERVAL);
464 }
465
466 /* Must be called under pool_mgr_lock held */
467 static void __sgv_pool_cached_purge(struct sgv_pool_obj *e)
468 {
469         int pages = 1 << e->order_or_pages;
470
471         list_del(&e->recycle_entry.sorted_recycling_list_entry);
472         list_del(&e->recycle_entry.recycling_list_entry);
473         e->owner_pool->acc.cached_entries--;
474         e->owner_pool->acc.cached_pages -= pages;
475         sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages;
476
477         return;
478 }
479
480 /* Must be called under pool_mgr_lock held */
481 static int sgv_pool_cached_purge(struct sgv_pool_obj *e, int t,
482         unsigned long rt)
483 {
484         EXTRACHECKS_BUG_ON(t == 0);
485
486         if (time_after(rt, (e->recycle_entry.time_stamp + t))) {
487                 __sgv_pool_cached_purge(e);
488                 return 0;
489         }
490         return 1;
491 }
492
493 /* Called under pool_mgr_lock held, but drops/reaquires it inside */
494 static int sgv_pool_oom_free_objs(int pgs)
495         __releases(&sgv_pools_mgr.mgr.pool_mgr_lock)
496         __acquires(&sgv_pools_mgr.mgr.pool_mgr_lock)
497 {
498         TRACE_MEM("Shrinking pools about %d pages", pgs);
499         while ((sgv_pools_mgr.mgr.throttle.inactive_pages_total >
500                         sgv_pools_mgr.mgr.throttle.lo_wmk) &&
501               (pgs > 0)) {
502                 struct sgv_pool_obj *e;
503
504                 sBUG_ON(list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list));
505
506                 e = list_entry(sgv_pools_mgr.mgr.sorted_recycling_list.next,
507                                struct sgv_pool_obj,
508                                recycle_entry.sorted_recycling_list_entry);
509
510                 __sgv_pool_cached_purge(e);
511                 pgs -= 1 << e->order_or_pages;
512
513                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
514                 sgv_dtor_and_free(e);
515                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
516         }
517
518         TRACE_MEM("Pages remaining %d ", pgs);
519         return pgs;
520 }
521
522 static int sgv_pool_hiwmk_check(int pages_to_alloc)
523 {
524         int res = 0;
525         int pages = pages_to_alloc;
526
527         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
528
529         pages += sgv_pools_mgr.mgr.throttle.active_pages_total;
530         pages += sgv_pools_mgr.mgr.throttle.inactive_pages_total;
531
532         if (unlikely((u32)pages > sgv_pools_mgr.mgr.throttle.hi_wmk)) {
533                 pages -= sgv_pools_mgr.mgr.throttle.hi_wmk;
534                 sgv_pools_mgr.mgr.throttle.releases_on_hiwmk++;
535
536                 pages = sgv_pool_oom_free_objs(pages);
537                 if (pages > 0) {
538                         TRACE(TRACE_OUT_OF_MEM, "Requested amount of "
539                             "memory (%d pages) for being executed "
540                             "commands together with the already "
541                             "allocated memory exceeds the allowed "
542                             "maximum %d. Should you increase "
543                             "scst_max_cmd_mem?", pages_to_alloc,
544                            sgv_pools_mgr.mgr.throttle.hi_wmk);
545                         sgv_pools_mgr.mgr.throttle.releases_failed++;
546                         res = -ENOMEM;
547                         goto out_unlock;
548                 }
549         }
550
551         sgv_pools_mgr.mgr.throttle.active_pages_total += pages_to_alloc;
552
553 out_unlock:
554         TRACE_MEM("pages_to_alloc %d, new active %d", pages_to_alloc,
555                 sgv_pools_mgr.mgr.throttle.active_pages_total);
556
557         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
558         return res;
559 }
560
561 static void sgv_pool_hiwmk_uncheck(int pages)
562 {
563         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
564         sgv_pools_mgr.mgr.throttle.active_pages_total -= pages;
565         TRACE_MEM("pages %d, new active %d", pages,
566                 sgv_pools_mgr.mgr.throttle.active_pages_total);
567         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
568         return;
569 }
570
571 static bool scst_check_allowed_mem(struct scst_mem_lim *mem_lim, int pages)
572 {
573         int alloced;
574         bool res = true;
575
576         alloced = atomic_add_return(pages, &mem_lim->alloced_pages);
577         if (unlikely(alloced > mem_lim->max_allowed_pages)) {
578                 TRACE(TRACE_OUT_OF_MEM, "Requested amount of memory "
579                         "(%d pages) for being executed commands on a device "
580                         "together with the already allocated memory exceeds "
581                         "the allowed maximum %d. Should you increase "
582                         "scst_max_dev_cmd_mem?", pages,
583                         mem_lim->max_allowed_pages);
584                 atomic_sub(pages, &mem_lim->alloced_pages);
585                 res = false;
586         }
587
588         TRACE_MEM("mem_lim %p, pages %d, res %d, new alloced %d", mem_lim,
589                 pages, res, atomic_read(&mem_lim->alloced_pages));
590
591         return res;
592 }
593
594 static void scst_uncheck_allowed_mem(struct scst_mem_lim *mem_lim, int pages)
595 {
596         atomic_sub(pages, &mem_lim->alloced_pages);
597
598         TRACE_MEM("mem_lim %p, pages %d, new alloced %d", mem_lim,
599                 pages, atomic_read(&mem_lim->alloced_pages));
600         return;
601 }
602
603 struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
604         gfp_t gfp_mask, int flags, int *count,
605         struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv)
606 {
607         struct sgv_pool_obj *obj;
608         int order, pages, cnt;
609         struct scatterlist *res = NULL;
610         int pages_to_alloc;
611         struct kmem_cache *cache;
612         int no_cached = flags & SCST_POOL_ALLOC_NO_CACHED;
613         bool allowed_mem_checked = false, hiwmk_checked = false;
614
615         TRACE_ENTRY();
616
617         if (unlikely(size == 0))
618                 goto out;
619
620         sBUG_ON((gfp_mask & __GFP_NOFAIL) == __GFP_NOFAIL);
621
622         pages = ((size + PAGE_SIZE - 1) >> PAGE_SHIFT);
623         order = get_order(size);
624
625         TRACE_MEM("size=%d, pages=%d, order=%d, flags=%x, *sgv %p", size, pages,
626                 order, flags, *sgv);
627
628         if (*sgv != NULL) {
629                 obj = *sgv;
630                 pages_to_alloc = (1 << order);
631                 cache = pool->caches[obj->order_or_pages];
632
633                 TRACE_MEM("Supplied sgv_obj %p, sgv_order %d", obj,
634                         obj->order_or_pages);
635
636                 EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
637                 EXTRACHECKS_BUG_ON(obj->sg_count != 0);
638
639                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
640                         goto out_fail_free_sg_entries;
641                 allowed_mem_checked = true;
642
643                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
644                         goto out_fail_free_sg_entries;
645                 hiwmk_checked = true;
646         } else if ((order < SGV_POOL_ELEMENTS) && !no_cached) {
647                 pages_to_alloc = (1 << order);
648                 cache = pool->caches[order];
649
650                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
651                         goto out_fail;
652                 allowed_mem_checked = true;
653
654                 obj = sgv_pool_cached_get(pool, order, gfp_mask);
655                 if (unlikely(obj == NULL)) {
656                         TRACE(TRACE_OUT_OF_MEM, "Allocation of "
657                                 "sgv_pool_obj failed (size %d)", size);
658                         goto out_fail;
659                 }
660
661                 if (obj->sg_count != 0) {
662                         TRACE_MEM("Cached sgv_obj %p", obj);
663                         EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
664                         atomic_inc(&pool->cache_acc[order].hit_alloc);
665                         goto success;
666                 }
667
668                 if (flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS) {
669                         if (!(flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL))
670                                 goto out_fail_free;
671                 }
672
673                 TRACE_MEM("Brand new sgv_obj %p", obj);
674
675                 if (order <= sgv_pools_mgr.sgv_max_local_order) {
676                         obj->sg_entries = obj->sg_entries_data;
677                         sg_init_table(obj->sg_entries, pages_to_alloc);
678                         TRACE_MEM("sg_entries %p", obj->sg_entries);
679                         if (sgv_pool_clustered(pool)) {
680                                 obj->trans_tbl = (struct trans_tbl_ent *)
681                                         (obj->sg_entries + pages_to_alloc);
682                                 TRACE_MEM("trans_tbl %p", obj->trans_tbl);
683                                 /*
684                                  * No need to clear trans_tbl, if needed, it
685                                  * will be fully rewritten in
686                                  * scst_alloc_sg_entries(),
687                                  */
688                         }
689                 } else {
690                         if (unlikely(sgv_alloc_arrays(obj, pages_to_alloc,
691                                         order, gfp_mask) != 0))
692                                 goto out_fail_free;
693                 }
694
695                 if ((flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS) &&
696                     (flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL))
697                         goto out_return;
698
699                 obj->allocator_priv = priv;
700
701                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
702                         goto out_fail_free_sg_entries;
703                 hiwmk_checked = true;
704         } else {
705                 int sz;
706
707                 pages_to_alloc = pages;
708
709                 if (unlikely(!scst_check_allowed_mem(mem_lim, pages_to_alloc)))
710                         goto out_fail;
711                 allowed_mem_checked = true;
712
713                 if (flags & SCST_POOL_NO_ALLOC_ON_CACHE_MISS)
714                         goto out_return2;
715
716                 cache = NULL;
717                 sz = sizeof(*obj) + pages*sizeof(obj->sg_entries[0]);
718
719                 obj = kmalloc(sz, gfp_mask);
720                 if (unlikely(obj == NULL)) {
721                         TRACE(TRACE_OUT_OF_MEM, "Allocation of "
722                                 "sgv_pool_obj failed (size %d)", size);
723                         goto out_fail;
724                 }
725                 memset(obj, 0, sizeof(*obj));
726
727                 obj->owner_pool = pool;
728                 obj->order_or_pages = -pages_to_alloc;
729                 obj->allocator_priv = priv;
730
731                 obj->sg_entries = obj->sg_entries_data;
732                 sg_init_table(obj->sg_entries, pages);
733
734                 if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0))
735                         goto out_fail_free_sg_entries;
736                 hiwmk_checked = true;
737
738                 TRACE_MEM("Big or no_cached sgv_obj %p (size %d)", obj, sz);
739         }
740
741         obj->sg_count = scst_alloc_sg_entries(obj->sg_entries,
742                 pages_to_alloc, gfp_mask, pool->clustering_type,
743                 obj->trans_tbl, &pool->alloc_fns, priv);
744         if (unlikely(obj->sg_count <= 0)) {
745                 obj->sg_count = 0;
746                 if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache)
747                         goto out_return1;
748                 else
749                         goto out_fail_free_sg_entries;
750         }
751
752         if (cache) {
753                 atomic_add(pages_to_alloc - obj->sg_count,
754                         &pool->cache_acc[order].merged);
755         } else {
756                 if (no_cached) {
757                         atomic_add(pages_to_alloc,
758                                 &pool->acc.other_pages);
759                         atomic_add(pages_to_alloc - obj->sg_count,
760                                 &pool->acc.other_merged);
761                 } else {
762                         atomic_add(pages_to_alloc,
763                                 &pool->acc.big_pages);
764                         atomic_add(pages_to_alloc - obj->sg_count,
765                                 &pool->acc.big_merged);
766                 }
767         }
768
769 success:
770         if (cache) {
771                 int sg;
772                 atomic_inc(&pool->cache_acc[order].total_alloc);
773                 if (sgv_pool_clustered(pool))
774                         cnt = obj->trans_tbl[pages-1].sg_num;
775                 else
776                         cnt = pages;
777                 sg = cnt-1;
778                 obj->orig_sg = sg;
779                 obj->orig_length = obj->sg_entries[sg].length;
780                 if (sgv_pool_clustered(pool)) {
781                         obj->sg_entries[sg].length =
782                                 (pages - obj->trans_tbl[sg].pg_count) << PAGE_SHIFT;
783                 }
784         } else {
785                 cnt = obj->sg_count;
786                 if (no_cached)
787                         atomic_inc(&pool->acc.other_alloc);
788                 else
789                         atomic_inc(&pool->acc.big_alloc);
790         }
791
792         *count = cnt;
793         res = obj->sg_entries;
794         *sgv = obj;
795
796         if (size & ~PAGE_MASK)
797                 obj->sg_entries[cnt-1].length -=
798                         PAGE_SIZE - (size & ~PAGE_MASK);
799
800         TRACE_MEM("sgv_obj=%p, sg_entries %p (size=%d, pages=%d, sg_count=%d, "
801                 "count=%d, last_len=%d)", obj, obj->sg_entries, size, pages,
802                 obj->sg_count, *count, obj->sg_entries[obj->orig_sg].length);
803
804 out:
805         TRACE_EXIT_HRES(res);
806         return res;
807
808 out_return:
809         obj->allocator_priv = priv;
810         obj->owner_pool = pool;
811
812 out_return1:
813         *sgv = obj;
814         TRACE_MEM("Returning failed sgv_obj %p (count %d)", obj, *count);
815
816 out_return2:
817         *count = pages_to_alloc;
818         res = NULL;
819         goto out_uncheck;
820
821 out_fail_free_sg_entries:
822         if (obj->sg_entries != obj->sg_entries_data) {
823                 if (obj->trans_tbl !=
824                         (struct trans_tbl_ent *)obj->sg_entries_data) {
825                         /* kfree() handles NULL parameter */
826                         kfree(obj->trans_tbl);
827                         obj->trans_tbl = NULL;
828                 }
829                 kfree(obj->sg_entries);
830                 obj->sg_entries = NULL;
831         }
832
833 out_fail_free:
834         if (cache)
835                 sgv_pool_cached_put(obj);
836         else
837                 kfree(obj);
838
839 out_fail:
840         res = NULL;
841         *count = 0;
842         *sgv = NULL;
843         TRACE_MEM("%s", "Allocation failed");
844
845 out_uncheck:
846         if (hiwmk_checked)
847                 sgv_pool_hiwmk_uncheck(pages_to_alloc);
848         if (allowed_mem_checked)
849                 scst_uncheck_allowed_mem(mem_lim, pages_to_alloc);
850         goto out;
851 }
852 EXPORT_SYMBOL(sgv_pool_alloc);
853
854 void *sgv_get_priv(struct sgv_pool_obj *sgv)
855 {
856         return sgv->allocator_priv;
857 }
858 EXPORT_SYMBOL(sgv_get_priv);
859
860 void sgv_pool_free(struct sgv_pool_obj *sgv, struct scst_mem_lim *mem_lim)
861 {
862         int pages;
863
864         TRACE_MEM("Freeing sgv_obj %p, order %d, sg_entries %p, "
865                 "sg_count %d, allocator_priv %p", sgv, sgv->order_or_pages,
866                 sgv->sg_entries, sgv->sg_count, sgv->allocator_priv);
867
868         if (sgv->order_or_pages >= 0) {
869                 sgv->sg_entries[sgv->orig_sg].length = sgv->orig_length;
870                 pages = (sgv->sg_count != 0) ? 1 << sgv->order_or_pages : 0;
871                 sgv_pool_cached_put(sgv);
872         } else {
873                 sgv->owner_pool->alloc_fns.free_pages_fn(sgv->sg_entries,
874                         sgv->sg_count, sgv->allocator_priv);
875                 pages = (sgv->sg_count != 0) ? -sgv->order_or_pages : 0;
876                 kfree(sgv);
877                 sgv_pool_hiwmk_uncheck(pages);
878         }
879
880         scst_uncheck_allowed_mem(mem_lim, pages);
881
882         return;
883 }
884 EXPORT_SYMBOL(sgv_pool_free);
885
886 struct scatterlist *scst_alloc(int size, gfp_t gfp_mask, int *count)
887 {
888         struct scatterlist *res;
889         int pages = (size >> PAGE_SHIFT) + ((size & ~PAGE_MASK) != 0);
890         struct sgv_pool_alloc_fns sys_alloc_fns = {
891                 scst_alloc_sys_pages, scst_free_sys_sg_entries };
892         int no_fail = ((gfp_mask & __GFP_NOFAIL) == __GFP_NOFAIL);
893
894         TRACE_ENTRY();
895
896         atomic_inc(&sgv_pools_mgr.sgv_other_total_alloc);
897
898         if (unlikely(sgv_pool_hiwmk_check(pages) != 0)) {
899                 if (!no_fail) {
900                         res = NULL;
901                         goto out;
902                 } else {
903                         /*
904                          * Update active_pages_total since alloc can't fail.
905                          * If it wasn't updated then the counter would cross 0
906                          * on free again.
907                          */
908                         sgv_pool_hiwmk_uncheck(-pages);
909                  }
910         }
911
912         res = kmalloc(pages*sizeof(*res), gfp_mask);
913         if (res == NULL) {
914                 TRACE(TRACE_OUT_OF_MEM, "Unable to allocate sg for %d pages",
915                         pages);
916                 goto out_uncheck;
917         }
918
919         sg_init_table(res, pages);
920
921         /*
922          * If we allow use clustering here, we will have troubles in
923          * scst_free() to figure out how many pages are in the SG vector.
924          * So, always don't use clustering.
925          */
926         *count = scst_alloc_sg_entries(res, pages, gfp_mask, sgv_no_clustering,
927                         NULL, &sys_alloc_fns, NULL);
928         if (*count <= 0)
929                 goto out_free;
930
931 out:
932         TRACE_MEM("Alloced sg %p (count %d) \"no fail\" %d", res, *count, no_fail);
933
934         TRACE_EXIT_HRES(res);
935         return res;
936
937 out_free:
938         kfree(res);
939         res = NULL;
940
941 out_uncheck:
942         if (!no_fail)
943                 sgv_pool_hiwmk_uncheck(pages);
944         goto out;
945 }
946 EXPORT_SYMBOL(scst_alloc);
947
948 void scst_free(struct scatterlist *sg, int count)
949 {
950         TRACE_MEM("Freeing sg=%p", sg);
951
952         sgv_pool_hiwmk_uncheck(count);
953
954         scst_free_sys_sg_entries(sg, count, NULL);
955         kfree(sg);
956         return;
957 }
958 EXPORT_SYMBOL(scst_free);
959
960 static void sgv_pool_cached_init(struct sgv_pool *pool)
961 {
962         int i;
963         for (i = 0; i < SGV_POOL_ELEMENTS; i++)
964                 INIT_LIST_HEAD(&pool->recycling_lists[i]);
965 }
966
967 int sgv_pool_init(struct sgv_pool *pool, const char *name,
968         enum sgv_clustering_types clustering_type)
969 {
970         int res = -ENOMEM;
971         int i;
972         struct sgv_pool_obj *obj;
973
974         TRACE_ENTRY();
975
976         memset(pool, 0, sizeof(*pool));
977
978         atomic_set(&pool->acc.other_alloc, 0);
979         atomic_set(&pool->acc.big_alloc, 0);
980         atomic_set(&pool->acc.other_pages, 0);
981         atomic_set(&pool->acc.big_pages, 0);
982         atomic_set(&pool->acc.other_merged, 0);
983         atomic_set(&pool->acc.big_merged, 0);
984
985         pool->clustering_type = clustering_type;
986         pool->alloc_fns.alloc_pages_fn = scst_alloc_sys_pages;
987         pool->alloc_fns.free_pages_fn = scst_free_sys_sg_entries;
988
989         TRACE_MEM("name %s, sizeof(*obj)=%zd, clustering_type=%d", name,
990                 sizeof(*obj), clustering_type);
991
992         strncpy(pool->name, name, sizeof(pool->name)-1);
993         pool->name[sizeof(pool->name)-1] = '\0';
994
995         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
996                 int size;
997
998                 atomic_set(&pool->cache_acc[i].total_alloc, 0);
999                 atomic_set(&pool->cache_acc[i].hit_alloc, 0);
1000                 atomic_set(&pool->cache_acc[i].merged, 0);
1001
1002                 if (i <= sgv_pools_mgr.sgv_max_local_order) {
1003                         size = sizeof(*obj) + (1 << i) *
1004                                 (sizeof(obj->sg_entries[0]) +
1005                                  ((clustering_type != sgv_no_clustering) ?
1006                                         sizeof(obj->trans_tbl[0]) : 0));
1007                 } else if (i <= sgv_pools_mgr.sgv_max_trans_order) {
1008                         /*
1009                          * sgv ie sg_entries is allocated outside object, but
1010                          * ttbl is still embedded.
1011                          */
1012                         size = sizeof(*obj) + (1 << i) *
1013                                 (((clustering_type != sgv_no_clustering) ?
1014                                         sizeof(obj->trans_tbl[0]) : 0));
1015                 } else {
1016                         size = sizeof(*obj);
1017                         /* both sgv and ttbl are kallocated() */
1018                 }
1019
1020                 TRACE_MEM("pages=%d, size=%d", 1 << i, size);
1021
1022                 scnprintf(pool->cache_names[i], sizeof(pool->cache_names[i]),
1023                         "%s-%luK", name, (PAGE_SIZE >> 10) << i);
1024                 pool->caches[i] = kmem_cache_create(pool->cache_names[i],
1025                         size, 0, SCST_SLAB_FLAGS, NULL
1026 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
1027                         , NULL);
1028 #else
1029                         );
1030 #endif
1031                 if (pool->caches[i] == NULL) {
1032                         TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool cache "
1033                                 "%s(%d) failed", name, i);
1034                         goto out_free;
1035                 }
1036         }
1037
1038         sgv_pool_cached_init(pool);
1039
1040         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1041         list_add_tail(&pool->sgv_pool_list_entry,
1042                 &sgv_pools_mgr.scst_sgv_pool_list);
1043         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1044
1045         res = 0;
1046
1047 out:
1048         TRACE_EXIT_RES(res);
1049         return res;
1050
1051 out_free:
1052         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1053                 if (pool->caches[i]) {
1054                         kmem_cache_destroy(pool->caches[i]);
1055                         pool->caches[i] = NULL;
1056                 } else
1057                         break;
1058         }
1059         goto out;
1060 }
1061
1062 static void sgv_pool_evaluate_local_order(struct scst_sgv_pools_manager *pmgr)
1063 {
1064         int space4sgv_ttbl = PAGE_SIZE - sizeof(struct sgv_pool_obj);
1065
1066         pmgr->sgv_max_local_order = get_order(
1067                 (((space4sgv_ttbl /
1068                   (sizeof(struct trans_tbl_ent) + sizeof(struct scatterlist))) *
1069                         PAGE_SIZE) & PAGE_MASK)) - 1;
1070
1071         pmgr->sgv_max_trans_order = get_order(
1072                 (((space4sgv_ttbl / sizeof(struct trans_tbl_ent)) * PAGE_SIZE)
1073                  & PAGE_MASK)) - 1;
1074
1075         TRACE_MEM("sgv_max_local_order %d, sgv_max_trans_order %d",
1076                 pmgr->sgv_max_local_order, pmgr->sgv_max_trans_order);
1077         TRACE_MEM("max object size with embedded sgv & ttbl %zd",
1078                 (1 << pmgr->sgv_max_local_order) *
1079                 (sizeof(struct trans_tbl_ent) + sizeof(struct scatterlist))
1080                 + sizeof(struct sgv_pool_obj));
1081         TRACE_MEM("max object size with embedded sgv (!clustered) %zd",
1082                 (1 << pmgr->sgv_max_local_order) *
1083                 (sizeof(struct scatterlist))
1084                 + sizeof(struct sgv_pool_obj));
1085         TRACE_MEM("max object size with embedded ttbl %zd",
1086                 (1 << pmgr->sgv_max_trans_order) * sizeof(struct trans_tbl_ent)
1087                 + sizeof(struct sgv_pool_obj));
1088 }
1089
1090 void sgv_pool_flush(struct sgv_pool *pool)
1091 {
1092         int i;
1093
1094         TRACE_ENTRY();
1095
1096         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1097                 struct sgv_pool_obj *e;
1098
1099                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1100                 while (!list_empty(&pool->recycling_lists[i])) {
1101                         e = list_entry(pool->recycling_lists[i].next,
1102                                 struct sgv_pool_obj,
1103                                 recycle_entry.recycling_list_entry);
1104
1105                         __sgv_pool_cached_purge(e);
1106                         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1107
1108                         EXTRACHECKS_BUG_ON(e->owner_pool != pool);
1109                         sgv_dtor_and_free(e);
1110
1111                         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1112                 }
1113                 spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1114         }
1115
1116         TRACE_EXIT();
1117         return;
1118 }
1119 EXPORT_SYMBOL(sgv_pool_flush);
1120
1121 void sgv_pool_deinit(struct sgv_pool *pool)
1122 {
1123         int i;
1124
1125         TRACE_ENTRY();
1126
1127         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1128         list_del(&pool->sgv_pool_list_entry);
1129         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1130
1131         sgv_pool_flush(pool);
1132
1133         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1134                 if (pool->caches[i])
1135                         kmem_cache_destroy(pool->caches[i]);
1136                 pool->caches[i] = NULL;
1137         }
1138
1139         TRACE_EXIT();
1140         return;
1141 }
1142
1143 void sgv_pool_set_allocator(struct sgv_pool *pool,
1144         struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *),
1145         void (*free_pages_fn)(struct scatterlist *, int, void *))
1146 {
1147         pool->alloc_fns.alloc_pages_fn = alloc_pages_fn;
1148         pool->alloc_fns.free_pages_fn = free_pages_fn;
1149         return;
1150 }
1151 EXPORT_SYMBOL(sgv_pool_set_allocator);
1152
1153 struct sgv_pool *sgv_pool_create(const char *name,
1154         enum sgv_clustering_types clustering_type)
1155 {
1156         struct sgv_pool *pool;
1157         int rc;
1158
1159         TRACE_ENTRY();
1160
1161         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
1162         if (pool == NULL) {
1163                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of sgv_pool failed");
1164                 goto out;
1165         }
1166
1167         rc = sgv_pool_init(pool, name, clustering_type);
1168         if (rc != 0)
1169                 goto out_free;
1170
1171 out:
1172         TRACE_EXIT_RES(pool != NULL);
1173         return pool;
1174
1175 out_free:
1176         kfree(pool);
1177         pool = NULL;
1178         goto out;
1179 }
1180 EXPORT_SYMBOL(sgv_pool_create);
1181
1182 void sgv_pool_destroy(struct sgv_pool *pool)
1183 {
1184         TRACE_ENTRY();
1185
1186         sgv_pool_deinit(pool);
1187         kfree(pool);
1188
1189         TRACE_EXIT();
1190 }
1191 EXPORT_SYMBOL(sgv_pool_destroy);
1192
1193 static int sgv_pool_cached_shrinker(int nr, gfp_t gfpm)
1194 {
1195         TRACE_ENTRY();
1196
1197         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1198
1199         if (nr > 0) {
1200                 struct sgv_pool_obj *e;
1201                 unsigned long rt = jiffies;
1202
1203                 while (!list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list)) {
1204                         e = list_entry(
1205                                 sgv_pools_mgr.mgr.sorted_recycling_list.next,
1206                                 struct sgv_pool_obj,
1207                                 recycle_entry.sorted_recycling_list_entry);
1208
1209                         if (sgv_pool_cached_purge(e, SHRINK_TIME_AFTER, rt) == 0) {
1210                                 nr -= 1 << e->order_or_pages;
1211                                 spin_unlock_bh(
1212                                         &sgv_pools_mgr.mgr.pool_mgr_lock);
1213                                 sgv_dtor_and_free(e);
1214                                 spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1215                         } else
1216                                 break;
1217
1218                         if (nr <= 0)
1219                                 break;
1220                 }
1221         }
1222
1223         nr = sgv_pools_mgr.mgr.throttle.inactive_pages_total;
1224
1225         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1226
1227         TRACE_EXIT();
1228         return nr;
1229 }
1230
1231 static void sgv_pool_cached_pitbool(void *p)
1232 {
1233         u32 total_pages;
1234         struct sgv_pool_obj *e;
1235         unsigned long rt = jiffies;
1236
1237         TRACE_ENTRY();
1238
1239         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1240
1241         sgv_pools_mgr.mgr.pitbool_running = 0;
1242
1243         while (!list_empty(&sgv_pools_mgr.mgr.sorted_recycling_list)) {
1244                 e = list_entry(sgv_pools_mgr.mgr.sorted_recycling_list.next,
1245                         struct sgv_pool_obj,
1246                         recycle_entry.sorted_recycling_list_entry);
1247
1248                 if (sgv_pool_cached_purge(e, PURGE_TIME_AFTER, rt) == 0) {
1249                         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1250                         sgv_dtor_and_free(e);
1251                         spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1252                 } else
1253                         break;
1254         }
1255
1256         total_pages = sgv_pools_mgr.mgr.throttle.inactive_pages_total;
1257
1258         spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
1259
1260         if (total_pages) {
1261                 schedule_delayed_work(&sgv_pools_mgr.mgr.apit_pool,
1262                         PURGE_INTERVAL);
1263         }
1264
1265         TRACE_EXIT();
1266         return;
1267 }
1268
1269 /* Both parameters in pages */
1270 int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
1271 {
1272         int res;
1273         struct scst_sgv_pools_manager *pools = &sgv_pools_mgr;
1274
1275         TRACE_ENTRY();
1276
1277         memset(pools, 0, sizeof(*pools));
1278
1279         sgv_pools_mgr.mgr.throttle.hi_wmk = mem_hwmark;
1280         sgv_pools_mgr.mgr.throttle.lo_wmk = mem_lwmark;
1281
1282         sgv_pool_evaluate_local_order(&sgv_pools_mgr);
1283
1284         atomic_set(&pools->sgv_other_total_alloc, 0);
1285         INIT_LIST_HEAD(&pools->scst_sgv_pool_list);
1286         mutex_init(&pools->scst_sgv_pool_mutex);
1287
1288         INIT_LIST_HEAD(&pools->mgr.sorted_recycling_list);
1289         spin_lock_init(&pools->mgr.pool_mgr_lock);
1290
1291         res = sgv_pool_init(&pools->default_set.norm, "sgv", sgv_no_clustering);
1292         if (res != 0)
1293                 goto out;
1294
1295         res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust",
1296                 sgv_full_clustering);
1297         if (res != 0)
1298                 goto out_free_clust;
1299
1300         res = sgv_pool_init(&pools->default_set.dma, "sgv-dma",
1301                 sgv_no_clustering);
1302         if (res != 0)
1303                 goto out_free_norm;
1304
1305 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20))
1306         INIT_DELAYED_WORK(&pools->mgr.apit_pool,
1307                 (void (*)(struct work_struct *))sgv_pool_cached_pitbool);
1308 #else
1309         INIT_WORK(&pools->mgr.apit_pool, sgv_pool_cached_pitbool, NULL);
1310 #endif
1311
1312 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
1313         pools->mgr.sgv_shrinker = set_shrinker(DEFAULT_SEEKS,
1314                 sgv_pool_cached_shrinker);
1315 #else
1316         pools->mgr.sgv_shrinker.shrink = sgv_pool_cached_shrinker;
1317         pools->mgr.sgv_shrinker.seeks = DEFAULT_SEEKS;
1318         register_shrinker(&pools->mgr.sgv_shrinker);
1319 #endif
1320
1321 out:
1322         TRACE_EXIT_RES(res);
1323         return res;
1324
1325 out_free_norm:
1326         sgv_pool_deinit(&pools->default_set.norm);
1327
1328 out_free_clust:
1329         sgv_pool_deinit(&pools->default_set.norm_clust);
1330         goto out;
1331 }
1332
1333 void scst_sgv_pools_deinit(void)
1334 {
1335         struct scst_sgv_pools_manager *pools = &sgv_pools_mgr;
1336
1337         TRACE_ENTRY();
1338
1339 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
1340         remove_shrinker(pools->mgr.sgv_shrinker);
1341 #else
1342         unregister_shrinker(&pools->mgr.sgv_shrinker);
1343 #endif
1344
1345         cancel_delayed_work(&pools->mgr.apit_pool);
1346
1347         sgv_pool_deinit(&pools->default_set.dma);
1348         sgv_pool_deinit(&pools->default_set.norm);
1349         sgv_pool_deinit(&pools->default_set.norm_clust);
1350
1351         flush_scheduled_work();
1352
1353         TRACE_EXIT();
1354         return;
1355 }
1356
1357 static void scst_do_sgv_read(struct seq_file *seq, const struct sgv_pool *pool)
1358 {
1359         int i, total = 0, hit = 0, merged = 0, allocated = 0;
1360         int oa, om;
1361
1362         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1363                 int t;
1364
1365                 hit += atomic_read(&pool->cache_acc[i].hit_alloc);
1366                 total += atomic_read(&pool->cache_acc[i].total_alloc);
1367
1368                 t = atomic_read(&pool->cache_acc[i].total_alloc) -
1369                         atomic_read(&pool->cache_acc[i].hit_alloc);
1370                 allocated += t * (1 << i);
1371                 merged += atomic_read(&pool->cache_acc[i].merged);
1372         }
1373
1374         seq_printf(seq, "\n%-30s %-11d %-11d %-11d %d/%d (P/O)\n", pool->name,
1375                 hit, total, (allocated != 0) ? merged*100/allocated : 0,
1376                 pool->acc.cached_pages, pool->acc.cached_entries);
1377
1378         for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
1379                 int t = atomic_read(&pool->cache_acc[i].total_alloc) -
1380                         atomic_read(&pool->cache_acc[i].hit_alloc);
1381                 allocated = t * (1 << i);
1382                 merged = atomic_read(&pool->cache_acc[i].merged);
1383
1384                 seq_printf(seq, "  %-28s %-11d %-11d %d\n",
1385                         pool->cache_names[i],
1386                         atomic_read(&pool->cache_acc[i].hit_alloc),
1387                         atomic_read(&pool->cache_acc[i].total_alloc),
1388                         (allocated != 0) ? merged*100/allocated : 0);
1389         }
1390
1391         allocated = atomic_read(&pool->acc.big_pages);
1392         merged = atomic_read(&pool->acc.big_merged);
1393         oa = atomic_read(&pool->acc.other_pages);
1394         om = atomic_read(&pool->acc.other_merged);
1395
1396         seq_printf(seq, "  %-40s %d/%-9d %d/%d\n", "big/other",
1397                    atomic_read(&pool->acc.big_alloc),
1398                    atomic_read(&pool->acc.other_alloc),
1399                    (allocated != 0) ? merged*100/allocated : 0,
1400                    (oa != 0) ? om/oa : 0);
1401
1402         return;
1403 }
1404
1405 int sgv_pool_procinfo_show(struct seq_file *seq, void *v)
1406 {
1407         struct sgv_pool *pool;
1408
1409         TRACE_ENTRY();
1410
1411         seq_printf(seq, "%-42s %d/%d\n%-42s %d/%d\n%-42s %d/%d\n\n",
1412                 "Inactive/active pages",
1413                 sgv_pools_mgr.mgr.throttle.inactive_pages_total,
1414                 sgv_pools_mgr.mgr.throttle.active_pages_total,
1415                 "Hi/lo watermarks [pages]", sgv_pools_mgr.mgr.throttle.hi_wmk,
1416                 sgv_pools_mgr.mgr.throttle.lo_wmk,
1417                 "Hi watermark releases/failures",
1418                 sgv_pools_mgr.mgr.throttle.releases_on_hiwmk,
1419                 sgv_pools_mgr.mgr.throttle.releases_failed);
1420
1421         seq_printf(seq, "%-30s %-11s %-11s %-11s %-11s", "Name", "Hit", "Total",
1422                 "% merged", "Cached");
1423
1424         mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1425         list_for_each_entry(pool, &sgv_pools_mgr.scst_sgv_pool_list,
1426                         sgv_pool_list_entry) {
1427                 scst_do_sgv_read(seq, pool);
1428         }
1429         mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
1430
1431         seq_printf(seq, "\n%-42s %-11d\n", "other",
1432                 atomic_read(&sgv_pools_mgr.sgv_other_total_alloc));
1433
1434         TRACE_EXIT();
1435         return 0;
1436 }