]>
Commit | Line | Data |
---|---|---|
1403b1a3 PN |
1 | /* |
2 | * Copyright (c) Red Hat Inc. | |
3 | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sub license, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the | |
12 | * next paragraph) shall be included in all copies or substantial portions | |
13 | * of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | |
21 | * DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: Dave Airlie <airlied@redhat.com> | |
24 | * Jerome Glisse <jglisse@redhat.com> | |
25 | * Pauli Nieminen <suokkos@gmail.com> | |
26 | */ | |
27 | ||
28 | /* simple list based uncached page pool | |
29 | * - Pool collects resently freed pages for reuse | |
30 | * - Use page->lru to keep a free list | |
31 | * - doesn't track currently in use pages | |
32 | */ | |
33 | #include <linux/list.h> | |
34 | #include <linux/spinlock.h> | |
35 | #include <linux/highmem.h> | |
36 | #include <linux/mm_types.h> | |
07458661 | 37 | #include <linux/module.h> |
1403b1a3 PN |
38 | #include <linux/mm.h> |
39 | ||
40 | #include <asm/atomic.h> | |
41 | #include <asm/agp.h> | |
42 | ||
43 | #include "ttm/ttm_bo_driver.h" | |
44 | #include "ttm/ttm_page_alloc.h" | |
45 | ||
46 | ||
47 | #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) | |
48 | #define SMALL_ALLOCATION 16 | |
49 | #define FREE_ALL_PAGES (~0U) | |
50 | /* times are in msecs */ | |
51 | #define PAGE_FREE_INTERVAL 1000 | |
52 | ||
53 | /** | |
54 | * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. | |
55 | * | |
56 | * @lock: Protects the shared pool from concurrnet access. Must be used with | |
57 | * irqsave/irqrestore variants because pool allocator maybe called from | |
58 | * delayed work. | |
59 | * @fill_lock: Prevent concurrent calls to fill. | |
60 | * @list: Pool of free uc/wc pages for fast reuse. | |
61 | * @gfp_flags: Flags to pass for alloc_page. | |
62 | * @npages: Number of pages in pool. | |
63 | */ | |
64 | struct ttm_page_pool { | |
65 | spinlock_t lock; | |
66 | bool fill_lock; | |
67 | struct list_head list; | |
68 | int gfp_flags; | |
69 | unsigned npages; | |
07458661 PN |
70 | char *name; |
71 | unsigned long nfrees; | |
72 | unsigned long nrefills; | |
1403b1a3 PN |
73 | }; |
74 | ||
75 | struct ttm_pool_opts { | |
76 | unsigned alloc_size; | |
77 | unsigned max_size; | |
78 | unsigned small; | |
79 | }; | |
80 | ||
81 | #define NUM_POOLS 4 | |
82 | ||
83 | /** | |
84 | * struct ttm_pool_manager - Holds memory pools for fst allocation | |
85 | * | |
86 | * Manager is read only object for pool code so it doesn't need locking. | |
87 | * | |
88 | * @free_interval: minimum number of jiffies between freeing pages from pool. | |
89 | * @page_alloc_inited: reference counting for pool allocation. | |
90 | * @work: Work that is used to shrink the pool. Work is only run when there is | |
91 | * some pages to free. | |
92 | * @small_allocation: Limit in number of pages what is small allocation. | |
93 | * | |
94 | * @pools: All pool objects in use. | |
95 | **/ | |
96 | struct ttm_pool_manager { | |
97 | struct shrinker mm_shrink; | |
98 | atomic_t page_alloc_inited; | |
99 | struct ttm_pool_opts options; | |
100 | ||
101 | union { | |
102 | struct ttm_page_pool pools[NUM_POOLS]; | |
103 | struct { | |
104 | struct ttm_page_pool wc_pool; | |
105 | struct ttm_page_pool uc_pool; | |
106 | struct ttm_page_pool wc_pool_dma32; | |
107 | struct ttm_page_pool uc_pool_dma32; | |
108 | } ; | |
109 | }; | |
110 | }; | |
111 | ||
112 | static struct ttm_pool_manager _manager = { | |
113 | .page_alloc_inited = ATOMIC_INIT(0) | |
114 | }; | |
115 | ||
975efdb1 | 116 | #ifndef CONFIG_X86 |
1403b1a3 PN |
117 | static int set_pages_array_wb(struct page **pages, int addrinarray) |
118 | { | |
119 | #ifdef TTM_HAS_AGP | |
120 | int i; | |
121 | ||
122 | for (i = 0; i < addrinarray; i++) | |
123 | unmap_page_from_agp(pages[i]); | |
124 | #endif | |
125 | return 0; | |
126 | } | |
127 | ||
128 | static int set_pages_array_wc(struct page **pages, int addrinarray) | |
129 | { | |
130 | #ifdef TTM_HAS_AGP | |
131 | int i; | |
132 | ||
133 | for (i = 0; i < addrinarray; i++) | |
134 | map_page_into_agp(pages[i]); | |
135 | #endif | |
136 | return 0; | |
137 | } | |
138 | ||
139 | static int set_pages_array_uc(struct page **pages, int addrinarray) | |
140 | { | |
141 | #ifdef TTM_HAS_AGP | |
142 | int i; | |
143 | ||
144 | for (i = 0; i < addrinarray; i++) | |
145 | map_page_into_agp(pages[i]); | |
146 | #endif | |
147 | return 0; | |
148 | } | |
149 | #endif | |
150 | ||
151 | /** | |
152 | * Select the right pool or requested caching state and ttm flags. */ | |
153 | static struct ttm_page_pool *ttm_get_pool(int flags, | |
154 | enum ttm_caching_state cstate) | |
155 | { | |
156 | int pool_index; | |
157 | ||
158 | if (cstate == tt_cached) | |
159 | return NULL; | |
160 | ||
161 | if (cstate == tt_wc) | |
162 | pool_index = 0x0; | |
163 | else | |
164 | pool_index = 0x1; | |
165 | ||
166 | if (flags & TTM_PAGE_FLAG_DMA32) | |
167 | pool_index |= 0x2; | |
168 | ||
169 | return &_manager.pools[pool_index]; | |
170 | } | |
171 | ||
172 | /* set memory back to wb and free the pages. */ | |
173 | static void ttm_pages_put(struct page *pages[], unsigned npages) | |
174 | { | |
175 | unsigned i; | |
176 | if (set_pages_array_wb(pages, npages)) | |
177 | printk(KERN_ERR "[ttm] Failed to set %d pages to wb!\n", | |
178 | npages); | |
179 | for (i = 0; i < npages; ++i) | |
180 | __free_page(pages[i]); | |
181 | } | |
182 | ||
183 | static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, | |
184 | unsigned freed_pages) | |
185 | { | |
186 | pool->npages -= freed_pages; | |
07458661 | 187 | pool->nfrees += freed_pages; |
1403b1a3 PN |
188 | } |
189 | ||
190 | /** | |
191 | * Free pages from pool. | |
192 | * | |
193 | * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC | |
194 | * number of pages in one go. | |
195 | * | |
196 | * @pool: to free the pages from | |
197 | * @free_all: If set to true will free all pages in pool | |
198 | **/ | |
199 | static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) | |
200 | { | |
201 | unsigned long irq_flags; | |
202 | struct page *p; | |
203 | struct page **pages_to_free; | |
204 | unsigned freed_pages = 0, | |
205 | npages_to_free = nr_free; | |
206 | ||
207 | if (NUM_PAGES_TO_ALLOC < nr_free) | |
208 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
209 | ||
210 | pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), | |
211 | GFP_KERNEL); | |
212 | if (!pages_to_free) { | |
213 | printk(KERN_ERR "Failed to allocate memory for pool free operation.\n"); | |
214 | return 0; | |
215 | } | |
216 | ||
217 | restart: | |
218 | spin_lock_irqsave(&pool->lock, irq_flags); | |
219 | ||
220 | list_for_each_entry_reverse(p, &pool->list, lru) { | |
221 | if (freed_pages >= npages_to_free) | |
222 | break; | |
223 | ||
224 | pages_to_free[freed_pages++] = p; | |
225 | /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ | |
226 | if (freed_pages >= NUM_PAGES_TO_ALLOC) { | |
227 | /* remove range of pages from the pool */ | |
228 | __list_del(p->lru.prev, &pool->list); | |
229 | ||
230 | ttm_pool_update_free_locked(pool, freed_pages); | |
231 | /** | |
232 | * Because changing page caching is costly | |
233 | * we unlock the pool to prevent stalling. | |
234 | */ | |
235 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
236 | ||
237 | ttm_pages_put(pages_to_free, freed_pages); | |
238 | if (likely(nr_free != FREE_ALL_PAGES)) | |
239 | nr_free -= freed_pages; | |
240 | ||
241 | if (NUM_PAGES_TO_ALLOC >= nr_free) | |
242 | npages_to_free = nr_free; | |
243 | else | |
244 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
245 | ||
246 | freed_pages = 0; | |
247 | ||
248 | /* free all so restart the processing */ | |
249 | if (nr_free) | |
250 | goto restart; | |
251 | ||
252 | /* Not allowed to fall tough or break because | |
253 | * following context is inside spinlock while we are | |
254 | * outside here. | |
255 | */ | |
256 | goto out; | |
257 | ||
258 | } | |
259 | } | |
260 | ||
1403b1a3 PN |
261 | /* remove range of pages from the pool */ |
262 | if (freed_pages) { | |
263 | __list_del(&p->lru, &pool->list); | |
264 | ||
265 | ttm_pool_update_free_locked(pool, freed_pages); | |
266 | nr_free -= freed_pages; | |
267 | } | |
268 | ||
269 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
270 | ||
271 | if (freed_pages) | |
272 | ttm_pages_put(pages_to_free, freed_pages); | |
273 | out: | |
274 | kfree(pages_to_free); | |
275 | return nr_free; | |
276 | } | |
277 | ||
278 | /* Get good estimation how many pages are free in pools */ | |
279 | static int ttm_pool_get_num_unused_pages(void) | |
280 | { | |
281 | unsigned i; | |
282 | int total = 0; | |
283 | for (i = 0; i < NUM_POOLS; ++i) | |
284 | total += _manager.pools[i].npages; | |
285 | ||
286 | return total; | |
287 | } | |
288 | ||
289 | /** | |
290 | * Calback for mm to request pool to reduce number of page held. | |
291 | */ | |
292 | static int ttm_pool_mm_shrink(int shrink_pages, gfp_t gfp_mask) | |
293 | { | |
294 | static atomic_t start_pool = ATOMIC_INIT(0); | |
295 | unsigned i; | |
296 | unsigned pool_offset = atomic_add_return(1, &start_pool); | |
297 | struct ttm_page_pool *pool; | |
298 | ||
299 | pool_offset = pool_offset % NUM_POOLS; | |
300 | /* select start pool in round robin fashion */ | |
301 | for (i = 0; i < NUM_POOLS; ++i) { | |
302 | unsigned nr_free = shrink_pages; | |
303 | if (shrink_pages == 0) | |
304 | break; | |
305 | pool = &_manager.pools[(i + pool_offset)%NUM_POOLS]; | |
306 | shrink_pages = ttm_page_pool_free(pool, nr_free); | |
307 | } | |
308 | /* return estimated number of unused pages in pool */ | |
309 | return ttm_pool_get_num_unused_pages(); | |
310 | } | |
311 | ||
312 | static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) | |
313 | { | |
314 | manager->mm_shrink.shrink = &ttm_pool_mm_shrink; | |
315 | manager->mm_shrink.seeks = 1; | |
316 | register_shrinker(&manager->mm_shrink); | |
317 | } | |
318 | ||
319 | static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) | |
320 | { | |
321 | unregister_shrinker(&manager->mm_shrink); | |
322 | } | |
323 | ||
324 | static int ttm_set_pages_caching(struct page **pages, | |
325 | enum ttm_caching_state cstate, unsigned cpages) | |
326 | { | |
327 | int r = 0; | |
328 | /* Set page caching */ | |
329 | switch (cstate) { | |
330 | case tt_uncached: | |
331 | r = set_pages_array_uc(pages, cpages); | |
332 | if (r) | |
333 | printk(KERN_ERR "[ttm] Failed to set %d pages to uc!\n", | |
334 | cpages); | |
335 | break; | |
336 | case tt_wc: | |
337 | r = set_pages_array_wc(pages, cpages); | |
338 | if (r) | |
339 | printk(KERN_ERR "[ttm] Failed to set %d pages to wc!\n", | |
340 | cpages); | |
341 | break; | |
342 | default: | |
343 | break; | |
344 | } | |
345 | return r; | |
346 | } | |
347 | ||
348 | /** | |
349 | * Free pages the pages that failed to change the caching state. If there is | |
350 | * any pages that have changed their caching state already put them to the | |
351 | * pool. | |
352 | */ | |
353 | static void ttm_handle_caching_state_failure(struct list_head *pages, | |
354 | int ttm_flags, enum ttm_caching_state cstate, | |
355 | struct page **failed_pages, unsigned cpages) | |
356 | { | |
357 | unsigned i; | |
358 | /* Failed pages has to be reed */ | |
359 | for (i = 0; i < cpages; ++i) { | |
360 | list_del(&failed_pages[i]->lru); | |
361 | __free_page(failed_pages[i]); | |
362 | } | |
363 | } | |
364 | ||
365 | /** | |
366 | * Allocate new pages with correct caching. | |
367 | * | |
368 | * This function is reentrant if caller updates count depending on number of | |
369 | * pages returned in pages array. | |
370 | */ | |
371 | static int ttm_alloc_new_pages(struct list_head *pages, int gfp_flags, | |
372 | int ttm_flags, enum ttm_caching_state cstate, unsigned count) | |
373 | { | |
374 | struct page **caching_array; | |
375 | struct page *p; | |
376 | int r = 0; | |
377 | unsigned i, cpages; | |
378 | unsigned max_cpages = min(count, | |
379 | (unsigned)(PAGE_SIZE/sizeof(struct page *))); | |
380 | ||
381 | /* allocate array for page caching change */ | |
382 | caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); | |
383 | ||
384 | if (!caching_array) { | |
385 | printk(KERN_ERR "[ttm] unable to allocate table for new pages."); | |
386 | return -ENOMEM; | |
387 | } | |
388 | ||
389 | for (i = 0, cpages = 0; i < count; ++i) { | |
390 | p = alloc_page(gfp_flags); | |
391 | ||
392 | if (!p) { | |
393 | printk(KERN_ERR "[ttm] unable to get page %u\n", i); | |
394 | ||
395 | /* store already allocated pages in the pool after | |
396 | * setting the caching state */ | |
397 | if (cpages) { | |
398 | r = ttm_set_pages_caching(caching_array, cstate, cpages); | |
399 | if (r) | |
400 | ttm_handle_caching_state_failure(pages, | |
401 | ttm_flags, cstate, | |
402 | caching_array, cpages); | |
403 | } | |
404 | r = -ENOMEM; | |
405 | goto out; | |
406 | } | |
407 | ||
408 | #ifdef CONFIG_HIGHMEM | |
409 | /* gfp flags of highmem page should never be dma32 so we | |
410 | * we should be fine in such case | |
411 | */ | |
412 | if (!PageHighMem(p)) | |
413 | #endif | |
414 | { | |
415 | caching_array[cpages++] = p; | |
416 | if (cpages == max_cpages) { | |
417 | ||
418 | r = ttm_set_pages_caching(caching_array, | |
419 | cstate, cpages); | |
420 | if (r) { | |
421 | ttm_handle_caching_state_failure(pages, | |
422 | ttm_flags, cstate, | |
423 | caching_array, cpages); | |
424 | goto out; | |
425 | } | |
426 | cpages = 0; | |
427 | } | |
428 | } | |
429 | ||
430 | list_add(&p->lru, pages); | |
431 | } | |
432 | ||
433 | if (cpages) { | |
434 | r = ttm_set_pages_caching(caching_array, cstate, cpages); | |
435 | if (r) | |
436 | ttm_handle_caching_state_failure(pages, | |
437 | ttm_flags, cstate, | |
438 | caching_array, cpages); | |
439 | } | |
440 | out: | |
441 | kfree(caching_array); | |
442 | ||
443 | return r; | |
444 | } | |
445 | ||
446 | /** | |
447 | * Fill the given pool if there isn't enough pages and requested number of | |
448 | * pages is small. | |
449 | */ | |
450 | static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, | |
451 | int ttm_flags, enum ttm_caching_state cstate, unsigned count, | |
452 | unsigned long *irq_flags) | |
453 | { | |
454 | struct page *p; | |
455 | int r; | |
456 | unsigned cpages = 0; | |
457 | /** | |
458 | * Only allow one pool fill operation at a time. | |
459 | * If pool doesn't have enough pages for the allocation new pages are | |
460 | * allocated from outside of pool. | |
461 | */ | |
462 | if (pool->fill_lock) | |
463 | return; | |
464 | ||
465 | pool->fill_lock = true; | |
466 | ||
467 | /* If allocation request is small and there is not enough | |
468 | * pages in pool we fill the pool first */ | |
469 | if (count < _manager.options.small | |
470 | && count > pool->npages) { | |
471 | struct list_head new_pages; | |
472 | unsigned alloc_size = _manager.options.alloc_size; | |
473 | ||
474 | /** | |
475 | * Can't change page caching if in irqsave context. We have to | |
476 | * drop the pool->lock. | |
477 | */ | |
478 | spin_unlock_irqrestore(&pool->lock, *irq_flags); | |
479 | ||
480 | INIT_LIST_HEAD(&new_pages); | |
481 | r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, | |
482 | cstate, alloc_size); | |
483 | spin_lock_irqsave(&pool->lock, *irq_flags); | |
484 | ||
485 | if (!r) { | |
486 | list_splice(&new_pages, &pool->list); | |
07458661 | 487 | ++pool->nrefills; |
1403b1a3 PN |
488 | pool->npages += alloc_size; |
489 | } else { | |
490 | printk(KERN_ERR "[ttm] Failed to fill pool (%p).", pool); | |
491 | /* If we have any pages left put them to the pool. */ | |
492 | list_for_each_entry(p, &pool->list, lru) { | |
493 | ++cpages; | |
494 | } | |
495 | list_splice(&new_pages, &pool->list); | |
496 | pool->npages += cpages; | |
497 | } | |
498 | ||
499 | } | |
500 | pool->fill_lock = false; | |
501 | } | |
502 | ||
503 | /** | |
504 | * Cut count nubmer of pages from the pool and put them to return list | |
505 | * | |
506 | * @return count of pages still to allocate to fill the request. | |
507 | */ | |
508 | static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, | |
509 | struct list_head *pages, int ttm_flags, | |
510 | enum ttm_caching_state cstate, unsigned count) | |
511 | { | |
512 | unsigned long irq_flags; | |
513 | struct list_head *p; | |
514 | unsigned i; | |
515 | ||
516 | spin_lock_irqsave(&pool->lock, irq_flags); | |
517 | ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); | |
518 | ||
519 | if (count >= pool->npages) { | |
520 | /* take all pages from the pool */ | |
521 | list_splice_init(&pool->list, pages); | |
522 | count -= pool->npages; | |
523 | pool->npages = 0; | |
524 | goto out; | |
525 | } | |
526 | /* find the last pages to include for requested number of pages. Split | |
527 | * pool to begin and halves to reduce search space. */ | |
528 | if (count <= pool->npages/2) { | |
529 | i = 0; | |
530 | list_for_each(p, &pool->list) { | |
531 | if (++i == count) | |
532 | break; | |
533 | } | |
534 | } else { | |
535 | i = pool->npages + 1; | |
536 | list_for_each_prev(p, &pool->list) { | |
537 | if (--i == count) | |
538 | break; | |
539 | } | |
540 | } | |
541 | /* Cut count number of pages from pool */ | |
542 | list_cut_position(pages, &pool->list, p); | |
543 | pool->npages -= count; | |
544 | count = 0; | |
545 | out: | |
546 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
547 | return count; | |
548 | } | |
549 | ||
550 | /* | |
551 | * On success pages list will hold count number of correctly | |
552 | * cached pages. | |
553 | */ | |
554 | int ttm_get_pages(struct list_head *pages, int flags, | |
555 | enum ttm_caching_state cstate, unsigned count) | |
556 | { | |
557 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
558 | struct page *p = NULL; | |
559 | int gfp_flags = 0; | |
560 | int r; | |
561 | ||
562 | /* set zero flag for page allocation if required */ | |
563 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) | |
564 | gfp_flags |= __GFP_ZERO; | |
565 | ||
566 | /* No pool for cached pages */ | |
567 | if (pool == NULL) { | |
568 | if (flags & TTM_PAGE_FLAG_DMA32) | |
569 | gfp_flags |= GFP_DMA32; | |
570 | else | |
571 | gfp_flags |= __GFP_HIGHMEM; | |
572 | ||
573 | for (r = 0; r < count; ++r) { | |
574 | p = alloc_page(gfp_flags); | |
575 | if (!p) { | |
576 | ||
577 | printk(KERN_ERR "[ttm] unable to allocate page."); | |
578 | return -ENOMEM; | |
579 | } | |
580 | ||
581 | list_add(&p->lru, pages); | |
582 | } | |
583 | return 0; | |
584 | } | |
585 | ||
586 | ||
587 | /* combine zero flag to pool flags */ | |
588 | gfp_flags |= pool->gfp_flags; | |
589 | ||
590 | /* First we take pages from the pool */ | |
591 | count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count); | |
592 | ||
593 | /* clear the pages coming from the pool if requested */ | |
594 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { | |
595 | list_for_each_entry(p, pages, lru) { | |
596 | clear_page(page_address(p)); | |
597 | } | |
598 | } | |
599 | ||
600 | /* If pool didn't have enough pages allocate new one. */ | |
601 | if (count > 0) { | |
602 | /* ttm_alloc_new_pages doesn't reference pool so we can run | |
603 | * multiple requests in parallel. | |
604 | **/ | |
605 | r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count); | |
606 | if (r) { | |
607 | /* If there is any pages in the list put them back to | |
608 | * the pool. */ | |
609 | printk(KERN_ERR "[ttm] Failed to allocate extra pages " | |
610 | "for large request."); | |
611 | ttm_put_pages(pages, 0, flags, cstate); | |
612 | return r; | |
613 | } | |
614 | } | |
615 | ||
616 | ||
617 | return 0; | |
618 | } | |
619 | ||
620 | /* Put all pages in pages list to correct pool to wait for reuse */ | |
621 | void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags, | |
622 | enum ttm_caching_state cstate) | |
623 | { | |
624 | unsigned long irq_flags; | |
625 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
626 | struct page *p, *tmp; | |
627 | ||
628 | if (pool == NULL) { | |
629 | /* No pool for this memory type so free the pages */ | |
630 | ||
631 | list_for_each_entry_safe(p, tmp, pages, lru) { | |
632 | __free_page(p); | |
633 | } | |
634 | /* Make the pages list empty */ | |
635 | INIT_LIST_HEAD(pages); | |
636 | return; | |
637 | } | |
638 | if (page_count == 0) { | |
639 | list_for_each_entry_safe(p, tmp, pages, lru) { | |
640 | ++page_count; | |
641 | } | |
642 | } | |
643 | ||
644 | spin_lock_irqsave(&pool->lock, irq_flags); | |
645 | list_splice_init(pages, &pool->list); | |
646 | pool->npages += page_count; | |
647 | /* Check that we don't go over the pool limit */ | |
648 | page_count = 0; | |
649 | if (pool->npages > _manager.options.max_size) { | |
650 | page_count = pool->npages - _manager.options.max_size; | |
651 | /* free at least NUM_PAGES_TO_ALLOC number of pages | |
652 | * to reduce calls to set_memory_wb */ | |
653 | if (page_count < NUM_PAGES_TO_ALLOC) | |
654 | page_count = NUM_PAGES_TO_ALLOC; | |
655 | } | |
656 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
657 | if (page_count) | |
658 | ttm_page_pool_free(pool, page_count); | |
659 | } | |
660 | ||
07458661 PN |
661 | static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, |
662 | char *name) | |
1403b1a3 PN |
663 | { |
664 | spin_lock_init(&pool->lock); | |
665 | pool->fill_lock = false; | |
666 | INIT_LIST_HEAD(&pool->list); | |
07458661 | 667 | pool->npages = pool->nfrees = 0; |
1403b1a3 | 668 | pool->gfp_flags = flags; |
07458661 | 669 | pool->name = name; |
1403b1a3 PN |
670 | } |
671 | ||
672 | int ttm_page_alloc_init(unsigned max_pages) | |
673 | { | |
674 | if (atomic_add_return(1, &_manager.page_alloc_inited) > 1) | |
675 | return 0; | |
676 | ||
677 | printk(KERN_INFO "[ttm] Initializing pool allocator.\n"); | |
678 | ||
07458661 | 679 | ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER, "wc"); |
1403b1a3 | 680 | |
07458661 | 681 | ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER, "uc"); |
1403b1a3 | 682 | |
07458661 PN |
683 | ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32, |
684 | "wc dma"); | |
1403b1a3 | 685 | |
07458661 PN |
686 | ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32, |
687 | "uc dma"); | |
1403b1a3 PN |
688 | |
689 | _manager.options.max_size = max_pages; | |
690 | _manager.options.small = SMALL_ALLOCATION; | |
691 | _manager.options.alloc_size = NUM_PAGES_TO_ALLOC; | |
692 | ||
693 | ttm_pool_mm_shrink_init(&_manager); | |
694 | ||
695 | return 0; | |
696 | } | |
697 | ||
698 | void ttm_page_alloc_fini() | |
699 | { | |
700 | int i; | |
701 | ||
702 | if (atomic_sub_return(1, &_manager.page_alloc_inited) > 0) | |
703 | return; | |
704 | ||
705 | printk(KERN_INFO "[ttm] Finilizing pool allocator.\n"); | |
706 | ttm_pool_mm_shrink_fini(&_manager); | |
707 | ||
708 | for (i = 0; i < NUM_POOLS; ++i) | |
709 | ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES); | |
710 | } | |
07458661 PN |
711 | |
712 | int ttm_page_alloc_debugfs(struct seq_file *m, void *data) | |
713 | { | |
714 | struct ttm_page_pool *p; | |
715 | unsigned i; | |
716 | char *h[] = {"pool", "refills", "pages freed", "size"}; | |
717 | if (atomic_read(&_manager.page_alloc_inited) == 0) { | |
718 | seq_printf(m, "No pool allocator running.\n"); | |
719 | return 0; | |
720 | } | |
721 | seq_printf(m, "%6s %12s %13s %8s\n", | |
722 | h[0], h[1], h[2], h[3]); | |
723 | for (i = 0; i < NUM_POOLS; ++i) { | |
724 | p = &_manager.pools[i]; | |
725 | ||
726 | seq_printf(m, "%6s %12ld %13ld %8d\n", | |
727 | p->name, p->nrefills, | |
728 | p->nfrees, p->npages); | |
729 | } | |
730 | return 0; | |
731 | } | |
732 | EXPORT_SYMBOL(ttm_page_alloc_debugfs); |