]>
Commit | Line | Data |
---|---|---|
1403b1a3 PN |
1 | /* |
2 | * Copyright (c) Red Hat Inc. | |
3 | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sub license, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the | |
12 | * next paragraph) shall be included in all copies or substantial portions | |
13 | * of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | |
21 | * DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: Dave Airlie <airlied@redhat.com> | |
24 | * Jerome Glisse <jglisse@redhat.com> | |
25 | * Pauli Nieminen <suokkos@gmail.com> | |
26 | */ | |
27 | ||
28 | /* simple list based uncached page pool | |
29 | * - Pool collects resently freed pages for reuse | |
30 | * - Use page->lru to keep a free list | |
31 | * - doesn't track currently in use pages | |
32 | */ | |
33 | #include <linux/list.h> | |
34 | #include <linux/spinlock.h> | |
35 | #include <linux/highmem.h> | |
36 | #include <linux/mm_types.h> | |
07458661 | 37 | #include <linux/module.h> |
1403b1a3 | 38 | #include <linux/mm.h> |
4cdc840a | 39 | #include <linux/seq_file.h> /* for seq_printf */ |
2125b8a4 | 40 | #include <linux/slab.h> |
1403b1a3 PN |
41 | |
42 | #include <asm/atomic.h> | |
43 | #include <asm/agp.h> | |
44 | ||
45 | #include "ttm/ttm_bo_driver.h" | |
46 | #include "ttm/ttm_page_alloc.h" | |
47 | ||
48 | ||
49 | #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) | |
50 | #define SMALL_ALLOCATION 16 | |
51 | #define FREE_ALL_PAGES (~0U) | |
52 | /* times are in msecs */ | |
53 | #define PAGE_FREE_INTERVAL 1000 | |
54 | ||
55 | /** | |
56 | * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. | |
57 | * | |
58 | * @lock: Protects the shared pool from concurrnet access. Must be used with | |
59 | * irqsave/irqrestore variants because pool allocator maybe called from | |
60 | * delayed work. | |
61 | * @fill_lock: Prevent concurrent calls to fill. | |
62 | * @list: Pool of free uc/wc pages for fast reuse. | |
63 | * @gfp_flags: Flags to pass for alloc_page. | |
64 | * @npages: Number of pages in pool. | |
65 | */ | |
66 | struct ttm_page_pool { | |
67 | spinlock_t lock; | |
68 | bool fill_lock; | |
69 | struct list_head list; | |
70 | int gfp_flags; | |
71 | unsigned npages; | |
07458661 PN |
72 | char *name; |
73 | unsigned long nfrees; | |
74 | unsigned long nrefills; | |
1403b1a3 PN |
75 | }; |
76 | ||
c96af79e PN |
77 | /** |
78 | * Limits for the pool. They are handled without locks because only place where | |
79 | * they may change is in sysfs store. They won't have immediate effect anyway | |
4abe4389 | 80 | * so forcing serialization to access them is pointless. |
c96af79e PN |
81 | */ |
82 | ||
1403b1a3 PN |
83 | struct ttm_pool_opts { |
84 | unsigned alloc_size; | |
85 | unsigned max_size; | |
86 | unsigned small; | |
87 | }; | |
88 | ||
89 | #define NUM_POOLS 4 | |
90 | ||
91 | /** | |
92 | * struct ttm_pool_manager - Holds memory pools for fst allocation | |
93 | * | |
94 | * Manager is read only object for pool code so it doesn't need locking. | |
95 | * | |
96 | * @free_interval: minimum number of jiffies between freeing pages from pool. | |
97 | * @page_alloc_inited: reference counting for pool allocation. | |
98 | * @work: Work that is used to shrink the pool. Work is only run when there is | |
99 | * some pages to free. | |
100 | * @small_allocation: Limit in number of pages what is small allocation. | |
101 | * | |
102 | * @pools: All pool objects in use. | |
103 | **/ | |
104 | struct ttm_pool_manager { | |
c96af79e | 105 | struct kobject kobj; |
1403b1a3 PN |
106 | struct shrinker mm_shrink; |
107 | atomic_t page_alloc_inited; | |
108 | struct ttm_pool_opts options; | |
109 | ||
110 | union { | |
111 | struct ttm_page_pool pools[NUM_POOLS]; | |
112 | struct { | |
113 | struct ttm_page_pool wc_pool; | |
114 | struct ttm_page_pool uc_pool; | |
115 | struct ttm_page_pool wc_pool_dma32; | |
116 | struct ttm_page_pool uc_pool_dma32; | |
117 | } ; | |
118 | }; | |
119 | }; | |
120 | ||
c96af79e PN |
121 | static struct attribute ttm_page_pool_max = { |
122 | .name = "pool_max_size", | |
123 | .mode = S_IRUGO | S_IWUSR | |
124 | }; | |
125 | static struct attribute ttm_page_pool_small = { | |
126 | .name = "pool_small_allocation", | |
127 | .mode = S_IRUGO | S_IWUSR | |
128 | }; | |
129 | static struct attribute ttm_page_pool_alloc_size = { | |
130 | .name = "pool_allocation_size", | |
131 | .mode = S_IRUGO | S_IWUSR | |
132 | }; | |
133 | ||
134 | static struct attribute *ttm_pool_attrs[] = { | |
135 | &ttm_page_pool_max, | |
136 | &ttm_page_pool_small, | |
137 | &ttm_page_pool_alloc_size, | |
138 | NULL | |
139 | }; | |
140 | ||
141 | static void ttm_pool_kobj_release(struct kobject *kobj) | |
142 | { | |
143 | struct ttm_pool_manager *m = | |
144 | container_of(kobj, struct ttm_pool_manager, kobj); | |
145 | (void)m; | |
146 | } | |
147 | ||
148 | static ssize_t ttm_pool_store(struct kobject *kobj, | |
149 | struct attribute *attr, const char *buffer, size_t size) | |
150 | { | |
151 | struct ttm_pool_manager *m = | |
152 | container_of(kobj, struct ttm_pool_manager, kobj); | |
153 | int chars; | |
154 | unsigned val; | |
155 | chars = sscanf(buffer, "%u", &val); | |
156 | if (chars == 0) | |
157 | return size; | |
158 | ||
159 | /* Convert kb to number of pages */ | |
160 | val = val / (PAGE_SIZE >> 10); | |
161 | ||
162 | if (attr == &ttm_page_pool_max) | |
163 | m->options.max_size = val; | |
164 | else if (attr == &ttm_page_pool_small) | |
165 | m->options.small = val; | |
166 | else if (attr == &ttm_page_pool_alloc_size) { | |
167 | if (val > NUM_PAGES_TO_ALLOC*8) { | |
4abe4389 TH |
168 | printk(KERN_ERR TTM_PFX |
169 | "Setting allocation size to %lu " | |
170 | "is not allowed. Recommended size is " | |
171 | "%lu\n", | |
172 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), | |
173 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); | |
c96af79e PN |
174 | return size; |
175 | } else if (val > NUM_PAGES_TO_ALLOC) { | |
4abe4389 TH |
176 | printk(KERN_WARNING TTM_PFX |
177 | "Setting allocation size to " | |
178 | "larger than %lu is not recommended.\n", | |
179 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); | |
c96af79e PN |
180 | } |
181 | m->options.alloc_size = val; | |
182 | } | |
183 | ||
184 | return size; | |
185 | } | |
186 | ||
187 | static ssize_t ttm_pool_show(struct kobject *kobj, | |
188 | struct attribute *attr, char *buffer) | |
189 | { | |
190 | struct ttm_pool_manager *m = | |
191 | container_of(kobj, struct ttm_pool_manager, kobj); | |
192 | unsigned val = 0; | |
193 | ||
194 | if (attr == &ttm_page_pool_max) | |
195 | val = m->options.max_size; | |
196 | else if (attr == &ttm_page_pool_small) | |
197 | val = m->options.small; | |
198 | else if (attr == &ttm_page_pool_alloc_size) | |
199 | val = m->options.alloc_size; | |
200 | ||
201 | val = val * (PAGE_SIZE >> 10); | |
202 | ||
203 | return snprintf(buffer, PAGE_SIZE, "%u\n", val); | |
204 | } | |
205 | ||
206 | static const struct sysfs_ops ttm_pool_sysfs_ops = { | |
207 | .show = &ttm_pool_show, | |
208 | .store = &ttm_pool_store, | |
209 | }; | |
210 | ||
211 | static struct kobj_type ttm_pool_kobj_type = { | |
212 | .release = &ttm_pool_kobj_release, | |
213 | .sysfs_ops = &ttm_pool_sysfs_ops, | |
214 | .default_attrs = ttm_pool_attrs, | |
215 | }; | |
216 | ||
1403b1a3 PN |
217 | static struct ttm_pool_manager _manager = { |
218 | .page_alloc_inited = ATOMIC_INIT(0) | |
219 | }; | |
220 | ||
975efdb1 | 221 | #ifndef CONFIG_X86 |
1403b1a3 PN |
222 | static int set_pages_array_wb(struct page **pages, int addrinarray) |
223 | { | |
224 | #ifdef TTM_HAS_AGP | |
225 | int i; | |
226 | ||
227 | for (i = 0; i < addrinarray; i++) | |
228 | unmap_page_from_agp(pages[i]); | |
229 | #endif | |
230 | return 0; | |
231 | } | |
232 | ||
233 | static int set_pages_array_wc(struct page **pages, int addrinarray) | |
234 | { | |
235 | #ifdef TTM_HAS_AGP | |
236 | int i; | |
237 | ||
238 | for (i = 0; i < addrinarray; i++) | |
239 | map_page_into_agp(pages[i]); | |
240 | #endif | |
241 | return 0; | |
242 | } | |
243 | ||
244 | static int set_pages_array_uc(struct page **pages, int addrinarray) | |
245 | { | |
246 | #ifdef TTM_HAS_AGP | |
247 | int i; | |
248 | ||
249 | for (i = 0; i < addrinarray; i++) | |
250 | map_page_into_agp(pages[i]); | |
251 | #endif | |
252 | return 0; | |
253 | } | |
254 | #endif | |
255 | ||
256 | /** | |
257 | * Select the right pool or requested caching state and ttm flags. */ | |
258 | static struct ttm_page_pool *ttm_get_pool(int flags, | |
259 | enum ttm_caching_state cstate) | |
260 | { | |
261 | int pool_index; | |
262 | ||
263 | if (cstate == tt_cached) | |
264 | return NULL; | |
265 | ||
266 | if (cstate == tt_wc) | |
267 | pool_index = 0x0; | |
268 | else | |
269 | pool_index = 0x1; | |
270 | ||
271 | if (flags & TTM_PAGE_FLAG_DMA32) | |
272 | pool_index |= 0x2; | |
273 | ||
274 | return &_manager.pools[pool_index]; | |
275 | } | |
276 | ||
277 | /* set memory back to wb and free the pages. */ | |
278 | static void ttm_pages_put(struct page *pages[], unsigned npages) | |
279 | { | |
280 | unsigned i; | |
281 | if (set_pages_array_wb(pages, npages)) | |
4abe4389 | 282 | printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n", |
1403b1a3 PN |
283 | npages); |
284 | for (i = 0; i < npages; ++i) | |
285 | __free_page(pages[i]); | |
286 | } | |
287 | ||
288 | static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, | |
289 | unsigned freed_pages) | |
290 | { | |
291 | pool->npages -= freed_pages; | |
07458661 | 292 | pool->nfrees += freed_pages; |
1403b1a3 PN |
293 | } |
294 | ||
295 | /** | |
296 | * Free pages from pool. | |
297 | * | |
298 | * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC | |
299 | * number of pages in one go. | |
300 | * | |
301 | * @pool: to free the pages from | |
302 | * @free_all: If set to true will free all pages in pool | |
303 | **/ | |
304 | static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) | |
305 | { | |
306 | unsigned long irq_flags; | |
307 | struct page *p; | |
308 | struct page **pages_to_free; | |
309 | unsigned freed_pages = 0, | |
310 | npages_to_free = nr_free; | |
311 | ||
312 | if (NUM_PAGES_TO_ALLOC < nr_free) | |
313 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
314 | ||
315 | pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), | |
316 | GFP_KERNEL); | |
317 | if (!pages_to_free) { | |
4abe4389 TH |
318 | printk(KERN_ERR TTM_PFX |
319 | "Failed to allocate memory for pool free operation.\n"); | |
1403b1a3 PN |
320 | return 0; |
321 | } | |
322 | ||
323 | restart: | |
324 | spin_lock_irqsave(&pool->lock, irq_flags); | |
325 | ||
326 | list_for_each_entry_reverse(p, &pool->list, lru) { | |
327 | if (freed_pages >= npages_to_free) | |
328 | break; | |
329 | ||
330 | pages_to_free[freed_pages++] = p; | |
331 | /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ | |
332 | if (freed_pages >= NUM_PAGES_TO_ALLOC) { | |
333 | /* remove range of pages from the pool */ | |
334 | __list_del(p->lru.prev, &pool->list); | |
335 | ||
336 | ttm_pool_update_free_locked(pool, freed_pages); | |
337 | /** | |
338 | * Because changing page caching is costly | |
339 | * we unlock the pool to prevent stalling. | |
340 | */ | |
341 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
342 | ||
343 | ttm_pages_put(pages_to_free, freed_pages); | |
344 | if (likely(nr_free != FREE_ALL_PAGES)) | |
345 | nr_free -= freed_pages; | |
346 | ||
347 | if (NUM_PAGES_TO_ALLOC >= nr_free) | |
348 | npages_to_free = nr_free; | |
349 | else | |
350 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
351 | ||
352 | freed_pages = 0; | |
353 | ||
354 | /* free all so restart the processing */ | |
355 | if (nr_free) | |
356 | goto restart; | |
357 | ||
358 | /* Not allowed to fall tough or break because | |
359 | * following context is inside spinlock while we are | |
360 | * outside here. | |
361 | */ | |
362 | goto out; | |
363 | ||
364 | } | |
365 | } | |
366 | ||
1403b1a3 PN |
367 | /* remove range of pages from the pool */ |
368 | if (freed_pages) { | |
369 | __list_del(&p->lru, &pool->list); | |
370 | ||
371 | ttm_pool_update_free_locked(pool, freed_pages); | |
372 | nr_free -= freed_pages; | |
373 | } | |
374 | ||
375 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
376 | ||
377 | if (freed_pages) | |
378 | ttm_pages_put(pages_to_free, freed_pages); | |
379 | out: | |
380 | kfree(pages_to_free); | |
381 | return nr_free; | |
382 | } | |
383 | ||
384 | /* Get good estimation how many pages are free in pools */ | |
385 | static int ttm_pool_get_num_unused_pages(void) | |
386 | { | |
387 | unsigned i; | |
388 | int total = 0; | |
389 | for (i = 0; i < NUM_POOLS; ++i) | |
390 | total += _manager.pools[i].npages; | |
391 | ||
392 | return total; | |
393 | } | |
394 | ||
395 | /** | |
4abe4389 | 396 | * Callback for mm to request pool to reduce number of page held. |
1403b1a3 PN |
397 | */ |
398 | static int ttm_pool_mm_shrink(int shrink_pages, gfp_t gfp_mask) | |
399 | { | |
400 | static atomic_t start_pool = ATOMIC_INIT(0); | |
401 | unsigned i; | |
402 | unsigned pool_offset = atomic_add_return(1, &start_pool); | |
403 | struct ttm_page_pool *pool; | |
404 | ||
405 | pool_offset = pool_offset % NUM_POOLS; | |
406 | /* select start pool in round robin fashion */ | |
407 | for (i = 0; i < NUM_POOLS; ++i) { | |
408 | unsigned nr_free = shrink_pages; | |
409 | if (shrink_pages == 0) | |
410 | break; | |
411 | pool = &_manager.pools[(i + pool_offset)%NUM_POOLS]; | |
412 | shrink_pages = ttm_page_pool_free(pool, nr_free); | |
413 | } | |
414 | /* return estimated number of unused pages in pool */ | |
415 | return ttm_pool_get_num_unused_pages(); | |
416 | } | |
417 | ||
418 | static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) | |
419 | { | |
420 | manager->mm_shrink.shrink = &ttm_pool_mm_shrink; | |
421 | manager->mm_shrink.seeks = 1; | |
422 | register_shrinker(&manager->mm_shrink); | |
423 | } | |
424 | ||
425 | static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) | |
426 | { | |
427 | unregister_shrinker(&manager->mm_shrink); | |
428 | } | |
429 | ||
430 | static int ttm_set_pages_caching(struct page **pages, | |
431 | enum ttm_caching_state cstate, unsigned cpages) | |
432 | { | |
433 | int r = 0; | |
434 | /* Set page caching */ | |
435 | switch (cstate) { | |
436 | case tt_uncached: | |
437 | r = set_pages_array_uc(pages, cpages); | |
438 | if (r) | |
4abe4389 TH |
439 | printk(KERN_ERR TTM_PFX |
440 | "Failed to set %d pages to uc!\n", | |
441 | cpages); | |
1403b1a3 PN |
442 | break; |
443 | case tt_wc: | |
444 | r = set_pages_array_wc(pages, cpages); | |
445 | if (r) | |
4abe4389 TH |
446 | printk(KERN_ERR TTM_PFX |
447 | "Failed to set %d pages to wc!\n", | |
448 | cpages); | |
1403b1a3 PN |
449 | break; |
450 | default: | |
451 | break; | |
452 | } | |
453 | return r; | |
454 | } | |
455 | ||
456 | /** | |
457 | * Free pages the pages that failed to change the caching state. If there is | |
458 | * any pages that have changed their caching state already put them to the | |
459 | * pool. | |
460 | */ | |
461 | static void ttm_handle_caching_state_failure(struct list_head *pages, | |
462 | int ttm_flags, enum ttm_caching_state cstate, | |
463 | struct page **failed_pages, unsigned cpages) | |
464 | { | |
465 | unsigned i; | |
4abe4389 | 466 | /* Failed pages have to be freed */ |
1403b1a3 PN |
467 | for (i = 0; i < cpages; ++i) { |
468 | list_del(&failed_pages[i]->lru); | |
469 | __free_page(failed_pages[i]); | |
470 | } | |
471 | } | |
472 | ||
473 | /** | |
474 | * Allocate new pages with correct caching. | |
475 | * | |
476 | * This function is reentrant if caller updates count depending on number of | |
477 | * pages returned in pages array. | |
478 | */ | |
479 | static int ttm_alloc_new_pages(struct list_head *pages, int gfp_flags, | |
480 | int ttm_flags, enum ttm_caching_state cstate, unsigned count) | |
481 | { | |
482 | struct page **caching_array; | |
483 | struct page *p; | |
484 | int r = 0; | |
485 | unsigned i, cpages; | |
486 | unsigned max_cpages = min(count, | |
487 | (unsigned)(PAGE_SIZE/sizeof(struct page *))); | |
488 | ||
489 | /* allocate array for page caching change */ | |
490 | caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); | |
491 | ||
492 | if (!caching_array) { | |
4abe4389 TH |
493 | printk(KERN_ERR TTM_PFX |
494 | "Unable to allocate table for new pages."); | |
1403b1a3 PN |
495 | return -ENOMEM; |
496 | } | |
497 | ||
498 | for (i = 0, cpages = 0; i < count; ++i) { | |
499 | p = alloc_page(gfp_flags); | |
500 | ||
501 | if (!p) { | |
4abe4389 | 502 | printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i); |
1403b1a3 PN |
503 | |
504 | /* store already allocated pages in the pool after | |
505 | * setting the caching state */ | |
506 | if (cpages) { | |
4abe4389 TH |
507 | r = ttm_set_pages_caching(caching_array, |
508 | cstate, cpages); | |
1403b1a3 PN |
509 | if (r) |
510 | ttm_handle_caching_state_failure(pages, | |
511 | ttm_flags, cstate, | |
512 | caching_array, cpages); | |
513 | } | |
514 | r = -ENOMEM; | |
515 | goto out; | |
516 | } | |
517 | ||
518 | #ifdef CONFIG_HIGHMEM | |
519 | /* gfp flags of highmem page should never be dma32 so we | |
520 | * we should be fine in such case | |
521 | */ | |
522 | if (!PageHighMem(p)) | |
523 | #endif | |
524 | { | |
525 | caching_array[cpages++] = p; | |
526 | if (cpages == max_cpages) { | |
527 | ||
528 | r = ttm_set_pages_caching(caching_array, | |
529 | cstate, cpages); | |
530 | if (r) { | |
531 | ttm_handle_caching_state_failure(pages, | |
532 | ttm_flags, cstate, | |
533 | caching_array, cpages); | |
534 | goto out; | |
535 | } | |
536 | cpages = 0; | |
537 | } | |
538 | } | |
539 | ||
540 | list_add(&p->lru, pages); | |
541 | } | |
542 | ||
543 | if (cpages) { | |
544 | r = ttm_set_pages_caching(caching_array, cstate, cpages); | |
545 | if (r) | |
546 | ttm_handle_caching_state_failure(pages, | |
547 | ttm_flags, cstate, | |
548 | caching_array, cpages); | |
549 | } | |
550 | out: | |
551 | kfree(caching_array); | |
552 | ||
553 | return r; | |
554 | } | |
555 | ||
556 | /** | |
557 | * Fill the given pool if there isn't enough pages and requested number of | |
558 | * pages is small. | |
559 | */ | |
560 | static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, | |
561 | int ttm_flags, enum ttm_caching_state cstate, unsigned count, | |
562 | unsigned long *irq_flags) | |
563 | { | |
564 | struct page *p; | |
565 | int r; | |
566 | unsigned cpages = 0; | |
567 | /** | |
568 | * Only allow one pool fill operation at a time. | |
569 | * If pool doesn't have enough pages for the allocation new pages are | |
570 | * allocated from outside of pool. | |
571 | */ | |
572 | if (pool->fill_lock) | |
573 | return; | |
574 | ||
575 | pool->fill_lock = true; | |
576 | ||
577 | /* If allocation request is small and there is not enough | |
578 | * pages in pool we fill the pool first */ | |
579 | if (count < _manager.options.small | |
580 | && count > pool->npages) { | |
581 | struct list_head new_pages; | |
582 | unsigned alloc_size = _manager.options.alloc_size; | |
583 | ||
584 | /** | |
585 | * Can't change page caching if in irqsave context. We have to | |
586 | * drop the pool->lock. | |
587 | */ | |
588 | spin_unlock_irqrestore(&pool->lock, *irq_flags); | |
589 | ||
590 | INIT_LIST_HEAD(&new_pages); | |
591 | r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, | |
592 | cstate, alloc_size); | |
593 | spin_lock_irqsave(&pool->lock, *irq_flags); | |
594 | ||
595 | if (!r) { | |
596 | list_splice(&new_pages, &pool->list); | |
07458661 | 597 | ++pool->nrefills; |
1403b1a3 PN |
598 | pool->npages += alloc_size; |
599 | } else { | |
4abe4389 TH |
600 | printk(KERN_ERR TTM_PFX |
601 | "Failed to fill pool (%p).", pool); | |
1403b1a3 PN |
602 | /* If we have any pages left put them to the pool. */ |
603 | list_for_each_entry(p, &pool->list, lru) { | |
604 | ++cpages; | |
605 | } | |
606 | list_splice(&new_pages, &pool->list); | |
607 | pool->npages += cpages; | |
608 | } | |
609 | ||
610 | } | |
611 | pool->fill_lock = false; | |
612 | } | |
613 | ||
614 | /** | |
615 | * Cut count nubmer of pages from the pool and put them to return list | |
616 | * | |
617 | * @return count of pages still to allocate to fill the request. | |
618 | */ | |
619 | static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, | |
620 | struct list_head *pages, int ttm_flags, | |
621 | enum ttm_caching_state cstate, unsigned count) | |
622 | { | |
623 | unsigned long irq_flags; | |
624 | struct list_head *p; | |
625 | unsigned i; | |
626 | ||
627 | spin_lock_irqsave(&pool->lock, irq_flags); | |
628 | ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); | |
629 | ||
630 | if (count >= pool->npages) { | |
631 | /* take all pages from the pool */ | |
632 | list_splice_init(&pool->list, pages); | |
633 | count -= pool->npages; | |
634 | pool->npages = 0; | |
635 | goto out; | |
636 | } | |
637 | /* find the last pages to include for requested number of pages. Split | |
638 | * pool to begin and halves to reduce search space. */ | |
639 | if (count <= pool->npages/2) { | |
640 | i = 0; | |
641 | list_for_each(p, &pool->list) { | |
642 | if (++i == count) | |
643 | break; | |
644 | } | |
645 | } else { | |
646 | i = pool->npages + 1; | |
647 | list_for_each_prev(p, &pool->list) { | |
648 | if (--i == count) | |
649 | break; | |
650 | } | |
651 | } | |
652 | /* Cut count number of pages from pool */ | |
653 | list_cut_position(pages, &pool->list, p); | |
654 | pool->npages -= count; | |
655 | count = 0; | |
656 | out: | |
657 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
658 | return count; | |
659 | } | |
660 | ||
661 | /* | |
662 | * On success pages list will hold count number of correctly | |
663 | * cached pages. | |
664 | */ | |
665 | int ttm_get_pages(struct list_head *pages, int flags, | |
666 | enum ttm_caching_state cstate, unsigned count) | |
667 | { | |
668 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
669 | struct page *p = NULL; | |
7c2a9acf | 670 | int gfp_flags = GFP_USER; |
1403b1a3 PN |
671 | int r; |
672 | ||
673 | /* set zero flag for page allocation if required */ | |
674 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) | |
675 | gfp_flags |= __GFP_ZERO; | |
676 | ||
677 | /* No pool for cached pages */ | |
678 | if (pool == NULL) { | |
679 | if (flags & TTM_PAGE_FLAG_DMA32) | |
680 | gfp_flags |= GFP_DMA32; | |
681 | else | |
e8613c0e | 682 | gfp_flags |= GFP_HIGHUSER; |
1403b1a3 PN |
683 | |
684 | for (r = 0; r < count; ++r) { | |
685 | p = alloc_page(gfp_flags); | |
686 | if (!p) { | |
687 | ||
4abe4389 TH |
688 | printk(KERN_ERR TTM_PFX |
689 | "Unable to allocate page."); | |
1403b1a3 PN |
690 | return -ENOMEM; |
691 | } | |
692 | ||
693 | list_add(&p->lru, pages); | |
694 | } | |
695 | return 0; | |
696 | } | |
697 | ||
698 | ||
699 | /* combine zero flag to pool flags */ | |
700 | gfp_flags |= pool->gfp_flags; | |
701 | ||
702 | /* First we take pages from the pool */ | |
703 | count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count); | |
704 | ||
705 | /* clear the pages coming from the pool if requested */ | |
706 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { | |
707 | list_for_each_entry(p, pages, lru) { | |
708 | clear_page(page_address(p)); | |
709 | } | |
710 | } | |
711 | ||
712 | /* If pool didn't have enough pages allocate new one. */ | |
713 | if (count > 0) { | |
714 | /* ttm_alloc_new_pages doesn't reference pool so we can run | |
715 | * multiple requests in parallel. | |
716 | **/ | |
717 | r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count); | |
718 | if (r) { | |
719 | /* If there is any pages in the list put them back to | |
720 | * the pool. */ | |
4abe4389 TH |
721 | printk(KERN_ERR TTM_PFX |
722 | "Failed to allocate extra pages " | |
723 | "for large request."); | |
1403b1a3 PN |
724 | ttm_put_pages(pages, 0, flags, cstate); |
725 | return r; | |
726 | } | |
727 | } | |
728 | ||
729 | ||
730 | return 0; | |
731 | } | |
732 | ||
733 | /* Put all pages in pages list to correct pool to wait for reuse */ | |
734 | void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags, | |
735 | enum ttm_caching_state cstate) | |
736 | { | |
737 | unsigned long irq_flags; | |
738 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
739 | struct page *p, *tmp; | |
740 | ||
741 | if (pool == NULL) { | |
742 | /* No pool for this memory type so free the pages */ | |
743 | ||
744 | list_for_each_entry_safe(p, tmp, pages, lru) { | |
745 | __free_page(p); | |
746 | } | |
747 | /* Make the pages list empty */ | |
748 | INIT_LIST_HEAD(pages); | |
749 | return; | |
750 | } | |
751 | if (page_count == 0) { | |
752 | list_for_each_entry_safe(p, tmp, pages, lru) { | |
753 | ++page_count; | |
754 | } | |
755 | } | |
756 | ||
757 | spin_lock_irqsave(&pool->lock, irq_flags); | |
758 | list_splice_init(pages, &pool->list); | |
759 | pool->npages += page_count; | |
760 | /* Check that we don't go over the pool limit */ | |
761 | page_count = 0; | |
762 | if (pool->npages > _manager.options.max_size) { | |
763 | page_count = pool->npages - _manager.options.max_size; | |
764 | /* free at least NUM_PAGES_TO_ALLOC number of pages | |
765 | * to reduce calls to set_memory_wb */ | |
766 | if (page_count < NUM_PAGES_TO_ALLOC) | |
767 | page_count = NUM_PAGES_TO_ALLOC; | |
768 | } | |
769 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
770 | if (page_count) | |
771 | ttm_page_pool_free(pool, page_count); | |
772 | } | |
773 | ||
07458661 PN |
774 | static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, |
775 | char *name) | |
1403b1a3 PN |
776 | { |
777 | spin_lock_init(&pool->lock); | |
778 | pool->fill_lock = false; | |
779 | INIT_LIST_HEAD(&pool->list); | |
07458661 | 780 | pool->npages = pool->nfrees = 0; |
1403b1a3 | 781 | pool->gfp_flags = flags; |
07458661 | 782 | pool->name = name; |
1403b1a3 PN |
783 | } |
784 | ||
c96af79e | 785 | int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) |
1403b1a3 | 786 | { |
c96af79e | 787 | int ret; |
1403b1a3 PN |
788 | if (atomic_add_return(1, &_manager.page_alloc_inited) > 1) |
789 | return 0; | |
790 | ||
4abe4389 | 791 | printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n"); |
1403b1a3 | 792 | |
07458661 | 793 | ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER, "wc"); |
1403b1a3 | 794 | |
07458661 | 795 | ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER, "uc"); |
1403b1a3 | 796 | |
07458661 PN |
797 | ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32, |
798 | "wc dma"); | |
1403b1a3 | 799 | |
07458661 PN |
800 | ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32, |
801 | "uc dma"); | |
1403b1a3 PN |
802 | |
803 | _manager.options.max_size = max_pages; | |
804 | _manager.options.small = SMALL_ALLOCATION; | |
805 | _manager.options.alloc_size = NUM_PAGES_TO_ALLOC; | |
806 | ||
c96af79e PN |
807 | kobject_init(&_manager.kobj, &ttm_pool_kobj_type); |
808 | ret = kobject_add(&_manager.kobj, &glob->kobj, "pool"); | |
809 | if (unlikely(ret != 0)) { | |
810 | kobject_put(&_manager.kobj); | |
811 | return ret; | |
812 | } | |
813 | ||
1403b1a3 PN |
814 | ttm_pool_mm_shrink_init(&_manager); |
815 | ||
816 | return 0; | |
817 | } | |
818 | ||
819 | void ttm_page_alloc_fini() | |
820 | { | |
821 | int i; | |
822 | ||
823 | if (atomic_sub_return(1, &_manager.page_alloc_inited) > 0) | |
824 | return; | |
825 | ||
4abe4389 | 826 | printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n"); |
1403b1a3 PN |
827 | ttm_pool_mm_shrink_fini(&_manager); |
828 | ||
829 | for (i = 0; i < NUM_POOLS; ++i) | |
830 | ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES); | |
c96af79e PN |
831 | |
832 | kobject_put(&_manager.kobj); | |
1403b1a3 | 833 | } |
07458661 PN |
834 | |
835 | int ttm_page_alloc_debugfs(struct seq_file *m, void *data) | |
836 | { | |
837 | struct ttm_page_pool *p; | |
838 | unsigned i; | |
839 | char *h[] = {"pool", "refills", "pages freed", "size"}; | |
840 | if (atomic_read(&_manager.page_alloc_inited) == 0) { | |
841 | seq_printf(m, "No pool allocator running.\n"); | |
842 | return 0; | |
843 | } | |
844 | seq_printf(m, "%6s %12s %13s %8s\n", | |
845 | h[0], h[1], h[2], h[3]); | |
846 | for (i = 0; i < NUM_POOLS; ++i) { | |
847 | p = &_manager.pools[i]; | |
848 | ||
849 | seq_printf(m, "%6s %12ld %13ld %8d\n", | |
850 | p->name, p->nrefills, | |
851 | p->nfrees, p->npages); | |
852 | } | |
853 | return 0; | |
854 | } | |
855 | EXPORT_SYMBOL(ttm_page_alloc_debugfs); |