page allocator: smarter retry of costly-order allocations

[net-next-2.6.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d1cf4f05dcdaff12c4e261495efe53a3b3dd7209..0a502e99ee22bb162f472ad5d4a9f3c2ec960c75 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1461,7 +1461,8 @@ __alloc_pages_internal(gfp_t gfp_mask, unsigned int order,
        struct task_struct *p = current;
        int do_retry;
        int alloc_flags;
-       int did_some_progress;
+       unsigned long did_some_progress;
+       unsigned long pages_reclaimed = 0;
 
        might_sleep_if(wait);
 
@@ -1611,14 +1612,26 @@ nofail_alloc:
         * Don't let big-order allocations loop unless the caller explicitly
         * requests that.  Wait for some write requests to complete then retry.
         *
-        * In this implementation, __GFP_REPEAT means __GFP_NOFAIL for order
-        * <= 3, but that may not be true in other implementations.
+        * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
+        * means __GFP_NOFAIL, but that may not be true in other
+        * implementations.
+        *
+        * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
+        * specified, then we retry until we no longer reclaim any pages
+        * (above), or we've reclaimed an order of pages at least as
+        * large as the allocation's order. In both cases, if the
+        * allocation still fails, we stop retrying.
         */
+       pages_reclaimed += did_some_progress;
        do_retry = 0;
        if (!(gfp_mask & __GFP_NORETRY)) {
-               if ((order <= PAGE_ALLOC_COSTLY_ORDER) ||
-                                               (gfp_mask & __GFP_REPEAT))
+               if (order <= PAGE_ALLOC_COSTLY_ORDER) {
                        do_retry = 1;
+               } else {
+                       if (gfp_mask & __GFP_REPEAT &&
+                               pages_reclaimed < (1 << order))
+                                       do_retry = 1;
+               }
                if (gfp_mask & __GFP_NOFAIL)
                        do_retry = 1;
        }
@@ -2524,7 +2537,9 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
        struct page *page;
        unsigned long end_pfn = start_pfn + size;
        unsigned long pfn;
+       struct zone *z;
 
+       z = &NODE_DATA(nid)->node_zones[zone];
        for (pfn = start_pfn; pfn < end_pfn; pfn++) {
                /*
                 * There can be holes in boot-time mem_map[]s
@@ -2542,7 +2557,6 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                init_page_count(page);
                reset_page_mapcount(page);
                SetPageReserved(page);
-
                /*
                 * Mark the block movable so that blocks are reserved for
                 * movable at startup. This will force kernel allocations
@@ -2551,8 +2565,15 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                 * kernel allocations are made. Later some blocks near
                 * the start are marked MIGRATE_RESERVE by
                 * setup_zone_migrate_reserve()
+                *
+                * bitmap is created for zone's valid pfn range. but memmap
+                * can be created for invalid pages (for alignment)
+                * check here not to call set_pageblock_migratetype() against
+                * pfn out of zone.
                 */
-               if ((pfn & (pageblock_nr_pages-1)))
+               if ((z->zone_start_pfn <= pfn)
+                   && (pfn < z->zone_start_pfn + z->spanned_pages)
+                   && !(pfn & (pageblock_nr_pages - 1)))
                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
 
                INIT_LIST_HEAD(&page->lru);
@@ -4464,6 +4485,8 @@ void set_pageblock_flags_group(struct page *page, unsigned long flags,
        pfn = page_to_pfn(page);
        bitmap = get_pageblock_bitmap(zone, pfn);
        bitidx = pfn_to_bitidx(zone, pfn);
+       VM_BUG_ON(pfn < zone->zone_start_pfn);
+       VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
 
        for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
                if (flags & value)