xps: Transmit Packet Steering

[net-next-2.6.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 0fa9de8361bd9e182909f701a3f2166da4d5a7c7..c4a3558589ab15de3ac9abb135ee499d17544142 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -509,6 +509,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
                return NULL;
        page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
        list_del(&page->lru);
+       set_page_refcounted(page);
        h->free_huge_pages--;
        h->free_huge_pages_node[nid]--;
        return page;
@@ -868,12 +869,6 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
 
        spin_lock(&hugetlb_lock);
        if (page) {
-               /*
-                * This page is now managed by the hugetlb allocator and has
-                * no users -- drop the buddy allocator's reference.
-                */
-               put_page_testzero(page);
-               VM_BUG_ON(page_count(page));
                r_nid = page_to_nid(page);
                set_compound_page_dtor(page, free_huge_page);
                /*
@@ -936,16 +931,13 @@ retry:
        spin_unlock(&hugetlb_lock);
        for (i = 0; i < needed; i++) {
                page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
-               if (!page) {
+               if (!page)
                        /*
                         * We were not able to allocate enough pages to
                         * satisfy the entire reservation so we free what
                         * we've allocated so far.
                         */
-                       spin_lock(&hugetlb_lock);
-                       needed = 0;
                        goto free;
-               }
 
                list_add(&page->lru, &surplus_list);
        }
@@ -972,31 +964,31 @@ retry:
        needed += allocated;
        h->resv_huge_pages += delta;
        ret = 0;
-free:
+
+       spin_unlock(&hugetlb_lock);
        /* Free the needed pages to the hugetlb pool */
        list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
                if ((--needed) < 0)
                        break;
                list_del(&page->lru);
+               /*
+                * This page is now managed by the hugetlb allocator and has
+                * no users -- drop the buddy allocator's reference.
+                */
+               put_page_testzero(page);
+               VM_BUG_ON(page_count(page));
                enqueue_huge_page(h, page);
        }
 
        /* Free unnecessary surplus pages to the buddy allocator */
+free:
        if (!list_empty(&surplus_list)) {
-               spin_unlock(&hugetlb_lock);
                list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
                        list_del(&page->lru);
-                       /*
-                        * The page has a reference count of zero already, so
-                        * call free_huge_page directly instead of using
-                        * put_page.  This must be done with hugetlb_lock
-                        * unlocked which is safe because free_huge_page takes
-                        * hugetlb_lock before deciding how to free the page.
-                        */
-                       free_huge_page(page);
+                       put_page(page);
                }
-               spin_lock(&hugetlb_lock);
        }
+       spin_lock(&hugetlb_lock);
 
        return ret;
 }
@@ -1123,7 +1115,6 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
                }
        }
 
-       set_page_refcounted(page);
        set_page_private(page, (unsigned long) mapping);
 
        vma_commit_reservation(h, vma, addr);
@@ -2457,8 +2448,11 @@ retry_avoidcopy:
         * When the original hugepage is shared one, it does not have
         * anon_vma prepared.
         */
-       if (unlikely(anon_vma_prepare(vma)))
+       if (unlikely(anon_vma_prepare(vma))) {
+               /* Caller expects lock to be held */
+               spin_lock(&mm->page_table_lock);
                return VM_FAULT_OOM;
+       }
 
        copy_user_huge_page(new_page, old_page, address, vma);
        __SetPageUptodate(new_page);
@@ -2598,7 +2592,8 @@ retry:
                 * So we need to block hugepage fault by PG_hwpoison bit check.
                 */
                if (unlikely(PageHWPoison(page))) {
-                       ret = VM_FAULT_HWPOISON;
+                       ret = VM_FAULT_HWPOISON | 
+                             VM_FAULT_SET_HINDEX(h - hstates);
                        goto backout_unlocked;
                }
                page_dup_rmap(page);
@@ -2665,7 +2660,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                        migration_entry_wait(mm, (pmd_t *)ptep, address);
                        return 0;
                } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
-                       return VM_FAULT_HWPOISON;
+                       return VM_FAULT_HWPOISON_LARGE | 
+                              VM_FAULT_SET_HINDEX(h - hstates);
        }
 
        ptep = huge_pte_alloc(mm, address, huge_page_size(h));
@@ -2955,18 +2951,41 @@ void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
        hugetlb_acct_memory(h, -(chg - freed));
 }
 
+#ifdef CONFIG_MEMORY_FAILURE
+
+/* Should be called in hugetlb_lock */
+static int is_hugepage_on_freelist(struct page *hpage)
+{
+       struct page *page;
+       struct page *tmp;
+       struct hstate *h = page_hstate(hpage);
+       int nid = page_to_nid(hpage);
+
+       list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
+               if (page == hpage)
+                       return 1;
+       return 0;
+}
+
 /*
  * This function is called from memory failure code.
  * Assume the caller holds page lock of the head page.
  */
-void __isolate_hwpoisoned_huge_page(struct page *hpage)
+int dequeue_hwpoisoned_huge_page(struct page *hpage)
 {
        struct hstate *h = page_hstate(hpage);
        int nid = page_to_nid(hpage);
+       int ret = -EBUSY;
 
        spin_lock(&hugetlb_lock);
-       list_del(&hpage->lru);
-       h->free_huge_pages--;
-       h->free_huge_pages_node[nid]--;
+       if (is_hugepage_on_freelist(hpage)) {
+               list_del(&hpage->lru);
+               set_page_refcounted(hpage);
+               h->free_huge_pages--;
+               h->free_huge_pages_node[nid]--;
+               ret = 0;
+       }
        spin_unlock(&hugetlb_lock);
+       return ret;
 }
+#endif