[PATCH] hugepage: serialize hugepage allocation and instantiation
Currently, no lock or mutex is held between allocating a hugepage and inserting it into the pagetables / page cache. When we do go to insert the page into pagetables or page cache, we recheck and may free the newly allocated hugepage. However, since the number of hugepages in the system is strictly limited, and it's usualy to want to use all of them, this can still lead to spurious allocation failures. For example, suppose two processes are both mapping (MAP_SHARED) the same hugepage file, large enough to consume the entire available hugepage pool. If they race instantiating the last page in the mapping, they will both attempt to allocate the last available hugepage. One will fail, of course, returning OOM from the fault and thus causing the process to be killed, despite the fact that the entire mapping can, in fact, be instantiated. The patch fixes this race by the simple method of adding a (sleeping) mutex to serialize the hugepage fault path between allocation and insertion into pagetables and/or page cache. It would be possible to avoid the serialization by catching the allocation failures, waiting on some condition, then rechecking to see if someone else has instantiated the page for us. Given the likely frequency of hugepage instantiations, it seems very doubtful it's worth the extra complexity. This patch causes no regression on the libhugetlbfs testsuite, and one test, which can trigger this race now passes where it previously failed. Actually, the test still sometimes fails, though less often and only as a shmat() failure, rather processes getting OOM killed by the VM. The dodgy heuristic tests in fs/hugetlbfs/inode.c for whether there's enough hugepage space aren't protected by the new mutex, and would be ugly to do so, so there's still a race there. Another patch to replace those tests with something saner for this reason as well as others coming... Signed-off-by: David Gibson <dwg@au1.ibm.com> Cc: William Lee Irwin III <wli@holomorphy.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
parent
79ac6ba40e
commit
3935baa9bc
1 changed files with 18 additions and 7 deletions
25
mm/hugetlb.c
25
mm/hugetlb.c
|
@ -13,6 +13,7 @@
|
||||||
#include <linux/pagemap.h>
|
#include <linux/pagemap.h>
|
||||||
#include <linux/mempolicy.h>
|
#include <linux/mempolicy.h>
|
||||||
#include <linux/cpuset.h>
|
#include <linux/cpuset.h>
|
||||||
|
#include <linux/mutex.h>
|
||||||
|
|
||||||
#include <asm/page.h>
|
#include <asm/page.h>
|
||||||
#include <asm/pgtable.h>
|
#include <asm/pgtable.h>
|
||||||
|
@ -26,6 +27,10 @@ unsigned long max_huge_pages;
|
||||||
static struct list_head hugepage_freelists[MAX_NUMNODES];
|
static struct list_head hugepage_freelists[MAX_NUMNODES];
|
||||||
static unsigned int nr_huge_pages_node[MAX_NUMNODES];
|
static unsigned int nr_huge_pages_node[MAX_NUMNODES];
|
||||||
static unsigned int free_huge_pages_node[MAX_NUMNODES];
|
static unsigned int free_huge_pages_node[MAX_NUMNODES];
|
||||||
|
/*
|
||||||
|
* Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
|
||||||
|
*/
|
||||||
|
static DEFINE_SPINLOCK(hugetlb_lock);
|
||||||
|
|
||||||
static void clear_huge_page(struct page *page, unsigned long addr)
|
static void clear_huge_page(struct page *page, unsigned long addr)
|
||||||
{
|
{
|
||||||
|
@ -50,11 +55,6 @@ static void copy_huge_page(struct page *dst, struct page *src,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
|
|
||||||
*/
|
|
||||||
static DEFINE_SPINLOCK(hugetlb_lock);
|
|
||||||
|
|
||||||
static void enqueue_huge_page(struct page *page)
|
static void enqueue_huge_page(struct page *page)
|
||||||
{
|
{
|
||||||
int nid = page_to_nid(page);
|
int nid = page_to_nid(page);
|
||||||
|
@ -508,14 +508,24 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
pte_t *ptep;
|
pte_t *ptep;
|
||||||
pte_t entry;
|
pte_t entry;
|
||||||
int ret;
|
int ret;
|
||||||
|
static DEFINE_MUTEX(hugetlb_instantiation_mutex);
|
||||||
|
|
||||||
ptep = huge_pte_alloc(mm, address);
|
ptep = huge_pte_alloc(mm, address);
|
||||||
if (!ptep)
|
if (!ptep)
|
||||||
return VM_FAULT_OOM;
|
return VM_FAULT_OOM;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Serialize hugepage allocation and instantiation, so that we don't
|
||||||
|
* get spurious allocation failures if two CPUs race to instantiate
|
||||||
|
* the same page in the page cache.
|
||||||
|
*/
|
||||||
|
mutex_lock(&hugetlb_instantiation_mutex);
|
||||||
entry = *ptep;
|
entry = *ptep;
|
||||||
if (pte_none(entry))
|
if (pte_none(entry)) {
|
||||||
return hugetlb_no_page(mm, vma, address, ptep, write_access);
|
ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
|
||||||
|
mutex_unlock(&hugetlb_instantiation_mutex);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
ret = VM_FAULT_MINOR;
|
ret = VM_FAULT_MINOR;
|
||||||
|
|
||||||
|
@ -525,6 +535,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
if (write_access && !pte_write(entry))
|
if (write_access && !pte_write(entry))
|
||||||
ret = hugetlb_cow(mm, vma, address, ptep, entry);
|
ret = hugetlb_cow(mm, vma, address, ptep, entry);
|
||||||
spin_unlock(&mm->page_table_lock);
|
spin_unlock(&mm->page_table_lock);
|
||||||
|
mutex_unlock(&hugetlb_instantiation_mutex);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue