fix SMP data race in pagetable setup vs walking
There is a possible data race in the page table walking code. After the split ptlock patches, it actually seems to have been introduced to the core code, but even before that I think it would have impacted some architectures (powerpc and sparc64, at least, walk the page tables without taking locks eg. see find_linux_pte()). The race is as follows: The pte page is allocated, zeroed, and its struct page gets its spinlock initialized. The mm-wide ptl is then taken, and then the pte page is inserted into the pagetables. At this point, the spinlock is not guaranteed to have ordered the previous stores to initialize the pte page with the subsequent store to put it in the page tables. So another Linux page table walker might be walking down (without any locks, because we have split-leaf-ptls), and find that new pte we've inserted. It might try to take the spinlock before the store from the other CPU initializes it. And subsequently it might read a pte_t out before stores from the other CPU have cleared the memory. There are also similar races in higher levels of the page tables. They obviously don't involve the spinlock, but could see uninitialized memory. Arch code and hardware pagetable walkers that walk the pagetables without locks could see similar uninitialized memory problems, regardless of whether split ptes are enabled or not. I prefer to put the barriers in core code, because that's where the higher level logic happens, but the page table accessors are per-arch, and open-coding them everywhere I don't think is an option. I'll put the read-side barriers in alpha arch code for now (other architectures perform data-dependent loads in order). Signed-off-by: Nick Piggin <npiggin@suse.de> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
73f10281ea
commit
362a61ad61
2 changed files with 40 additions and 2 deletions
|
@ -287,17 +287,34 @@ extern inline pte_t pte_mkspecial(pte_t pte) { return pte; }
|
||||||
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
|
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
|
||||||
#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
|
#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The smp_read_barrier_depends() in the following functions are required to
|
||||||
|
* order the load of *dir (the pointer in the top level page table) with any
|
||||||
|
* subsequent load of the returned pmd_t *ret (ret is data dependent on *dir).
|
||||||
|
*
|
||||||
|
* If this ordering is not enforced, the CPU might load an older value of
|
||||||
|
* *ret, which may be uninitialized data. See mm/memory.c:__pte_alloc for
|
||||||
|
* more details.
|
||||||
|
*
|
||||||
|
* Note that we never change the mm->pgd pointer after the task is running, so
|
||||||
|
* pgd_offset does not require such a barrier.
|
||||||
|
*/
|
||||||
|
|
||||||
/* Find an entry in the second-level page table.. */
|
/* Find an entry in the second-level page table.. */
|
||||||
extern inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address)
|
extern inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address)
|
||||||
{
|
{
|
||||||
return (pmd_t *) pgd_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PAGE - 1));
|
pmd_t *ret = (pmd_t *) pgd_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PAGE - 1));
|
||||||
|
smp_read_barrier_depends(); /* see above */
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Find an entry in the third-level page table.. */
|
/* Find an entry in the third-level page table.. */
|
||||||
extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address)
|
extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address)
|
||||||
{
|
{
|
||||||
return (pte_t *) pmd_page_vaddr(*dir)
|
pte_t *ret = (pte_t *) pmd_page_vaddr(*dir)
|
||||||
+ ((address >> PAGE_SHIFT) & (PTRS_PER_PAGE - 1));
|
+ ((address >> PAGE_SHIFT) & (PTRS_PER_PAGE - 1));
|
||||||
|
smp_read_barrier_depends(); /* see above */
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
#define pte_offset_map(dir,addr) pte_offset_kernel((dir),(addr))
|
#define pte_offset_map(dir,addr) pte_offset_kernel((dir),(addr))
|
||||||
|
|
21
mm/memory.c
21
mm/memory.c
|
@ -311,6 +311,21 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
|
||||||
if (!new)
|
if (!new)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Ensure all pte setup (eg. pte page lock and page clearing) are
|
||||||
|
* visible before the pte is made visible to other CPUs by being
|
||||||
|
* put into page tables.
|
||||||
|
*
|
||||||
|
* The other side of the story is the pointer chasing in the page
|
||||||
|
* table walking code (when walking the page table without locking;
|
||||||
|
* ie. most of the time). Fortunately, these data accesses consist
|
||||||
|
* of a chain of data-dependent loads, meaning most CPUs (alpha
|
||||||
|
* being the notable exception) will already guarantee loads are
|
||||||
|
* seen in-order. See the alpha page table accessors for the
|
||||||
|
* smp_read_barrier_depends() barriers in page table walking code.
|
||||||
|
*/
|
||||||
|
smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
|
||||||
|
|
||||||
spin_lock(&mm->page_table_lock);
|
spin_lock(&mm->page_table_lock);
|
||||||
if (!pmd_present(*pmd)) { /* Has another populated it ? */
|
if (!pmd_present(*pmd)) { /* Has another populated it ? */
|
||||||
mm->nr_ptes++;
|
mm->nr_ptes++;
|
||||||
|
@ -329,6 +344,8 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
|
||||||
if (!new)
|
if (!new)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
||||||
|
smp_wmb(); /* See comment in __pte_alloc */
|
||||||
|
|
||||||
spin_lock(&init_mm.page_table_lock);
|
spin_lock(&init_mm.page_table_lock);
|
||||||
if (!pmd_present(*pmd)) { /* Has another populated it ? */
|
if (!pmd_present(*pmd)) { /* Has another populated it ? */
|
||||||
pmd_populate_kernel(&init_mm, pmd, new);
|
pmd_populate_kernel(&init_mm, pmd, new);
|
||||||
|
@ -2619,6 +2636,8 @@ int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
|
||||||
if (!new)
|
if (!new)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
||||||
|
smp_wmb(); /* See comment in __pte_alloc */
|
||||||
|
|
||||||
spin_lock(&mm->page_table_lock);
|
spin_lock(&mm->page_table_lock);
|
||||||
if (pgd_present(*pgd)) /* Another has populated it */
|
if (pgd_present(*pgd)) /* Another has populated it */
|
||||||
pud_free(mm, new);
|
pud_free(mm, new);
|
||||||
|
@ -2640,6 +2659,8 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
|
||||||
if (!new)
|
if (!new)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
||||||
|
smp_wmb(); /* See comment in __pte_alloc */
|
||||||
|
|
||||||
spin_lock(&mm->page_table_lock);
|
spin_lock(&mm->page_table_lock);
|
||||||
#ifndef __ARCH_HAS_4LEVEL_HACK
|
#ifndef __ARCH_HAS_4LEVEL_HACK
|
||||||
if (pud_present(*pud)) /* Another has populated it */
|
if (pud_present(*pud)) /* Another has populated it */
|
||||||
|
|
Loading…
Reference in a new issue