2005-04-16 16:20:36 -06:00
|
|
|
#ifndef _I386_PGTABLE_3LEVEL_H
|
|
|
|
|
#define _I386_PGTABLE_3LEVEL_H
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Intel Physical Address Extension (PAE) Mode - three-level page
|
|
|
|
|
* tables on PPro+ CPUs.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define pte_ERROR(e) \
|
|
|
|
|
printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
|
|
|
|
|
#define pmd_ERROR(e) \
|
|
|
|
|
printk("%s:%d: bad pmd %p(%016Lx).\n", __FILE__, __LINE__, &(e), pmd_val(e))
|
|
|
|
|
#define pgd_ERROR(e) \
|
|
|
|
|
printk("%s:%d: bad pgd %p(%016Lx).\n", __FILE__, __LINE__, &(e), pgd_val(e))
|
|
|
|
|
|
|
|
|
|
#define pud_none(pud) 0
|
|
|
|
|
#define pud_bad(pud) 0
|
|
|
|
|
#define pud_present(pud) 1
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* All present pages with !NX bit are kernel-executable:
|
|
|
|
|
*/
|
|
|
|
|
static inline int pte_exec_kernel(pte_t pte)
|
|
|
|
|
{
|
2007-07-16 00:38:17 -06:00
|
|
|
return !(pte_val(pte) & _PAGE_NX);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Rules for using set_pte: the pte being assigned *must* be
|
|
|
|
|
* either not present or in a state where the hardware will
|
|
|
|
|
* not attempt to update the pte. In places where this is
|
|
|
|
|
* not possible, use pte_get_and_clear to obtain the old pte
|
|
|
|
|
* value and then use set_pte to update it. -ben
|
|
|
|
|
*/
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_set_pte(pte_t *ptep, pte_t pte)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
|
ptep->pte_high = pte.pte_high;
|
|
|
|
|
smp_wmb();
|
|
|
|
|
ptep->pte_low = pte.pte_low;
|
|
|
|
|
}
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pte_t *ptep , pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
native_set_pte(ptep, pte);
|
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-10-01 00:29:36 -06:00
|
|
|
/*
|
|
|
|
|
* Since this is only called on user PTEs, and the page fault handler
|
|
|
|
|
* must handle the already racy situation of simultaneous page faults,
|
|
|
|
|
* we are justified in merely clearing the PTE present bit, followed
|
|
|
|
|
* by a set. The ordering here is important.
|
|
|
|
|
*/
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pte_t *ptep, pte_t pte)
|
2006-10-01 00:29:36 -06:00
|
|
|
{
|
|
|
|
|
ptep->pte_low = 0;
|
|
|
|
|
smp_wmb();
|
|
|
|
|
ptep->pte_high = pte.pte_high;
|
|
|
|
|
smp_wmb();
|
|
|
|
|
ptep->pte_low = pte.pte_low;
|
|
|
|
|
}
|
|
|
|
|
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
set_64bit((unsigned long long *)(ptep),native_pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
set_64bit((unsigned long long *)(pmdp),native_pmd_val(pmd));
|
|
|
|
|
}
|
|
|
|
|
static inline void native_set_pud(pud_t *pudp, pud_t pud)
|
|
|
|
|
{
|
|
|
|
|
*pudp = pud;
|
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
[PATCH] x86/PAE: Fix pte_clear for the >4GB RAM case
Proposed fix for ptep_get_and_clear_full PAE bug. Pte_clear had the same bug,
so use the same fix for both. Turns out pmd_clear had it as well, but pgds
are not affected.
The problem is rather intricate. Page table entries in PAE mode are 64-bits
wide, but the only atomic 8-byte write operation available in 32-bit mode is
cmpxchg8b, which is expensive (at least on P4), and thus avoided. But it can
happen that the processor may prefetch entries into the TLB in the middle of an
operation which clears a page table entry. So one must always clear the P-bit
in the low word of the page table entry first when clearing it.
Since the sequence *ptep = __pte(0) leaves the order of the write dependent on
the compiler, it must be coded explicitly as a clear of the low word followed
by a clear of the high word. Further, there must be a write memory barrier
here to enforce proper ordering by the compiler (and, in the future, by the
processor as well).
On > 4GB memory machines, the implementation of pte_clear for PAE was clearly
deficient, as it could leave virtual mappings of physical memory above 4GB
aliased to memory below 4GB in the TLB. The implementation of
ptep_get_and_clear_full has a similar bug, although not nearly as likely to
occur, since the mappings being cleared are in the process of being destroyed,
and should never be dereferenced again.
But, as luck would have it, it is possible to trigger bugs even without ever
dereferencing these bogus TLB mappings, even if the clear is followed fairly
soon after with a TLB flush or invalidation. The problem is that memory above
4GB may now be aliased into the first 4GB of memory, and in fact, may hit a
region of memory with non-memory semantics. These regions include AGP and PCI
space. As such, these memory regions are not cached by the processor. This
introduces the bug.
The processor can speculate memory operations, including memory writes, as long
as they are committed with the proper ordering. Speculating a memory write to
a linear address that has a bogus TLB mapping is possible. Normally, the
speculation is harmless. But for cached memory, it does leave the falsely
speculated cacheline unmodified, but in a dirty state. This cache line will be
eventually written back. If this cacheline happens to intersect a region of
memory that is not protected by the cache coherency protocol, it can corrupt
data in I/O memory, which is generally a very bad thing to do, and can cause
total system failure or just plain undefined behavior.
These bugs are extremely unlikely, but the severity is of such magnitude, and
the fix so simple that I think fixing them immediately is justified. Also,
they are nearly impossible to debug.
Signed-off-by: Zachary Amsden <zach@vmware.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-27 12:32:29 -06:00
|
|
|
/*
|
|
|
|
|
* For PTEs and PDEs, we must clear the P-bit first when clearing a page table
|
|
|
|
|
* entry, so clear the bottom half first and enforce ordering with a compiler
|
|
|
|
|
* barrier.
|
|
|
|
|
*/
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
|
[PATCH] x86/PAE: Fix pte_clear for the >4GB RAM case
Proposed fix for ptep_get_and_clear_full PAE bug. Pte_clear had the same bug,
so use the same fix for both. Turns out pmd_clear had it as well, but pgds
are not affected.
The problem is rather intricate. Page table entries in PAE mode are 64-bits
wide, but the only atomic 8-byte write operation available in 32-bit mode is
cmpxchg8b, which is expensive (at least on P4), and thus avoided. But it can
happen that the processor may prefetch entries into the TLB in the middle of an
operation which clears a page table entry. So one must always clear the P-bit
in the low word of the page table entry first when clearing it.
Since the sequence *ptep = __pte(0) leaves the order of the write dependent on
the compiler, it must be coded explicitly as a clear of the low word followed
by a clear of the high word. Further, there must be a write memory barrier
here to enforce proper ordering by the compiler (and, in the future, by the
processor as well).
On > 4GB memory machines, the implementation of pte_clear for PAE was clearly
deficient, as it could leave virtual mappings of physical memory above 4GB
aliased to memory below 4GB in the TLB. The implementation of
ptep_get_and_clear_full has a similar bug, although not nearly as likely to
occur, since the mappings being cleared are in the process of being destroyed,
and should never be dereferenced again.
But, as luck would have it, it is possible to trigger bugs even without ever
dereferencing these bogus TLB mappings, even if the clear is followed fairly
soon after with a TLB flush or invalidation. The problem is that memory above
4GB may now be aliased into the first 4GB of memory, and in fact, may hit a
region of memory with non-memory semantics. These regions include AGP and PCI
space. As such, these memory regions are not cached by the processor. This
introduces the bug.
The processor can speculate memory operations, including memory writes, as long
as they are committed with the proper ordering. Speculating a memory write to
a linear address that has a bogus TLB mapping is possible. Normally, the
speculation is harmless. But for cached memory, it does leave the falsely
speculated cacheline unmodified, but in a dirty state. This cache line will be
eventually written back. If this cacheline happens to intersect a region of
memory that is not protected by the cache coherency protocol, it can corrupt
data in I/O memory, which is generally a very bad thing to do, and can cause
total system failure or just plain undefined behavior.
These bugs are extremely unlikely, but the severity is of such magnitude, and
the fix so simple that I think fixing them immediately is justified. Also,
they are nearly impossible to debug.
Signed-off-by: Zachary Amsden <zach@vmware.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-27 12:32:29 -06:00
|
|
|
{
|
|
|
|
|
ptep->pte_low = 0;
|
|
|
|
|
smp_wmb();
|
|
|
|
|
ptep->pte_high = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline void native_pmd_clear(pmd_t *pmd)
|
[PATCH] x86/PAE: Fix pte_clear for the >4GB RAM case
Proposed fix for ptep_get_and_clear_full PAE bug. Pte_clear had the same bug,
so use the same fix for both. Turns out pmd_clear had it as well, but pgds
are not affected.
The problem is rather intricate. Page table entries in PAE mode are 64-bits
wide, but the only atomic 8-byte write operation available in 32-bit mode is
cmpxchg8b, which is expensive (at least on P4), and thus avoided. But it can
happen that the processor may prefetch entries into the TLB in the middle of an
operation which clears a page table entry. So one must always clear the P-bit
in the low word of the page table entry first when clearing it.
Since the sequence *ptep = __pte(0) leaves the order of the write dependent on
the compiler, it must be coded explicitly as a clear of the low word followed
by a clear of the high word. Further, there must be a write memory barrier
here to enforce proper ordering by the compiler (and, in the future, by the
processor as well).
On > 4GB memory machines, the implementation of pte_clear for PAE was clearly
deficient, as it could leave virtual mappings of physical memory above 4GB
aliased to memory below 4GB in the TLB. The implementation of
ptep_get_and_clear_full has a similar bug, although not nearly as likely to
occur, since the mappings being cleared are in the process of being destroyed,
and should never be dereferenced again.
But, as luck would have it, it is possible to trigger bugs even without ever
dereferencing these bogus TLB mappings, even if the clear is followed fairly
soon after with a TLB flush or invalidation. The problem is that memory above
4GB may now be aliased into the first 4GB of memory, and in fact, may hit a
region of memory with non-memory semantics. These regions include AGP and PCI
space. As such, these memory regions are not cached by the processor. This
introduces the bug.
The processor can speculate memory operations, including memory writes, as long
as they are committed with the proper ordering. Speculating a memory write to
a linear address that has a bogus TLB mapping is possible. Normally, the
speculation is harmless. But for cached memory, it does leave the falsely
speculated cacheline unmodified, but in a dirty state. This cache line will be
eventually written back. If this cacheline happens to intersect a region of
memory that is not protected by the cache coherency protocol, it can corrupt
data in I/O memory, which is generally a very bad thing to do, and can cause
total system failure or just plain undefined behavior.
These bugs are extremely unlikely, but the severity is of such magnitude, and
the fix so simple that I think fixing them immediately is justified. Also,
they are nearly impossible to debug.
Signed-off-by: Zachary Amsden <zach@vmware.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-27 12:32:29 -06:00
|
|
|
{
|
|
|
|
|
u32 *tmp = (u32 *)pmd;
|
|
|
|
|
*tmp = 0;
|
|
|
|
|
smp_wmb();
|
|
|
|
|
*(tmp + 1) = 0;
|
|
|
|
|
}
|
2007-05-02 11:27:13 -06:00
|
|
|
|
|
|
|
|
#ifndef CONFIG_PARAVIRT
|
|
|
|
|
#define set_pte(ptep, pte) native_set_pte(ptep, pte)
|
|
|
|
|
#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
|
|
|
|
|
#define set_pte_present(mm, addr, ptep, pte) native_set_pte_present(mm, addr, ptep, pte)
|
|
|
|
|
#define set_pte_atomic(ptep, pte) native_set_pte_atomic(ptep, pte)
|
|
|
|
|
#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
|
|
|
|
|
#define set_pud(pudp, pud) native_set_pud(pudp, pud)
|
|
|
|
|
#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
|
|
|
|
|
#define pmd_clear(pmd) native_pmd_clear(pmd)
|
2006-12-06 18:14:08 -07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Pentium-II erratum A13: in PAE mode we explicitly have to flush
|
|
|
|
|
* the TLB via cr3 if the top-level pgd is changed...
|
|
|
|
|
* We do not let the generic code free and clear pgd entries due to
|
|
|
|
|
* this erratum.
|
|
|
|
|
*/
|
|
|
|
|
static inline void pud_clear (pud_t * pud) { }
|
|
|
|
|
|
|
|
|
|
#define pud_page(pud) \
|
|
|
|
|
((struct page *) __va(pud_val(pud) & PAGE_MASK))
|
|
|
|
|
|
|
|
|
|
#define pud_page_vaddr(pud) \
|
|
|
|
|
((unsigned long) __va(pud_val(pud) & PAGE_MASK))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* Find an entry in the second-level page table.. */
|
|
|
|
|
#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \
|
|
|
|
|
pmd_index(address))
|
[PATCH] x86/PAE: Fix pte_clear for the >4GB RAM case
Proposed fix for ptep_get_and_clear_full PAE bug. Pte_clear had the same bug,
so use the same fix for both. Turns out pmd_clear had it as well, but pgds
are not affected.
The problem is rather intricate. Page table entries in PAE mode are 64-bits
wide, but the only atomic 8-byte write operation available in 32-bit mode is
cmpxchg8b, which is expensive (at least on P4), and thus avoided. But it can
happen that the processor may prefetch entries into the TLB in the middle of an
operation which clears a page table entry. So one must always clear the P-bit
in the low word of the page table entry first when clearing it.
Since the sequence *ptep = __pte(0) leaves the order of the write dependent on
the compiler, it must be coded explicitly as a clear of the low word followed
by a clear of the high word. Further, there must be a write memory barrier
here to enforce proper ordering by the compiler (and, in the future, by the
processor as well).
On > 4GB memory machines, the implementation of pte_clear for PAE was clearly
deficient, as it could leave virtual mappings of physical memory above 4GB
aliased to memory below 4GB in the TLB. The implementation of
ptep_get_and_clear_full has a similar bug, although not nearly as likely to
occur, since the mappings being cleared are in the process of being destroyed,
and should never be dereferenced again.
But, as luck would have it, it is possible to trigger bugs even without ever
dereferencing these bogus TLB mappings, even if the clear is followed fairly
soon after with a TLB flush or invalidation. The problem is that memory above
4GB may now be aliased into the first 4GB of memory, and in fact, may hit a
region of memory with non-memory semantics. These regions include AGP and PCI
space. As such, these memory regions are not cached by the processor. This
introduces the bug.
The processor can speculate memory operations, including memory writes, as long
as they are committed with the proper ordering. Speculating a memory write to
a linear address that has a bogus TLB mapping is possible. Normally, the
speculation is harmless. But for cached memory, it does leave the falsely
speculated cacheline unmodified, but in a dirty state. This cache line will be
eventually written back. If this cacheline happens to intersect a region of
memory that is not protected by the cache coherency protocol, it can corrupt
data in I/O memory, which is generally a very bad thing to do, and can cause
total system failure or just plain undefined behavior.
These bugs are extremely unlikely, but the severity is of such magnitude, and
the fix so simple that I think fixing them immediately is justified. Also,
they are nearly impossible to debug.
Signed-off-by: Zachary Amsden <zach@vmware.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-27 12:32:29 -06:00
|
|
|
|
2007-05-02 11:27:19 -06:00
|
|
|
#ifdef CONFIG_SMP
|
2007-05-02 11:27:13 -06:00
|
|
|
static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
|
pte_t res;
|
|
|
|
|
|
|
|
|
|
/* xchg acts as a barrier before the setting of the high bits */
|
|
|
|
|
res.pte_low = xchg(&ptep->pte_low, 0);
|
|
|
|
|
res.pte_high = ptep->pte_high;
|
|
|
|
|
ptep->pte_high = 0;
|
|
|
|
|
|
|
|
|
|
return res;
|
|
|
|
|
}
|
2007-05-02 11:27:19 -06:00
|
|
|
#else
|
|
|
|
|
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
|
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-09-26 00:32:30 -06:00
|
|
|
#define __HAVE_ARCH_PTE_SAME
|
2005-04-16 16:20:36 -06:00
|
|
|
static inline int pte_same(pte_t a, pte_t b)
|
|
|
|
|
{
|
|
|
|
|
return a.pte_low == b.pte_low && a.pte_high == b.pte_high;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define pte_page(x) pfn_to_page(pte_pfn(x))
|
|
|
|
|
|
|
|
|
|
static inline int pte_none(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
return !pte.pte_low && !pte.pte_high;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_pfn(pte_t pte)
|
|
|
|
|
{
|
2007-05-02 11:27:13 -06:00
|
|
|
return pte_val(pte) >> PAGE_SHIFT;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extern unsigned long long __supported_pte_mask;
|
|
|
|
|
|
|
|
|
|
static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
|
|
|
|
|
{
|
2007-05-02 11:27:13 -06:00
|
|
|
return __pte((((unsigned long long)page_nr << PAGE_SHIFT) |
|
|
|
|
|
pgprot_val(pgprot)) & __supported_pte_mask);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
|
|
|
|
|
{
|
2007-05-02 11:27:13 -06:00
|
|
|
return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) |
|
|
|
|
|
pgprot_val(pgprot)) & __supported_pte_mask);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Bits 0, 6 and 7 are taken in the low part of the pte,
|
|
|
|
|
* put the 32 bits of offset into the high part.
|
|
|
|
|
*/
|
|
|
|
|
#define pte_to_pgoff(pte) ((pte).pte_high)
|
|
|
|
|
#define pgoff_to_pte(off) ((pte_t) { _PAGE_FILE, (off) })
|
|
|
|
|
#define PTE_FILE_MAX_BITS 32
|
|
|
|
|
|
|
|
|
|
/* Encode and de-code a swap entry */
|
|
|
|
|
#define __swp_type(x) (((x).val) & 0x1f)
|
|
|
|
|
#define __swp_offset(x) ((x).val >> 5)
|
|
|
|
|
#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5})
|
|
|
|
|
#define __pte_to_swp_entry(pte) ((swp_entry_t){ (pte).pte_high })
|
|
|
|
|
#define __swp_entry_to_pte(x) ((pte_t){ 0, (x).val })
|
|
|
|
|
|
|
|
|
|
#define __pmd_free_tlb(tlb, x) do { } while (0)
|
|
|
|
|
|
|
|
|
|
#endif /* _I386_PGTABLE_3LEVEL_H */
|
