5311ab62cd
Normally when running in PAE mode, the 4th PMD maps the kernel address space, which can be shared among all processes (since they all need the same kernel mappings). Xen, however, does not allow guests to have the kernel pmd shared between page tables, so parameterize pgtable.c to allow both modes of operation. There are several side-effects of this. One is that vmalloc will update the kernel address space mappings, and those updates need to be propagated into all processes if the kernel mappings are not intrinsically shared. In the non-PAE case, this is done by maintaining a pgd_list of all processes; this list is used when all process pagetables must be updated. pgd_list is threaded via otherwise unused entries in the page structure for the pgd, which means that the pgd must be page-sized for this to work. Normally the PAE pgd is only 4x64 byte entries large, but Xen requires the PAE pgd to page aligned anyway, so this patch forces the pgd to be page aligned+sized when the kernel pmd is unshared, to accomodate both these requirements. Also, since there may be several distinct kernel pmds (if the user/kernel split is below 3G), there's no point in allocating them from a slab cache; they're just allocated with get_free_page and initialized appropriately. (Of course the could be cached if there is just a single kernel pmd - which is the default with a 3G user/kernel split - but it doesn't seem worthwhile to add yet another case into this code). [ Many thanks to wli for review comments. ] Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com> Signed-off-by: William Lee Irwin III <wli@holomorphy.com> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Zachary Amsden <zach@vmware.com> Cc: Christoph Lameter <clameter@sgi.com> Acked-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
365 lines
9.4 KiB
C
365 lines
9.4 KiB
C
/*
|
|
* linux/arch/i386/mm/pgtable.c
|
|
*/
|
|
|
|
#include <linux/sched.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/module.h>
|
|
|
|
#include <asm/system.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/fixmap.h>
|
|
#include <asm/e820.h>
|
|
#include <asm/tlb.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
void show_mem(void)
|
|
{
|
|
int total = 0, reserved = 0;
|
|
int shared = 0, cached = 0;
|
|
int highmem = 0;
|
|
struct page *page;
|
|
pg_data_t *pgdat;
|
|
unsigned long i;
|
|
unsigned long flags;
|
|
|
|
printk(KERN_INFO "Mem-info:\n");
|
|
show_free_areas();
|
|
printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
|
|
for_each_online_pgdat(pgdat) {
|
|
pgdat_resize_lock(pgdat, &flags);
|
|
for (i = 0; i < pgdat->node_spanned_pages; ++i) {
|
|
page = pgdat_page_nr(pgdat, i);
|
|
total++;
|
|
if (PageHighMem(page))
|
|
highmem++;
|
|
if (PageReserved(page))
|
|
reserved++;
|
|
else if (PageSwapCache(page))
|
|
cached++;
|
|
else if (page_count(page))
|
|
shared += page_count(page) - 1;
|
|
}
|
|
pgdat_resize_unlock(pgdat, &flags);
|
|
}
|
|
printk(KERN_INFO "%d pages of RAM\n", total);
|
|
printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
|
|
printk(KERN_INFO "%d reserved pages\n", reserved);
|
|
printk(KERN_INFO "%d pages shared\n", shared);
|
|
printk(KERN_INFO "%d pages swap cached\n", cached);
|
|
|
|
printk(KERN_INFO "%lu pages dirty\n", global_page_state(NR_FILE_DIRTY));
|
|
printk(KERN_INFO "%lu pages writeback\n",
|
|
global_page_state(NR_WRITEBACK));
|
|
printk(KERN_INFO "%lu pages mapped\n", global_page_state(NR_FILE_MAPPED));
|
|
printk(KERN_INFO "%lu pages slab\n",
|
|
global_page_state(NR_SLAB_RECLAIMABLE) +
|
|
global_page_state(NR_SLAB_UNRECLAIMABLE));
|
|
printk(KERN_INFO "%lu pages pagetables\n",
|
|
global_page_state(NR_PAGETABLE));
|
|
}
|
|
|
|
/*
|
|
* Associate a virtual page frame with a given physical page frame
|
|
* and protection flags for that frame.
|
|
*/
|
|
static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
|
|
{
|
|
pgd_t *pgd;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *pte;
|
|
|
|
pgd = swapper_pg_dir + pgd_index(vaddr);
|
|
if (pgd_none(*pgd)) {
|
|
BUG();
|
|
return;
|
|
}
|
|
pud = pud_offset(pgd, vaddr);
|
|
if (pud_none(*pud)) {
|
|
BUG();
|
|
return;
|
|
}
|
|
pmd = pmd_offset(pud, vaddr);
|
|
if (pmd_none(*pmd)) {
|
|
BUG();
|
|
return;
|
|
}
|
|
pte = pte_offset_kernel(pmd, vaddr);
|
|
if (pgprot_val(flags))
|
|
/* <pfn,flags> stored as-is, to permit clearing entries */
|
|
set_pte(pte, pfn_pte(pfn, flags));
|
|
else
|
|
pte_clear(&init_mm, vaddr, pte);
|
|
|
|
/*
|
|
* It's enough to flush this one mapping.
|
|
* (PGE mappings get flushed as well)
|
|
*/
|
|
__flush_tlb_one(vaddr);
|
|
}
|
|
|
|
/*
|
|
* Associate a large virtual page frame with a given physical page frame
|
|
* and protection flags for that frame. pfn is for the base of the page,
|
|
* vaddr is what the page gets mapped to - both must be properly aligned.
|
|
* The pmd must already be instantiated. Assumes PAE mode.
|
|
*/
|
|
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
|
|
{
|
|
pgd_t *pgd;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
|
|
if (vaddr & (PMD_SIZE-1)) { /* vaddr is misaligned */
|
|
printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
|
|
return; /* BUG(); */
|
|
}
|
|
if (pfn & (PTRS_PER_PTE-1)) { /* pfn is misaligned */
|
|
printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
|
|
return; /* BUG(); */
|
|
}
|
|
pgd = swapper_pg_dir + pgd_index(vaddr);
|
|
if (pgd_none(*pgd)) {
|
|
printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
|
|
return; /* BUG(); */
|
|
}
|
|
pud = pud_offset(pgd, vaddr);
|
|
pmd = pmd_offset(pud, vaddr);
|
|
set_pmd(pmd, pfn_pmd(pfn, flags));
|
|
/*
|
|
* It's enough to flush this one mapping.
|
|
* (PGE mappings get flushed as well)
|
|
*/
|
|
__flush_tlb_one(vaddr);
|
|
}
|
|
|
|
static int fixmaps;
|
|
unsigned long __FIXADDR_TOP = 0xfffff000;
|
|
EXPORT_SYMBOL(__FIXADDR_TOP);
|
|
|
|
void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
|
|
{
|
|
unsigned long address = __fix_to_virt(idx);
|
|
|
|
if (idx >= __end_of_fixed_addresses) {
|
|
BUG();
|
|
return;
|
|
}
|
|
set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
|
|
fixmaps++;
|
|
}
|
|
|
|
/**
|
|
* reserve_top_address - reserves a hole in the top of kernel address space
|
|
* @reserve - size of hole to reserve
|
|
*
|
|
* Can be used to relocate the fixmap area and poke a hole in the top
|
|
* of kernel address space to make room for a hypervisor.
|
|
*/
|
|
void reserve_top_address(unsigned long reserve)
|
|
{
|
|
BUG_ON(fixmaps > 0);
|
|
printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
|
|
(int)-reserve);
|
|
__FIXADDR_TOP = -reserve - PAGE_SIZE;
|
|
__VMALLOC_RESERVE += reserve;
|
|
}
|
|
|
|
pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
|
|
{
|
|
return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
|
|
}
|
|
|
|
struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
|
|
{
|
|
struct page *pte;
|
|
|
|
#ifdef CONFIG_HIGHPTE
|
|
pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT|__GFP_ZERO, 0);
|
|
#else
|
|
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
|
|
#endif
|
|
return pte;
|
|
}
|
|
|
|
void pmd_ctor(void *pmd, struct kmem_cache *cache, unsigned long flags)
|
|
{
|
|
memset(pmd, 0, PTRS_PER_PMD*sizeof(pmd_t));
|
|
}
|
|
|
|
/*
|
|
* List of all pgd's needed for non-PAE so it can invalidate entries
|
|
* in both cached and uncached pgd's; not needed for PAE since the
|
|
* kernel pmd is shared. If PAE were not to share the pmd a similar
|
|
* tactic would be needed. This is essentially codepath-based locking
|
|
* against pageattr.c; it is the unique case in which a valid change
|
|
* of kernel pagetables can't be lazily synchronized by vmalloc faults.
|
|
* vmalloc faults work because attached pagetables are never freed.
|
|
* The locking scheme was chosen on the basis of manfred's
|
|
* recommendations and having no core impact whatsoever.
|
|
* -- wli
|
|
*/
|
|
DEFINE_SPINLOCK(pgd_lock);
|
|
struct page *pgd_list;
|
|
|
|
static inline void pgd_list_add(pgd_t *pgd)
|
|
{
|
|
struct page *page = virt_to_page(pgd);
|
|
page->index = (unsigned long)pgd_list;
|
|
if (pgd_list)
|
|
set_page_private(pgd_list, (unsigned long)&page->index);
|
|
pgd_list = page;
|
|
set_page_private(page, (unsigned long)&pgd_list);
|
|
}
|
|
|
|
static inline void pgd_list_del(pgd_t *pgd)
|
|
{
|
|
struct page *next, **pprev, *page = virt_to_page(pgd);
|
|
next = (struct page *)page->index;
|
|
pprev = (struct page **)page_private(page);
|
|
*pprev = next;
|
|
if (next)
|
|
set_page_private(next, (unsigned long)pprev);
|
|
}
|
|
|
|
#if (PTRS_PER_PMD == 1)
|
|
/* Non-PAE pgd constructor */
|
|
void pgd_ctor(void *pgd, struct kmem_cache *cache, unsigned long unused)
|
|
{
|
|
unsigned long flags;
|
|
|
|
/* !PAE, no pagetable sharing */
|
|
memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
|
|
|
|
spin_lock_irqsave(&pgd_lock, flags);
|
|
|
|
/* must happen under lock */
|
|
clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
|
|
swapper_pg_dir + USER_PTRS_PER_PGD,
|
|
KERNEL_PGD_PTRS);
|
|
paravirt_alloc_pd_clone(__pa(pgd) >> PAGE_SHIFT,
|
|
__pa(swapper_pg_dir) >> PAGE_SHIFT,
|
|
USER_PTRS_PER_PGD,
|
|
KERNEL_PGD_PTRS);
|
|
pgd_list_add(pgd);
|
|
spin_unlock_irqrestore(&pgd_lock, flags);
|
|
}
|
|
#else /* PTRS_PER_PMD > 1 */
|
|
/* PAE pgd constructor */
|
|
void pgd_ctor(void *pgd, struct kmem_cache *cache, unsigned long unused)
|
|
{
|
|
/* PAE, kernel PMD may be shared */
|
|
|
|
if (SHARED_KERNEL_PMD) {
|
|
clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
|
|
swapper_pg_dir + USER_PTRS_PER_PGD,
|
|
KERNEL_PGD_PTRS);
|
|
} else {
|
|
unsigned long flags;
|
|
|
|
memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
|
|
spin_lock_irqsave(&pgd_lock, flags);
|
|
pgd_list_add(pgd);
|
|
spin_unlock_irqrestore(&pgd_lock, flags);
|
|
}
|
|
}
|
|
#endif /* PTRS_PER_PMD */
|
|
|
|
void pgd_dtor(void *pgd, struct kmem_cache *cache, unsigned long unused)
|
|
{
|
|
unsigned long flags; /* can be called from interrupt context */
|
|
|
|
BUG_ON(SHARED_KERNEL_PMD);
|
|
|
|
paravirt_release_pd(__pa(pgd) >> PAGE_SHIFT);
|
|
spin_lock_irqsave(&pgd_lock, flags);
|
|
pgd_list_del(pgd);
|
|
spin_unlock_irqrestore(&pgd_lock, flags);
|
|
}
|
|
|
|
#define UNSHARED_PTRS_PER_PGD \
|
|
(SHARED_KERNEL_PMD ? USER_PTRS_PER_PGD : PTRS_PER_PGD)
|
|
|
|
/* If we allocate a pmd for part of the kernel address space, then
|
|
make sure its initialized with the appropriate kernel mappings.
|
|
Otherwise use a cached zeroed pmd. */
|
|
static pmd_t *pmd_cache_alloc(int idx)
|
|
{
|
|
pmd_t *pmd;
|
|
|
|
if (idx >= USER_PTRS_PER_PGD) {
|
|
pmd = (pmd_t *)__get_free_page(GFP_KERNEL);
|
|
|
|
if (pmd)
|
|
memcpy(pmd,
|
|
(void *)pgd_page_vaddr(swapper_pg_dir[idx]),
|
|
sizeof(pmd_t) * PTRS_PER_PMD);
|
|
} else
|
|
pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
|
|
|
|
return pmd;
|
|
}
|
|
|
|
static void pmd_cache_free(pmd_t *pmd, int idx)
|
|
{
|
|
if (idx >= USER_PTRS_PER_PGD)
|
|
free_page((unsigned long)pmd);
|
|
else
|
|
kmem_cache_free(pmd_cache, pmd);
|
|
}
|
|
|
|
pgd_t *pgd_alloc(struct mm_struct *mm)
|
|
{
|
|
int i;
|
|
pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
|
|
|
|
if (PTRS_PER_PMD == 1 || !pgd)
|
|
return pgd;
|
|
|
|
for (i = 0; i < UNSHARED_PTRS_PER_PGD; ++i) {
|
|
pmd_t *pmd = pmd_cache_alloc(i);
|
|
|
|
if (!pmd)
|
|
goto out_oom;
|
|
|
|
paravirt_alloc_pd(__pa(pmd) >> PAGE_SHIFT);
|
|
set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
|
|
}
|
|
return pgd;
|
|
|
|
out_oom:
|
|
for (i--; i >= 0; i--) {
|
|
pgd_t pgdent = pgd[i];
|
|
void* pmd = (void *)__va(pgd_val(pgdent)-1);
|
|
paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
|
|
pmd_cache_free(pmd, i);
|
|
}
|
|
kmem_cache_free(pgd_cache, pgd);
|
|
return NULL;
|
|
}
|
|
|
|
void pgd_free(pgd_t *pgd)
|
|
{
|
|
int i;
|
|
|
|
/* in the PAE case user pgd entries are overwritten before usage */
|
|
if (PTRS_PER_PMD > 1)
|
|
for (i = 0; i < UNSHARED_PTRS_PER_PGD; ++i) {
|
|
pgd_t pgdent = pgd[i];
|
|
void* pmd = (void *)__va(pgd_val(pgdent)-1);
|
|
paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
|
|
pmd_cache_free(pmd, i);
|
|
}
|
|
/* in the non-PAE case, free_pgtables() clears user pgd entries */
|
|
kmem_cache_free(pgd_cache, pgd);
|
|
}
|