0697212a41
Implement read/write migration ptes We take the upper two swapfiles for the two types of migration ptes and define a series of macros in swapops.h. The VM is modified to handle the migration entries. migration entries can only be encountered when the page they are pointing to is locked. This limits the number of places one has to fix. We also check in copy_pte_range and in mprotect_pte_range() for migration ptes. We check for migration ptes in do_swap_cache and call a function that will then wait on the page lock. This allows us to effectively stop all accesses to apge. Migration entries are created by try_to_unmap if called for migration and removed by local functions in migrate.c From: Hugh Dickins <hugh@veritas.com> Several times while testing swapless page migration (I've no NUMA, just hacking it up to migrate recklessly while running load), I've hit the BUG_ON(!PageLocked(p)) in migration_entry_to_page. This comes from an orphaned migration entry, unrelated to the current correctly locked migration, but hit by remove_anon_migration_ptes as it checks an address in each vma of the anon_vma list. Such an orphan may be left behind if an earlier migration raced with fork: copy_one_pte can duplicate a migration entry from parent to child, after remove_anon_migration_ptes has checked the child vma, but before it has removed it from the parent vma. (If the process were later to fault on this orphaned entry, it would hit the same BUG from migration_entry_wait.) This could be fixed by locking anon_vma in copy_one_pte, but we'd rather not. There's no such problem with file pages, because vma_prio_tree_add adds child vma after parent vma, and the page table locking at each end is enough to serialize. Follow that example with anon_vma: add new vmas to the tail instead of the head. (There's no corresponding problem when inserting migration entries, because a missed pte will leave the page count and mapcount high, which is allowed for. And there's no corresponding problem when migrating via swap, because a leftover swap entry will be correctly faulted. But the swapless method has no refcounting of its entries.) From: Ingo Molnar <mingo@elte.hu> pte_unmap_unlock() takes the pte pointer as an argument. From: Hugh Dickins <hugh@veritas.com> Several times while testing swapless page migration, gcc has tried to exec a pointer instead of a string: smells like COW mappings are not being properly write-protected on fork. The protection in copy_one_pte looks very convincing, until at last you realize that the second arg to make_migration_entry is a boolean "write", and SWP_MIGRATION_READ is 30. Anyway, it's better done like in change_pte_range, using is_write_migration_entry and make_migration_entry_read. From: Hugh Dickins <hugh@veritas.com> Remove unnecessary obfuscation from sys_swapon's range check on swap type, which blew up causing memory corruption once swapless migration made MAX_SWAPFILES no longer 2 ^ MAX_SWAPFILES_SHIFT. Signed-off-by: Hugh Dickins <hugh@veritas.com> Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> Signed-off-by: Hugh Dickins <hugh@veritas.com> Signed-off-by: Christoph Lameter <clameter@engr.sgi.com> Signed-off-by: Ingo Molnar <mingo@elte.hu> From: Hugh Dickins <hugh@veritas.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
122 lines
3.3 KiB
C
122 lines
3.3 KiB
C
/*
|
|
* swapcache pages are stored in the swapper_space radix tree. We want to
|
|
* get good packing density in that tree, so the index should be dense in
|
|
* the low-order bits.
|
|
*
|
|
* We arrange the `type' and `offset' fields so that `type' is at the five
|
|
* high-order bits of the swp_entry_t and `offset' is right-aligned in the
|
|
* remaining bits.
|
|
*
|
|
* swp_entry_t's are *never* stored anywhere in their arch-dependent format.
|
|
*/
|
|
#define SWP_TYPE_SHIFT(e) (sizeof(e.val) * 8 - MAX_SWAPFILES_SHIFT)
|
|
#define SWP_OFFSET_MASK(e) ((1UL << SWP_TYPE_SHIFT(e)) - 1)
|
|
|
|
/*
|
|
* Store a type+offset into a swp_entry_t in an arch-independent format
|
|
*/
|
|
static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
|
|
{
|
|
swp_entry_t ret;
|
|
|
|
ret.val = (type << SWP_TYPE_SHIFT(ret)) |
|
|
(offset & SWP_OFFSET_MASK(ret));
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Extract the `type' field from a swp_entry_t. The swp_entry_t is in
|
|
* arch-independent format
|
|
*/
|
|
static inline unsigned swp_type(swp_entry_t entry)
|
|
{
|
|
return (entry.val >> SWP_TYPE_SHIFT(entry));
|
|
}
|
|
|
|
/*
|
|
* Extract the `offset' field from a swp_entry_t. The swp_entry_t is in
|
|
* arch-independent format
|
|
*/
|
|
static inline pgoff_t swp_offset(swp_entry_t entry)
|
|
{
|
|
return entry.val & SWP_OFFSET_MASK(entry);
|
|
}
|
|
|
|
/*
|
|
* Convert the arch-dependent pte representation of a swp_entry_t into an
|
|
* arch-independent swp_entry_t.
|
|
*/
|
|
static inline swp_entry_t pte_to_swp_entry(pte_t pte)
|
|
{
|
|
swp_entry_t arch_entry;
|
|
|
|
BUG_ON(pte_file(pte));
|
|
arch_entry = __pte_to_swp_entry(pte);
|
|
return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
|
|
}
|
|
|
|
/*
|
|
* Convert the arch-independent representation of a swp_entry_t into the
|
|
* arch-dependent pte representation.
|
|
*/
|
|
static inline pte_t swp_entry_to_pte(swp_entry_t entry)
|
|
{
|
|
swp_entry_t arch_entry;
|
|
|
|
arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
|
|
BUG_ON(pte_file(__swp_entry_to_pte(arch_entry)));
|
|
return __swp_entry_to_pte(arch_entry);
|
|
}
|
|
|
|
#ifdef CONFIG_MIGRATION
|
|
static inline swp_entry_t make_migration_entry(struct page *page, int write)
|
|
{
|
|
BUG_ON(!PageLocked(page));
|
|
return swp_entry(write ? SWP_MIGRATION_WRITE : SWP_MIGRATION_READ,
|
|
page_to_pfn(page));
|
|
}
|
|
|
|
static inline int is_migration_entry(swp_entry_t entry)
|
|
{
|
|
return unlikely(swp_type(entry) == SWP_MIGRATION_READ ||
|
|
swp_type(entry) == SWP_MIGRATION_WRITE);
|
|
}
|
|
|
|
static inline int is_write_migration_entry(swp_entry_t entry)
|
|
{
|
|
return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE);
|
|
}
|
|
|
|
static inline struct page *migration_entry_to_page(swp_entry_t entry)
|
|
{
|
|
struct page *p = pfn_to_page(swp_offset(entry));
|
|
/*
|
|
* Any use of migration entries may only occur while the
|
|
* corresponding page is locked
|
|
*/
|
|
BUG_ON(!PageLocked(p));
|
|
return p;
|
|
}
|
|
|
|
static inline void make_migration_entry_read(swp_entry_t *entry)
|
|
{
|
|
*entry = swp_entry(SWP_MIGRATION_READ, swp_offset(*entry));
|
|
}
|
|
|
|
extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
|
|
unsigned long address);
|
|
#else
|
|
|
|
#define make_migration_entry(page, write) swp_entry(0, 0)
|
|
#define is_migration_entry(swp) 0
|
|
#define migration_entry_to_page(swp) NULL
|
|
static inline void make_migration_entry_read(swp_entry_t *entryp) { }
|
|
static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
|
|
unsigned long address) { }
|
|
static inline int is_write_migration_entry(swp_entry_t entry)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#endif
|
|
|