Techwizz/kernel-fxtec-pro1x
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

mm: compaction: introduce sync-light migration for use by compaction

Browse source 
This patch adds a lightweight sync migrate operation MIGRATE_SYNC_LIGHT
mode that avoids writing back pages to backing storage.  Async compaction
maps to MIGRATE_ASYNC while sync compaction maps to MIGRATE_SYNC_LIGHT.
For other migrate_pages users such as memory hotplug, MIGRATE_SYNC is
used.

This avoids sync compaction stalling for an excessive length of time,
particularly when copying files to a USB stick where there might be a
large number of dirty pages backed by a filesystem that does not support
->writepages.

[aarcange@redhat.com: This patch is heavily based on Andrea's work]
[akpm@linux-foundation.org: fix fs/nfs/write.c build]
[akpm@linux-foundation.org: fix fs/btrfs/disk-io.c build]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Andy Isaacson <adi@hexapodia.org>
Cc: Nai Xia <nai.xia@gmail.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Mel Gorman 2012-01-12 17:19:43 -08:00 committed by Linus Torvalds
parent 66199712e9
commit a6bc32b899
11 changed files with 76 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
fs/btrfs/disk-io.c
View file

@ -872,7 +872,8 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
static int btree_migratepage(struct address_space *mapping, static int btree_migratepage(struct address_space *mapping,
struct page *newpage, struct page *page, bool sync) struct page *newpage, struct page *page,
enum migrate_mode mode)
{ {
/* /*
* we can't safely write a btree page from here, * we can't safely write a btree page from here,
@ -887,7 +888,7 @@ static int btree_migratepage(struct address_space *mapping,
if (page_has_private(page) && if (page_has_private(page) &&
!try_to_release_page(page, GFP_KERNEL)) !try_to_release_page(page, GFP_KERNEL))
return -EAGAIN; return -EAGAIN;
return migrate_page(mapping, newpage, page, sync); return migrate_page(mapping, newpage, page, mode);
} }
#endif #endif

2
fs/hugetlbfs/inode.c
View file

@ -584,7 +584,7 @@ static int hugetlbfs_set_page_dirty(struct page *page)
static int hugetlbfs_migrate_page(struct address_space *mapping, static int hugetlbfs_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, struct page *newpage, struct page *page,
bool sync) enum migrate_mode mode)
{ {
int rc; int rc;

2
fs/nfs/internal.h
View file

@ -332,7 +332,7 @@ void nfs_commit_release_pages(struct nfs_write_data *data);
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
extern int nfs_migrate_page(struct address_space *, extern int nfs_migrate_page(struct address_space *,
struct page *, struct page *, bool); struct page *, struct page *, enum migrate_mode);
#else #else
#define nfs_migrate_page NULL #define nfs_migrate_page NULL
#endif #endif

4
fs/nfs/write.c
View file

@ -1688,7 +1688,7 @@ int nfs_wb_page(struct inode *inode, struct page *page)
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage, int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
struct page *page, bool sync) struct page *page, enum migrate_mode mode)
{ {
/* /*
* If PagePrivate is set, then the page is currently associated with * If PagePrivate is set, then the page is currently associated with
@ -1703,7 +1703,7 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
nfs_fscache_release_page(page, GFP_KERNEL); nfs_fscache_release_page(page, GFP_KERNEL);
return migrate_page(mapping, newpage, page, sync); return migrate_page(mapping, newpage, page, mode);
} }
#endif #endif

6
include/linux/fs.h
View file

@ -525,6 +525,7 @@ enum positive_aop_returns {
struct page; struct page;
struct address_space; struct address_space;
struct writeback_control; struct writeback_control;
enum migrate_mode;
struct iov_iter { struct iov_iter {
const struct iovec *iov; const struct iovec *iov;
@ -614,7 +615,7 @@ struct address_space_operations {
* is false, it must not block. * is false, it must not block.
*/ */
int (*migratepage) (struct address_space *, int (*migratepage) (struct address_space *,
struct page *, struct page *, bool); struct page *, struct page *, enum migrate_mode);
int (*launder_page) (struct page *); int (*launder_page) (struct page *);
int (*is_partially_uptodate) (struct page *, read_descriptor_t *, int (*is_partially_uptodate) (struct page *, read_descriptor_t *,
unsigned long); unsigned long);
@ -2540,7 +2541,8 @@ extern int generic_check_addressable(unsigned, u64);
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
extern int buffer_migrate_page(struct address_space *, extern int buffer_migrate_page(struct address_space *,
struct page *, struct page *, bool); struct page *, struct page *,
enum migrate_mode);
#else #else
#define buffer_migrate_page NULL #define buffer_migrate_page NULL
#endif #endif

23
include/linux/migrate.h
View file

@ -6,18 +6,31 @@
typedef struct page *new_page_t(struct page *, unsigned long private, int **); typedef struct page *new_page_t(struct page *, unsigned long private, int **);
/*
* MIGRATE_ASYNC means never block
* MIGRATE_SYNC_LIGHT in the current implementation means to allow blocking
* on most operations but not ->writepage as the potential stall time
* is too significant
* MIGRATE_SYNC will block when migrating pages
*/
enum migrate_mode {
MIGRATE_ASYNC,
MIGRATE_SYNC_LIGHT,
MIGRATE_SYNC,
};
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
#define PAGE_MIGRATION 1 #define PAGE_MIGRATION 1
extern void putback_lru_pages(struct list_head *l); extern void putback_lru_pages(struct list_head *l);
extern int migrate_page(struct address_space *, extern int migrate_page(struct address_space *,
struct page *, struct page *, bool); struct page *, struct page *, enum migrate_mode);
extern int migrate_pages(struct list_head *l, new_page_t x, extern int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private, bool offlining, unsigned long private, bool offlining,
bool sync); enum migrate_mode mode);
extern int migrate_huge_pages(struct list_head *l, new_page_t x, extern int migrate_huge_pages(struct list_head *l, new_page_t x,
unsigned long private, bool offlining, unsigned long private, bool offlining,
bool sync); enum migrate_mode mode);
extern int fail_migrate_page(struct address_space *, extern int fail_migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *);
@ -36,10 +49,10 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,
static inline void putback_lru_pages(struct list_head *l) {} static inline void putback_lru_pages(struct list_head *l) {}
static inline int migrate_pages(struct list_head *l, new_page_t x, static inline int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private, bool offlining, unsigned long private, bool offlining,
bool sync) { return -ENOSYS; } enum migrate_mode mode) { return -ENOSYS; }
static inline int migrate_huge_pages(struct list_head *l, new_page_t x, static inline int migrate_huge_pages(struct list_head *l, new_page_t x,
unsigned long private, bool offlining, unsigned long private, bool offlining,
bool sync) { return -ENOSYS; } enum migrate_mode mode) { return -ENOSYS; }
static inline int migrate_prep(void) { return -ENOSYS; } static inline int migrate_prep(void) { return -ENOSYS; }
static inline int migrate_prep_local(void) { return -ENOSYS; } static inline int migrate_prep_local(void) { return -ENOSYS; }

2
mm/compaction.c
View file

@ -557,7 +557,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
nr_migrate = cc->nr_migratepages; nr_migrate = cc->nr_migratepages;
err = migrate_pages(&cc->migratepages, compaction_alloc, err = migrate_pages(&cc->migratepages, compaction_alloc,
(unsigned long)cc, false, (unsigned long)cc, false,
cc->sync); cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC);
update_nr_listpages(cc); update_nr_listpages(cc);
nr_remaining = cc->nr_migratepages; nr_remaining = cc->nr_migratepages;

2
mm/memory-failure.c
View file

@ -1557,7 +1557,7 @@ int soft_offline_page(struct page *page, int flags)
page_is_file_cache(page)); page_is_file_cache(page));
list_add(&page->lru, &pagelist); list_add(&page->lru, &pagelist);
ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
0, true); 0, MIGRATE_SYNC);
if (ret) { if (ret) {
putback_lru_pages(&pagelist); putback_lru_pages(&pagelist);
pr_info("soft offline: %#lx: migration failed %d, type %lx\n", pr_info("soft offline: %#lx: migration failed %d, type %lx\n",

2
mm/memory_hotplug.c
View file

@ -809,7 +809,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
} }
/* this function returns # of failed pages */ /* this function returns # of failed pages */
ret = migrate_pages(&source, hotremove_migrate_alloc, 0, ret = migrate_pages(&source, hotremove_migrate_alloc, 0,
true, true); true, MIGRATE_SYNC);
if (ret) if (ret)
putback_lru_pages(&source); putback_lru_pages(&source);
} }

2
mm/mempolicy.c
View file

@ -942,7 +942,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
if (!list_empty(&pagelist)) { if (!list_empty(&pagelist)) {
err = migrate_pages(&pagelist, new_node_page, dest, err = migrate_pages(&pagelist, new_node_page, dest,
false, true); false, MIGRATE_SYNC);
if (err) if (err)
putback_lru_pages(&pagelist); putback_lru_pages(&pagelist);
} }

78
mm/migrate.c
View file

@ -218,12 +218,13 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
#ifdef CONFIG_BLOCK #ifdef CONFIG_BLOCK
/* Returns true if all buffers are successfully locked */ /* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head, bool sync) static bool buffer_migrate_lock_buffers(struct buffer_head *head,
enum migrate_mode mode)
{ {
struct buffer_head *bh = head; struct buffer_head *bh = head;
/* Simple case, sync compaction */ /* Simple case, sync compaction */
if (sync) { if (mode != MIGRATE_ASYNC) {
do { do {
get_bh(bh); get_bh(bh);
lock_buffer(bh); lock_buffer(bh);
@ -259,7 +260,7 @@ static bool buffer_migrate_lock_buffers(struct buffer_head *head, bool sync)
} }
#else #else
static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
bool sync) enum migrate_mode mode)
{ {
return true; return true;
} }
@ -275,7 +276,7 @@ static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
*/ */
static int migrate_page_move_mapping(struct address_space *mapping, static int migrate_page_move_mapping(struct address_space *mapping,
struct page *newpage, struct page *page, struct page *newpage, struct page *page,
struct buffer_head *head, bool sync) struct buffer_head *head, enum migrate_mode mode)
{ {
int expected_count; int expected_count;
void **pslot; void **pslot;
@ -311,7 +312,8 @@ static int migrate_page_move_mapping(struct address_space *mapping,
* the mapping back due to an elevated page count, we would have to * the mapping back due to an elevated page count, we would have to
* block waiting on other references to be dropped. * block waiting on other references to be dropped.
*/ */
if (!sync && head && !buffer_migrate_lock_buffers(head, sync)) { if (mode == MIGRATE_ASYNC && head &&
!buffer_migrate_lock_buffers(head, mode)) {
page_unfreeze_refs(page, expected_count); page_unfreeze_refs(page, expected_count);
spin_unlock_irq(&mapping->tree_lock); spin_unlock_irq(&mapping->tree_lock);
return -EAGAIN; return -EAGAIN;
@ -472,13 +474,14 @@ EXPORT_SYMBOL(fail_migrate_page);
* Pages are locked upon entry and exit. * Pages are locked upon entry and exit.
*/ */
int migrate_page(struct address_space *mapping, int migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, bool sync) struct page *newpage, struct page *page,
enum migrate_mode mode)
{ {
int rc; int rc;
BUG_ON(PageWriteback(page)); /* Writeback must be complete */ BUG_ON(PageWriteback(page)); /* Writeback must be complete */
rc = migrate_page_move_mapping(mapping, newpage, page, NULL, sync); rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode);
if (rc) if (rc)
return rc; return rc;
@ -495,17 +498,17 @@ EXPORT_SYMBOL(migrate_page);
* exist. * exist.
*/ */
int buffer_migrate_page(struct address_space *mapping, int buffer_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, bool sync) struct page *newpage, struct page *page, enum migrate_mode mode)
{ {
struct buffer_head *bh, *head; struct buffer_head *bh, *head;
int rc; int rc;
if (!page_has_buffers(page)) if (!page_has_buffers(page))
return migrate_page(mapping, newpage, page, sync); return migrate_page(mapping, newpage, page, mode);
head = page_buffers(page); head = page_buffers(page);
rc = migrate_page_move_mapping(mapping, newpage, page, head, sync); rc = migrate_page_move_mapping(mapping, newpage, page, head, mode);
if (rc) if (rc)
return rc; return rc;
@ -515,8 +518,8 @@ int buffer_migrate_page(struct address_space *mapping,
* with an IRQ-safe spinlock held. In the sync case, the buffers * with an IRQ-safe spinlock held. In the sync case, the buffers
* need to be locked now * need to be locked now
*/ */
if (sync) if (mode != MIGRATE_ASYNC)
BUG_ON(!buffer_migrate_lock_buffers(head, sync)); BUG_ON(!buffer_migrate_lock_buffers(head, mode));
ClearPagePrivate(page); ClearPagePrivate(page);
set_page_private(newpage, page_private(page)); set_page_private(newpage, page_private(page));
@ -593,10 +596,11 @@ static int writeout(struct address_space *mapping, struct page *page)
* Default handling if a filesystem does not provide a migration function. * Default handling if a filesystem does not provide a migration function.
*/ */
static int fallback_migrate_page(struct address_space *mapping, static int fallback_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, bool sync) struct page *newpage, struct page *page, enum migrate_mode mode)
{ {
if (PageDirty(page)) { if (PageDirty(page)) {
if (!sync) /* Only writeback pages in full synchronous migration */
if (mode != MIGRATE_SYNC)
return -EBUSY; return -EBUSY;
return writeout(mapping, page); return writeout(mapping, page);
} }
@ -609,7 +613,7 @@ static int fallback_migrate_page(struct address_space *mapping,
!try_to_release_page(page, GFP_KERNEL)) !try_to_release_page(page, GFP_KERNEL))
return -EAGAIN; return -EAGAIN;
return migrate_page(mapping, newpage, page, sync); return migrate_page(mapping, newpage, page, mode);
} }
/* /*
@ -624,7 +628,7 @@ static int fallback_migrate_page(struct address_space *mapping,
* == 0 - success * == 0 - success
*/ */
static int move_to_new_page(struct page *newpage, struct page *page, static int move_to_new_page(struct page *newpage, struct page *page,
int remap_swapcache, bool sync) int remap_swapcache, enum migrate_mode mode)
{ {
struct address_space *mapping; struct address_space *mapping;
int rc; int rc;
@ -645,7 +649,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
mapping = page_mapping(page); mapping = page_mapping(page);
if (!mapping) if (!mapping)
rc = migrate_page(mapping, newpage, page, sync); rc = migrate_page(mapping, newpage, page, mode);
else if (mapping->a_ops->migratepage) else if (mapping->a_ops->migratepage)
/* /*
* Most pages have a mapping and most filesystems provide a * Most pages have a mapping and most filesystems provide a
@ -654,9 +658,9 @@ static int move_to_new_page(struct page *newpage, struct page *page,
* is the most common path for page migration. * is the most common path for page migration.
*/ */
rc = mapping->a_ops->migratepage(mapping, rc = mapping->a_ops->migratepage(mapping,
newpage, page, sync); newpage, page, mode);
else else
rc = fallback_migrate_page(mapping, newpage, page, sync); rc = fallback_migrate_page(mapping, newpage, page, mode);
if (rc) { if (rc) {
newpage->mapping = NULL; newpage->mapping = NULL;
@ -671,7 +675,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
} }
static int __unmap_and_move(struct page *page, struct page *newpage, static int __unmap_and_move(struct page *page, struct page *newpage,
int force, bool offlining, bool sync) int force, bool offlining, enum migrate_mode mode)
{ {
int rc = -EAGAIN; int rc = -EAGAIN;
int remap_swapcache = 1; int remap_swapcache = 1;
@ -680,7 +684,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
struct anon_vma *anon_vma = NULL; struct anon_vma *anon_vma = NULL;
if (!trylock_page(page)) { if (!trylock_page(page)) {
if (!force || !sync) if (!force || mode == MIGRATE_ASYNC)
goto out; goto out;
/* /*
@ -726,10 +730,12 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
if (PageWriteback(page)) { if (PageWriteback(page)) {
/* /*
* For !sync, there is no point retrying as the retry loop * Only in the case of a full syncronous migration is it
* is expected to be too short for PageWriteback to be cleared * necessary to wait for PageWriteback. In the async case,
* the retry loop is too short and in the sync-light case,
* the overhead of stalling is too much
*/ */
if (!sync) { if (mode != MIGRATE_SYNC) {
rc = -EBUSY; rc = -EBUSY;
goto uncharge; goto uncharge;
} }
@ -800,7 +806,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
skip_unmap: skip_unmap:
if (!page_mapped(page)) if (!page_mapped(page))
rc = move_to_new_page(newpage, page, remap_swapcache, sync); rc = move_to_new_page(newpage, page, remap_swapcache, mode);
if (rc && remap_swapcache) if (rc && remap_swapcache)
remove_migration_ptes(page, page); remove_migration_ptes(page, page);
@ -823,7 +829,8 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
* to the newly allocated page in newpage. * to the newly allocated page in newpage.
*/ */
static int unmap_and_move(new_page_t get_new_page, unsigned long private, static int unmap_and_move(new_page_t get_new_page, unsigned long private,
struct page *page, int force, bool offlining, bool sync) struct page *page, int force, bool offlining,
enum migrate_mode mode)
{ {
int rc = 0; int rc = 0;
int *result = NULL; int *result = NULL;
@ -843,7 +850,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
if (unlikely(split_huge_page(page))) if (unlikely(split_huge_page(page)))
goto out; goto out;
rc = __unmap_and_move(page, newpage, force, offlining, sync); rc = __unmap_and_move(page, newpage, force, offlining, mode);
out: out:
if (rc != -EAGAIN) { if (rc != -EAGAIN) {
/* /*
@ -891,7 +898,8 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
*/ */
static int unmap_and_move_huge_page(new_page_t get_new_page, static int unmap_and_move_huge_page(new_page_t get_new_page,
unsigned long private, struct page *hpage, unsigned long private, struct page *hpage,
int force, bool offlining, bool sync) int force, bool offlining,
enum migrate_mode mode)
{ {
int rc = 0; int rc = 0;
int *result = NULL; int *result = NULL;
@ -904,7 +912,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
rc = -EAGAIN; rc = -EAGAIN;
if (!trylock_page(hpage)) { if (!trylock_page(hpage)) {
if (!force || !sync) if (!force || mode != MIGRATE_SYNC)
goto out; goto out;
lock_page(hpage); lock_page(hpage);
} }
@ -915,7 +923,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
if (!page_mapped(hpage)) if (!page_mapped(hpage))
rc = move_to_new_page(new_hpage, hpage, 1, sync); rc = move_to_new_page(new_hpage, hpage, 1, mode);
if (rc) if (rc)
remove_migration_ptes(hpage, hpage); remove_migration_ptes(hpage, hpage);
@ -958,7 +966,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
*/ */
int migrate_pages(struct list_head *from, int migrate_pages(struct list_head *from,
new_page_t get_new_page, unsigned long private, bool offlining, new_page_t get_new_page, unsigned long private, bool offlining,
bool sync) enum migrate_mode mode)
{ {
int retry = 1; int retry = 1;
int nr_failed = 0; int nr_failed = 0;
@ -979,7 +987,7 @@ int migrate_pages(struct list_head *from,
rc = unmap_and_move(get_new_page, private, rc = unmap_and_move(get_new_page, private,
page, pass > 2, offlining, page, pass > 2, offlining,
sync); mode);
switch(rc) { switch(rc) {
case -ENOMEM: case -ENOMEM:
@ -1009,7 +1017,7 @@ int migrate_pages(struct list_head *from,
int migrate_huge_pages(struct list_head *from, int migrate_huge_pages(struct list_head *from,
new_page_t get_new_page, unsigned long private, bool offlining, new_page_t get_new_page, unsigned long private, bool offlining,
bool sync) enum migrate_mode mode)
{ {
int retry = 1; int retry = 1;
int nr_failed = 0; int nr_failed = 0;
@ -1026,7 +1034,7 @@ int migrate_huge_pages(struct list_head *from,
rc = unmap_and_move_huge_page(get_new_page, rc = unmap_and_move_huge_page(get_new_page,
private, page, pass > 2, offlining, private, page, pass > 2, offlining,
sync); mode);
switch(rc) { switch(rc) {
case -ENOMEM: case -ENOMEM:
@ -1155,7 +1163,7 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
err = 0; err = 0;
if (!list_empty(&pagelist)) { if (!list_empty(&pagelist)) {
err = migrate_pages(&pagelist, new_page_node, err = migrate_pages(&pagelist, new_page_node,
(unsigned long)pm, 0, true); (unsigned long)pm, 0, MIGRATE_SYNC);
if (err) if (err)
putback_lru_pages(&pagelist); putback_lru_pages(&pagelist);
} }