mm: numa: Do not group on RO pages
And here's a little something to make sure not the whole world ends up in a single group. As while we don't migrate shared executable pages, we do scan/fault on them. And since everybody links to libc, everybody ends up in the same group. Suggested-by: Rik van Riel <riel@redhat.com> Signed-off-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Mel Gorman <mgorman@suse.de> Reviewed-by: Rik van Riel <riel@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com> Link: http://lkml.kernel.org/r/1381141781-10992-47-git-send-email-mgorman@suse.de Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
7851a45cd3
commit
6688cc0547
4 changed files with 47 additions and 10 deletions
|
@ -1450,13 +1450,16 @@ struct task_struct {
|
||||||
/* Future-safe accessor for struct task_struct's cpus_allowed. */
|
/* Future-safe accessor for struct task_struct's cpus_allowed. */
|
||||||
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
|
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
|
||||||
|
|
||||||
|
#define TNF_MIGRATED 0x01
|
||||||
|
#define TNF_NO_GROUP 0x02
|
||||||
|
|
||||||
#ifdef CONFIG_NUMA_BALANCING
|
#ifdef CONFIG_NUMA_BALANCING
|
||||||
extern void task_numa_fault(int last_node, int node, int pages, bool migrated);
|
extern void task_numa_fault(int last_node, int node, int pages, int flags);
|
||||||
extern pid_t task_numa_group_id(struct task_struct *p);
|
extern pid_t task_numa_group_id(struct task_struct *p);
|
||||||
extern void set_numabalancing_state(bool enabled);
|
extern void set_numabalancing_state(bool enabled);
|
||||||
#else
|
#else
|
||||||
static inline void task_numa_fault(int last_node, int node, int pages,
|
static inline void task_numa_fault(int last_node, int node, int pages,
|
||||||
bool migrated)
|
int flags)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
static inline pid_t task_numa_group_id(struct task_struct *p)
|
static inline pid_t task_numa_group_id(struct task_struct *p)
|
||||||
|
|
|
@ -1361,9 +1361,10 @@ void task_numa_free(struct task_struct *p)
|
||||||
/*
|
/*
|
||||||
* Got a PROT_NONE fault for a page on @node.
|
* Got a PROT_NONE fault for a page on @node.
|
||||||
*/
|
*/
|
||||||
void task_numa_fault(int last_cpupid, int node, int pages, bool migrated)
|
void task_numa_fault(int last_cpupid, int node, int pages, int flags)
|
||||||
{
|
{
|
||||||
struct task_struct *p = current;
|
struct task_struct *p = current;
|
||||||
|
bool migrated = flags & TNF_MIGRATED;
|
||||||
int priv;
|
int priv;
|
||||||
|
|
||||||
if (!numabalancing_enabled)
|
if (!numabalancing_enabled)
|
||||||
|
@ -1394,7 +1395,7 @@ void task_numa_fault(int last_cpupid, int node, int pages, bool migrated)
|
||||||
priv = 1;
|
priv = 1;
|
||||||
} else {
|
} else {
|
||||||
priv = cpupid_match_pid(p, last_cpupid);
|
priv = cpupid_match_pid(p, last_cpupid);
|
||||||
if (!priv)
|
if (!priv && !(flags & TNF_NO_GROUP))
|
||||||
task_numa_group(p, last_cpupid);
|
task_numa_group(p, last_cpupid);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1285,6 +1285,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
int target_nid, last_cpupid = -1;
|
int target_nid, last_cpupid = -1;
|
||||||
bool page_locked;
|
bool page_locked;
|
||||||
bool migrated = false;
|
bool migrated = false;
|
||||||
|
int flags = 0;
|
||||||
|
|
||||||
spin_lock(&mm->page_table_lock);
|
spin_lock(&mm->page_table_lock);
|
||||||
if (unlikely(!pmd_same(pmd, *pmdp)))
|
if (unlikely(!pmd_same(pmd, *pmdp)))
|
||||||
|
@ -1298,6 +1299,14 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
if (page_nid == this_nid)
|
if (page_nid == this_nid)
|
||||||
count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
|
count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Avoid grouping on DSO/COW pages in specific and RO pages
|
||||||
|
* in general, RO pages shouldn't hurt as much anyway since
|
||||||
|
* they can be in shared cache state.
|
||||||
|
*/
|
||||||
|
if (!pmd_write(pmd))
|
||||||
|
flags |= TNF_NO_GROUP;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Acquire the page lock to serialise THP migrations but avoid dropping
|
* Acquire the page lock to serialise THP migrations but avoid dropping
|
||||||
* page_table_lock if at all possible
|
* page_table_lock if at all possible
|
||||||
|
@ -1343,8 +1352,10 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
spin_unlock(&mm->page_table_lock);
|
spin_unlock(&mm->page_table_lock);
|
||||||
migrated = migrate_misplaced_transhuge_page(mm, vma,
|
migrated = migrate_misplaced_transhuge_page(mm, vma,
|
||||||
pmdp, pmd, addr, page, target_nid);
|
pmdp, pmd, addr, page, target_nid);
|
||||||
if (migrated)
|
if (migrated) {
|
||||||
|
flags |= TNF_MIGRATED;
|
||||||
page_nid = target_nid;
|
page_nid = target_nid;
|
||||||
|
}
|
||||||
|
|
||||||
goto out;
|
goto out;
|
||||||
clear_pmdnuma:
|
clear_pmdnuma:
|
||||||
|
@ -1362,7 +1373,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
page_unlock_anon_vma_read(anon_vma);
|
page_unlock_anon_vma_read(anon_vma);
|
||||||
|
|
||||||
if (page_nid != -1)
|
if (page_nid != -1)
|
||||||
task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, migrated);
|
task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
30
mm/memory.c
30
mm/memory.c
|
@ -3547,6 +3547,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
int last_cpupid;
|
int last_cpupid;
|
||||||
int target_nid;
|
int target_nid;
|
||||||
bool migrated = false;
|
bool migrated = false;
|
||||||
|
int flags = 0;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The "pte" at this point cannot be used safely without
|
* The "pte" at this point cannot be used safely without
|
||||||
|
@ -3575,6 +3576,14 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
}
|
}
|
||||||
BUG_ON(is_zero_pfn(page_to_pfn(page)));
|
BUG_ON(is_zero_pfn(page_to_pfn(page)));
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Avoid grouping on DSO/COW pages in specific and RO pages
|
||||||
|
* in general, RO pages shouldn't hurt as much anyway since
|
||||||
|
* they can be in shared cache state.
|
||||||
|
*/
|
||||||
|
if (!pte_write(pte))
|
||||||
|
flags |= TNF_NO_GROUP;
|
||||||
|
|
||||||
last_cpupid = page_cpupid_last(page);
|
last_cpupid = page_cpupid_last(page);
|
||||||
page_nid = page_to_nid(page);
|
page_nid = page_to_nid(page);
|
||||||
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
|
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
|
||||||
|
@ -3586,12 +3595,14 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
|
|
||||||
/* Migrate to the requested node */
|
/* Migrate to the requested node */
|
||||||
migrated = migrate_misplaced_page(page, vma, target_nid);
|
migrated = migrate_misplaced_page(page, vma, target_nid);
|
||||||
if (migrated)
|
if (migrated) {
|
||||||
page_nid = target_nid;
|
page_nid = target_nid;
|
||||||
|
flags |= TNF_MIGRATED;
|
||||||
|
}
|
||||||
|
|
||||||
out:
|
out:
|
||||||
if (page_nid != -1)
|
if (page_nid != -1)
|
||||||
task_numa_fault(last_cpupid, page_nid, 1, migrated);
|
task_numa_fault(last_cpupid, page_nid, 1, flags);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3632,6 +3643,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
int page_nid = -1;
|
int page_nid = -1;
|
||||||
int target_nid;
|
int target_nid;
|
||||||
bool migrated = false;
|
bool migrated = false;
|
||||||
|
int flags = 0;
|
||||||
|
|
||||||
if (!pte_present(pteval))
|
if (!pte_present(pteval))
|
||||||
continue;
|
continue;
|
||||||
|
@ -3651,20 +3663,30 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
if (unlikely(!page))
|
if (unlikely(!page))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Avoid grouping on DSO/COW pages in specific and RO pages
|
||||||
|
* in general, RO pages shouldn't hurt as much anyway since
|
||||||
|
* they can be in shared cache state.
|
||||||
|
*/
|
||||||
|
if (!pte_write(pteval))
|
||||||
|
flags |= TNF_NO_GROUP;
|
||||||
|
|
||||||
last_cpupid = page_cpupid_last(page);
|
last_cpupid = page_cpupid_last(page);
|
||||||
page_nid = page_to_nid(page);
|
page_nid = page_to_nid(page);
|
||||||
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
|
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
|
||||||
pte_unmap_unlock(pte, ptl);
|
pte_unmap_unlock(pte, ptl);
|
||||||
if (target_nid != -1) {
|
if (target_nid != -1) {
|
||||||
migrated = migrate_misplaced_page(page, vma, target_nid);
|
migrated = migrate_misplaced_page(page, vma, target_nid);
|
||||||
if (migrated)
|
if (migrated) {
|
||||||
page_nid = target_nid;
|
page_nid = target_nid;
|
||||||
|
flags |= TNF_MIGRATED;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
put_page(page);
|
put_page(page);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (page_nid != -1)
|
if (page_nid != -1)
|
||||||
task_numa_fault(last_cpupid, page_nid, 1, migrated);
|
task_numa_fault(last_cpupid, page_nid, 1, flags);
|
||||||
|
|
||||||
pte = pte_offset_map_lock(mm, pmdp, addr, &ptl);
|
pte = pte_offset_map_lock(mm, pmdp, addr, &ptl);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue