2012-10-08 17:31:25 -06:00
|
|
|
/*
|
|
|
|
* mm/interval_tree.c - interval tree for mapping->i_mmap
|
|
|
|
*
|
|
|
|
* Copyright (C) 2012, Michel Lespinasse <walken@google.com>
|
|
|
|
*
|
|
|
|
* This file is released under the GPL v2.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/fs.h>
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 17:31:39 -06:00
|
|
|
#include <linux/rmap.h>
|
2012-10-08 17:31:35 -06:00
|
|
|
#include <linux/interval_tree_generic.h>
|
2012-10-08 17:31:25 -06:00
|
|
|
|
2012-10-08 17:31:35 -06:00
|
|
|
static inline unsigned long vma_start_pgoff(struct vm_area_struct *v)
|
|
|
|
{
|
|
|
|
return v->vm_pgoff;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long vma_last_pgoff(struct vm_area_struct *v)
|
|
|
|
{
|
2018-01-31 17:17:03 -07:00
|
|
|
return v->vm_pgoff + vma_pages(v) - 1;
|
2012-10-08 17:31:35 -06:00
|
|
|
}
|
|
|
|
|
2015-02-10 15:10:02 -07:00
|
|
|
INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb,
|
|
|
|
unsigned long, shared.rb_subtree_last,
|
2012-10-08 17:31:35 -06:00
|
|
|
vma_start_pgoff, vma_last_pgoff,, vma_interval_tree)
|
|
|
|
|
|
|
|
/* Insert node immediately after prev in the interval tree */
|
|
|
|
void vma_interval_tree_insert_after(struct vm_area_struct *node,
|
|
|
|
struct vm_area_struct *prev,
|
2017-09-08 17:15:08 -06:00
|
|
|
struct rb_root_cached *root)
|
2012-10-08 17:31:25 -06:00
|
|
|
{
|
|
|
|
struct rb_node **link;
|
|
|
|
struct vm_area_struct *parent;
|
2012-10-08 17:31:35 -06:00
|
|
|
unsigned long last = vma_last_pgoff(node);
|
2012-10-08 17:31:25 -06:00
|
|
|
|
2014-10-09 16:28:10 -06:00
|
|
|
VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node);
|
2012-10-08 17:31:25 -06:00
|
|
|
|
2015-02-10 15:10:02 -07:00
|
|
|
if (!prev->shared.rb.rb_right) {
|
2012-10-08 17:31:35 -06:00
|
|
|
parent = prev;
|
2015-02-10 15:10:02 -07:00
|
|
|
link = &prev->shared.rb.rb_right;
|
2012-10-08 17:31:25 -06:00
|
|
|
} else {
|
2015-02-10 15:10:02 -07:00
|
|
|
parent = rb_entry(prev->shared.rb.rb_right,
|
|
|
|
struct vm_area_struct, shared.rb);
|
|
|
|
if (parent->shared.rb_subtree_last < last)
|
|
|
|
parent->shared.rb_subtree_last = last;
|
|
|
|
while (parent->shared.rb.rb_left) {
|
|
|
|
parent = rb_entry(parent->shared.rb.rb_left,
|
|
|
|
struct vm_area_struct, shared.rb);
|
|
|
|
if (parent->shared.rb_subtree_last < last)
|
|
|
|
parent->shared.rb_subtree_last = last;
|
2012-10-08 17:31:25 -06:00
|
|
|
}
|
2015-02-10 15:10:02 -07:00
|
|
|
link = &parent->shared.rb.rb_left;
|
2012-10-08 17:31:25 -06:00
|
|
|
}
|
|
|
|
|
2015-02-10 15:10:02 -07:00
|
|
|
node->shared.rb_subtree_last = last;
|
|
|
|
rb_link_node(&node->shared.rb, &parent->shared.rb, link);
|
2017-09-08 17:15:08 -06:00
|
|
|
rb_insert_augmented(&node->shared.rb, &root->rb_root,
|
2012-10-08 17:31:35 -06:00
|
|
|
&vma_interval_tree_augment);
|
2012-10-08 17:31:25 -06:00
|
|
|
}
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 17:31:39 -06:00
|
|
|
|
|
|
|
static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc)
|
|
|
|
{
|
|
|
|
return vma_start_pgoff(avc->vma);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc)
|
|
|
|
{
|
|
|
|
return vma_last_pgoff(avc->vma);
|
|
|
|
}
|
|
|
|
|
|
|
|
INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last,
|
2012-10-08 17:31:45 -06:00
|
|
|
avc_start_pgoff, avc_last_pgoff,
|
|
|
|
static inline, __anon_vma_interval_tree)
|
|
|
|
|
|
|
|
void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
|
2017-09-08 17:15:08 -06:00
|
|
|
struct rb_root_cached *root)
|
2012-10-08 17:31:45 -06:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_VM_RB
|
|
|
|
node->cached_vma_start = avc_start_pgoff(node);
|
|
|
|
node->cached_vma_last = avc_last_pgoff(node);
|
|
|
|
#endif
|
|
|
|
__anon_vma_interval_tree_insert(node, root);
|
|
|
|
}
|
|
|
|
|
|
|
|
void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
|
2017-09-08 17:15:08 -06:00
|
|
|
struct rb_root_cached *root)
|
2012-10-08 17:31:45 -06:00
|
|
|
{
|
|
|
|
__anon_vma_interval_tree_remove(node, root);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct anon_vma_chain *
|
2017-09-08 17:15:08 -06:00
|
|
|
anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
|
2012-10-08 17:31:45 -06:00
|
|
|
unsigned long first, unsigned long last)
|
|
|
|
{
|
|
|
|
return __anon_vma_interval_tree_iter_first(root, first, last);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct anon_vma_chain *
|
|
|
|
anon_vma_interval_tree_iter_next(struct anon_vma_chain *node,
|
|
|
|
unsigned long first, unsigned long last)
|
|
|
|
{
|
|
|
|
return __anon_vma_interval_tree_iter_next(node, first, last);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_VM_RB
|
|
|
|
void anon_vma_interval_tree_verify(struct anon_vma_chain *node)
|
|
|
|
{
|
|
|
|
WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node));
|
|
|
|
WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node));
|
|
|
|
}
|
|
|
|
#endif
|