shmem: reduce pagefault lock contention
I'm running a shmem pagefault test case (see attached file) under a 64 CPU system. Profile shows shmem_inode_info->lock is heavily contented and 100% CPUs time are trying to get the lock. In the pagefault (no swap) case, shmem_getpage gets the lock twice, the last one is avoidable if we prealloc a page so we could reduce one time of locking. This is what below patch does. The result of the test case: 2.6.35-rc3: ~20s 2.6.35-rc3 + patch: ~12s so this is 40% improvement. One might argue if we could have better locking for shmem. But even shmem is lockless, the pagefault will soon have pagecache lock heavily contented because shmem must add new page to pagecache. So before we have better locking for pagecache, improving shmem locking doesn't have too much improvement. I did a similar pagefault test against a ramfs file, the test result is ~10.5s. [akpm@linux-foundation.org: fix comment, clean up code layout, elimintate code duplication] Signed-off-by: Shaohua Li <shaohua.li@intel.com> Cc: Hugh Dickins <hughd@google.com> Cc: "Zhang, Yanmin" <yanmin.zhang@intel.com> Cc: Tim Chen <tim.c.chen@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
7e496299d4
commit
ff36b80162
1 changed files with 49 additions and 21 deletions
70
mm/shmem.c
70
mm/shmem.c
|
@ -1222,6 +1222,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|||
struct shmem_sb_info *sbinfo;
|
||||
struct page *filepage = *pagep;
|
||||
struct page *swappage;
|
||||
struct page *prealloc_page = NULL;
|
||||
swp_entry_t *entry;
|
||||
swp_entry_t swap;
|
||||
gfp_t gfp;
|
||||
|
@ -1246,7 +1247,6 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|||
filepage = find_lock_page(mapping, idx);
|
||||
if (filepage && PageUptodate(filepage))
|
||||
goto done;
|
||||
error = 0;
|
||||
gfp = mapping_gfp_mask(mapping);
|
||||
if (!filepage) {
|
||||
/*
|
||||
|
@ -1257,7 +1257,19 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|||
if (error)
|
||||
goto failed;
|
||||
radix_tree_preload_end();
|
||||
if (sgp != SGP_READ && !prealloc_page) {
|
||||
/* We don't care if this fails */
|
||||
prealloc_page = shmem_alloc_page(gfp, info, idx);
|
||||
if (prealloc_page) {
|
||||
if (mem_cgroup_cache_charge(prealloc_page,
|
||||
current->mm, GFP_KERNEL)) {
|
||||
page_cache_release(prealloc_page);
|
||||
prealloc_page = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
error = 0;
|
||||
|
||||
spin_lock(&info->lock);
|
||||
shmem_recalc_inode(inode);
|
||||
|
@ -1405,28 +1417,38 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|||
if (!filepage) {
|
||||
int ret;
|
||||
|
||||
spin_unlock(&info->lock);
|
||||
filepage = shmem_alloc_page(gfp, info, idx);
|
||||
if (!filepage) {
|
||||
shmem_unacct_blocks(info->flags, 1);
|
||||
shmem_free_blocks(inode, 1);
|
||||
error = -ENOMEM;
|
||||
goto failed;
|
||||
}
|
||||
SetPageSwapBacked(filepage);
|
||||
if (!prealloc_page) {
|
||||
spin_unlock(&info->lock);
|
||||
filepage = shmem_alloc_page(gfp, info, idx);
|
||||
if (!filepage) {
|
||||
shmem_unacct_blocks(info->flags, 1);
|
||||
shmem_free_blocks(inode, 1);
|
||||
error = -ENOMEM;
|
||||
goto failed;
|
||||
}
|
||||
SetPageSwapBacked(filepage);
|
||||
|
||||
/* Precharge page while we can wait, compensate after */
|
||||
error = mem_cgroup_cache_charge(filepage, current->mm,
|
||||
GFP_KERNEL);
|
||||
if (error) {
|
||||
page_cache_release(filepage);
|
||||
shmem_unacct_blocks(info->flags, 1);
|
||||
shmem_free_blocks(inode, 1);
|
||||
filepage = NULL;
|
||||
goto failed;
|
||||
/*
|
||||
* Precharge page while we can wait, compensate
|
||||
* after
|
||||
*/
|
||||
error = mem_cgroup_cache_charge(filepage,
|
||||
current->mm, GFP_KERNEL);
|
||||
if (error) {
|
||||
page_cache_release(filepage);
|
||||
shmem_unacct_blocks(info->flags, 1);
|
||||
shmem_free_blocks(inode, 1);
|
||||
filepage = NULL;
|
||||
goto failed;
|
||||
}
|
||||
|
||||
spin_lock(&info->lock);
|
||||
} else {
|
||||
filepage = prealloc_page;
|
||||
prealloc_page = NULL;
|
||||
SetPageSwapBacked(filepage);
|
||||
}
|
||||
|
||||
spin_lock(&info->lock);
|
||||
entry = shmem_swp_alloc(info, idx, sgp);
|
||||
if (IS_ERR(entry))
|
||||
error = PTR_ERR(entry);
|
||||
|
@ -1467,13 +1489,19 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|||
}
|
||||
done:
|
||||
*pagep = filepage;
|
||||
return 0;
|
||||
error = 0;
|
||||
goto out;
|
||||
|
||||
failed:
|
||||
if (*pagep != filepage) {
|
||||
unlock_page(filepage);
|
||||
page_cache_release(filepage);
|
||||
}
|
||||
out:
|
||||
if (prealloc_page) {
|
||||
mem_cgroup_uncharge_cache_page(prealloc_page);
|
||||
page_cache_release(prealloc_page);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue