summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>2012-03-21 16:34:26 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2012-03-21 17:55:02 -0700
commit4331f7d339ee0b54603344b9d13662a9c022540c (patch)
treeea67ad19a4dff0f8bf7d335bb7ab148ecfb4d329
parent2ff76f1193f8481f7e6c29304eea4006e8e51569 (diff)
memcg: fix performance of mem_cgroup_begin_update_page_stat()
mem_cgroup_begin_update_page_stat() should be very fast because it's called very frequently. Now, it needs to look up page_cgroup and its memcg....this is slow. This patch adds a global variable to check "any memcg is moving or not". With this, the caller doesn't need to visit page_cgroup and memcg. Here is a test result. A test program makes page faults onto a file, MAP_SHARED and makes each page's page_mapcount(page) > 1, and free the range by madvise() and page fault again. This program causes 26214400 times of page fault onto a file(size was 1G.) and shows shows the cost of mem_cgroup_begin_update_page_stat(). Before this patch for mem_cgroup_begin_update_page_stat() [kamezawa@bluextal test]$ time ./mmap 1G real 0m21.765s user 0m5.999s sys 0m15.434s 27.46% mmap mmap [.] reader 21.15% mmap [kernel.kallsyms] [k] page_fault 9.17% mmap [kernel.kallsyms] [k] filemap_fault 2.96% mmap [kernel.kallsyms] [k] __do_fault 2.83% mmap [kernel.kallsyms] [k] __mem_cgroup_begin_update_page_stat After this patch [root@bluextal test]# time ./mmap 1G real 0m21.373s user 0m6.113s sys 0m15.016s In usual path, calls to __mem_cgroup_begin_update_page_stat() goes away. Note: we may be able to remove this optimization in future if we can get pointer to memcg directly from struct page. [akpm@linux-foundation.org: don't return a void] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Acked-by: Greg Thelen <gthelen@google.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Michal Hocko <mhocko@suse.cz> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Ying Han <yinghan@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--include/linux/memcontrol.h5
-rw-r--r--mm/memcontrol.c9
2 files changed, 12 insertions, 2 deletions
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index bf7ae01fc93b..f94efd2f6c27 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -144,6 +144,8 @@ static inline bool mem_cgroup_disabled(void)
void __mem_cgroup_begin_update_page_stat(struct page *page, bool *locked,
unsigned long *flags);
+extern atomic_t memcg_moving;
+
static inline void mem_cgroup_begin_update_page_stat(struct page *page,
bool *locked, unsigned long *flags)
{
@@ -151,7 +153,8 @@ static inline void mem_cgroup_begin_update_page_stat(struct page *page,
return;
rcu_read_lock();
*locked = false;
- return __mem_cgroup_begin_update_page_stat(page, locked, flags);
+ if (atomic_read(&memcg_moving))
+ __mem_cgroup_begin_update_page_stat(page, locked, flags);
}
void __mem_cgroup_end_update_page_stat(struct page *page,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0e13b2aeea61..eb1004f207b3 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1306,8 +1306,13 @@ int mem_cgroup_swappiness(struct mem_cgroup *memcg)
* rcu_read_unlock()
* start move here.
*/
+
+/* for quick checking without looking up memcg */
+atomic_t memcg_moving __read_mostly;
+
static void mem_cgroup_start_move(struct mem_cgroup *memcg)
{
+ atomic_inc(&memcg_moving);
atomic_inc(&memcg->moving_account);
synchronize_rcu();
}
@@ -1318,8 +1323,10 @@ static void mem_cgroup_end_move(struct mem_cgroup *memcg)
* Now, mem_cgroup_clear_mc() may call this function with NULL.
* We check NULL in callee rather than caller.
*/
- if (memcg)
+ if (memcg) {
+ atomic_dec(&memcg_moving);
atomic_dec(&memcg->moving_account);
+ }
}
/*