rmap: support file thp

Naive approach: on mapping/unmapping the page as compound we update
->_mapcount on each 4k page.  That's not efficient, but it's not obvious
how we can optimize this.  We can look into optimization later.

PG_double_map optimization doesn't work for file pages since lifecycle
of file pages is different comparing to anon pages: file page can be
mapped again at any time.

Link: http://lkml.kernel.org/r/1466021202-61880-11-git-send-email-kirill.shutemov@linux.intel.com
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 2b0fad8..b46bb56 100644 (file)
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -165,7 +165,7 @@ void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
                           unsigned long, int);
 void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
                unsigned long, bool);
-void page_add_file_rmap(struct page *);
+void page_add_file_rmap(struct page *, bool);
 void page_remove_rmap(struct page *, bool);
 
 void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index bc5abcb..90f5dd2 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3301,18 +3301,22 @@ static void __split_huge_page(struct page *page, struct list_head *list)
 
 int total_mapcount(struct page *page)
 {
-       int i, ret;
+       int i, compound, ret;
 
        VM_BUG_ON_PAGE(PageTail(page), page);
 
        if (likely(!PageCompound(page)))
                return atomic_read(&page->_mapcount) + 1;
 
-       ret = compound_mapcount(page);
+       compound = compound_mapcount(page);
        if (PageHuge(page))
-               return ret;
+               return compound;
+       ret = compound;
        for (i = 0; i < HPAGE_PMD_NR; i++)
                ret += atomic_read(&page[i]._mapcount) + 1;
+       /* File pages has compound_mapcount included in _mapcount */
+       if (!PageAnon(page))
+               return ret - compound * HPAGE_PMD_NR;
        if (PageDoubleMap(page))
                ret -= HPAGE_PMD_NR;
        return ret;

diff --git a/mm/memory.c b/mm/memory.c
index 1991105..30cda24 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1494,7 +1494,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
        /* Ok, finally just insert the thing.. */
        get_page(page);
        inc_mm_counter_fast(mm, mm_counter_file(page));
-       page_add_file_rmap(page);
+       page_add_file_rmap(page, false);
        set_pte_at(mm, addr, pte, mk_pte(page, prot));
 
        retval = 0;
@@ -2962,7 +2962,7 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
                lru_cache_add_active_or_unevictable(page, vma);
        } else {
                inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
-               page_add_file_rmap(page);
+               page_add_file_rmap(page, false);
        }
        set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
 

diff --git a/mm/migrate.c b/mm/migrate.c
index f278005..e85a72c 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -259,7 +259,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
        } else if (PageAnon(new))
                page_add_anon_rmap(new, vma, addr, false);
        else
-               page_add_file_rmap(new);
+               page_add_file_rmap(new, false);
 
        if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
                mlock_vma_page(new);

diff --git a/mm/rmap.c b/mm/rmap.c
index 701b93f..2b336c4 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1270,18 +1270,34 @@ void page_add_new_anon_rmap(struct page *page,
  *
  * The caller needs to hold the pte lock.
  */
-void page_add_file_rmap(struct page *page)
+void page_add_file_rmap(struct page *page, bool compound)
 {
+       int i, nr = 1;
+
+       VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
        lock_page_memcg(page);
-       if (atomic_inc_and_test(&page->_mapcount)) {
-               __inc_zone_page_state(page, NR_FILE_MAPPED);
-               mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
+       if (compound && PageTransHuge(page)) {
+               for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
+                       if (atomic_inc_and_test(&page[i]._mapcount))
+                               nr++;
+               }
+               if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
+                       goto out;
+       } else {
+               if (!atomic_inc_and_test(&page->_mapcount))
+                       goto out;
        }
+       __mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, nr);
+       mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
+out:
        unlock_page_memcg(page);
 }
 
-static void page_remove_file_rmap(struct page *page)
+static void page_remove_file_rmap(struct page *page, bool compound)
 {
+       int i, nr = 1;
+
+       VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
        lock_page_memcg(page);
 
        /* Hugepages are not counted in NR_FILE_MAPPED for now. */
@@ -1292,15 +1308,24 @@ static void page_remove_file_rmap(struct page *page)
        }
 
        /* page still mapped by someone else? */
-       if (!atomic_add_negative(-1, &page->_mapcount))
-               goto out;
+       if (compound && PageTransHuge(page)) {
+               for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
+                       if (atomic_add_negative(-1, &page[i]._mapcount))
+                               nr++;
+               }
+               if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
+                       goto out;
+       } else {
+               if (!atomic_add_negative(-1, &page->_mapcount))
+                       goto out;
+       }
 
        /*
         * We use the irq-unsafe __{inc|mod}_zone_page_stat because
         * these counters are not modified in interrupt context, and
         * pte lock(a spinlock) is held, which implies preemption disabled.
         */
-       __dec_zone_page_state(page, NR_FILE_MAPPED);
+       __mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, -nr);
        mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
 
        if (unlikely(PageMlocked(page)))
@@ -1356,11 +1381,8 @@ static void page_remove_anon_compound_rmap(struct page *page)
  */
 void page_remove_rmap(struct page *page, bool compound)
 {
-       if (!PageAnon(page)) {
-               VM_BUG_ON_PAGE(compound && !PageHuge(page), page);
-               page_remove_file_rmap(page);
-               return;
-       }
+       if (!PageAnon(page))
+               return page_remove_file_rmap(page, compound);
 
        if (compound)
                return page_remove_anon_compound_rmap(page);

diff --git a/mm/util.c b/mm/util.c
index b756ee3..8d010ef 100644 (file)
--- a/mm/util.c
+++ b/mm/util.c
@@ -412,6 +412,12 @@ int __page_mapcount(struct page *page)
        int ret;
 
        ret = atomic_read(&page->_mapcount) + 1;
+       /*
+        * For file THP page->_mapcount contains total number of mapping
+        * of the page: no need to look into compound_mapcount.
+        */
+       if (!PageAnon(page) && !PageHuge(page))
+               return ret;
        page = compound_head(page);
        ret += atomic_read(compound_mapcount_ptr(page)) + 1;
        if (PageDoubleMap(page))