timecounter: provide a macro to initialize the cyclecounter mask field.

[cascardo/linux.git] / mm / rmap.c

diff --git a/mm/rmap.c b/mm/rmap.c
index 19886fb..c5bc241 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -23,7 +23,7 @@
  * inode->i_mutex      (while writing or truncating, not reading or faulting)
  *   mm->mmap_sem
  *     page->flags PG_locked (lock_page)
- *       mapping->i_mmap_mutex
+ *       mapping->i_mmap_rwsem
  *         anon_vma->rwsem
  *           mm->page_table_lock or pte_lock
  *             zone->lru_lock (in mark_page_accessed, isolate_lru_page)
@@ -274,6 +274,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 {
        struct anon_vma_chain *avc;
        struct anon_vma *anon_vma;
+       int error;
 
        /* Don't bother if the parent process has no anon_vma here. */
        if (!pvma->anon_vma)
@@ -283,8 +284,9 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
         * First, attach the new VMA to the parent VMA's anon_vmas,
         * so rmap can find non-COWed pages in child processes.
         */
-       if (anon_vma_clone(vma, pvma))
-               return -ENOMEM;
+       error = anon_vma_clone(vma, pvma);
+       if (error)
+               return error;
 
        /* Then add our own anon_vma. */
        anon_vma = anon_vma_alloc();
@@ -581,7 +583,8 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
         * without holding anon_vma lock for write.  So when looking for a
         * genuine pmde (in which to find pte), test present and !THP together.
         */
-       pmde = ACCESS_ONCE(*pmd);
+       pmde = *pmd;
+       barrier();
        if (!pmd_present(pmde) || pmd_trans_huge(pmde))
                pmd = NULL;
 out:
@@ -1051,7 +1054,7 @@ void page_add_file_rmap(struct page *page)
                __inc_zone_page_state(page, NR_FILE_MAPPED);
                mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
        }
-       mem_cgroup_end_page_stat(memcg, locked, flags);
+       mem_cgroup_end_page_stat(memcg, &locked, &flags);
 }
 
 static void page_remove_file_rmap(struct page *page)
@@ -1081,7 +1084,7 @@ static void page_remove_file_rmap(struct page *page)
        if (unlikely(PageMlocked(page)))
                clear_page_mlock(page);
 out:
-       mem_cgroup_end_page_stat(memcg, locked, flags);
+       mem_cgroup_end_page_stat(memcg, &locked, &flags);
 }
 
 /**
@@ -1258,7 +1261,7 @@ out_mlock:
        /*
         * We need mmap_sem locking, Otherwise VM_LOCKED check makes
         * unstable result and race. Plus, We can't wait here because
-        * we now hold anon_vma->rwsem or mapping->i_mmap_mutex.
+        * we now hold anon_vma->rwsem or mapping->i_mmap_rwsem.
         * if trylock failed, the page remain in evictable lru and later
         * vmscan could retry to move the page to unevictable lru if the
         * page is actually mlocked.
@@ -1378,7 +1381,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
 
                /* Nuke the page table entry. */
                flush_cache_page(vma, address, pte_pfn(*pte));
-               pteval = ptep_clear_flush(vma, address, pte);
+               pteval = ptep_clear_flush_notify(vma, address, pte);
 
                /* If nonlinear, store the file page offset in the pte. */
                if (page->index != linear_page_index(vma, address)) {
@@ -1633,7 +1636,7 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page,
 static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc)
 {
        struct anon_vma *anon_vma;
-       pgoff_t pgoff = page_to_pgoff(page);
+       pgoff_t pgoff;
        struct anon_vma_chain *avc;
        int ret = SWAP_AGAIN;
 
@@ -1641,6 +1644,7 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc)
        if (!anon_vma)
                return ret;
 
+       pgoff = page_to_pgoff(page);
        anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
                struct vm_area_struct *vma = avc->vma;
                unsigned long address = vma_address(page, vma);
@@ -1674,7 +1678,7 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc)
 static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
 {
        struct address_space *mapping = page->mapping;
-       pgoff_t pgoff = page_to_pgoff(page);
+       pgoff_t pgoff;
        struct vm_area_struct *vma;
        int ret = SWAP_AGAIN;
 
@@ -1682,13 +1686,15 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
         * The page lock not only makes sure that page->mapping cannot
         * suddenly be NULLified by truncation, it makes sure that the
         * structure at mapping cannot be freed and reused yet,
-        * so we can safely take mapping->i_mmap_mutex.
+        * so we can safely take mapping->i_mmap_rwsem.
         */
        VM_BUG_ON_PAGE(!PageLocked(page), page);
 
        if (!mapping)
                return ret;
-       mutex_lock(&mapping->i_mmap_mutex);
+
+       pgoff = page_to_pgoff(page);
+       i_mmap_lock_read(mapping);
        vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
                unsigned long address = vma_address(page, vma);
 
@@ -1709,9 +1715,8 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
                goto done;
 
        ret = rwc->file_nonlinear(page, mapping, rwc->arg);
-
 done:
-       mutex_unlock(&mapping->i_mmap_mutex);
+       i_mmap_unlock_read(mapping);
        return ret;
 }