mm: memcontrol: group kmem init and exit functions together

[cascardo/linux.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 2aec424..05dd182 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -106,8 +106,6 @@ struct scan_control {
        unsigned long nr_reclaimed;
 };
 
-#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
-
 #ifdef ARCH_HAS_PREFETCH
 #define prefetch_prev_lru_page(_page, _base, _field)                   \
        do {                                                            \
@@ -197,11 +195,13 @@ static unsigned long zone_reclaimable_pages(struct zone *zone)
        unsigned long nr;
 
        nr = zone_page_state(zone, NR_ACTIVE_FILE) +
-            zone_page_state(zone, NR_INACTIVE_FILE);
+            zone_page_state(zone, NR_INACTIVE_FILE) +
+            zone_page_state(zone, NR_ISOLATED_FILE);
 
        if (get_nr_swap_pages() > 0)
                nr += zone_page_state(zone, NR_ACTIVE_ANON) +
-                     zone_page_state(zone, NR_INACTIVE_ANON);
+                     zone_page_state(zone, NR_INACTIVE_ANON) +
+                     zone_page_state(zone, NR_ISOLATED_ANON);
 
        return nr;
 }
@@ -411,7 +411,7 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
        struct shrinker *shrinker;
        unsigned long freed = 0;
 
-       if (memcg && !memcg_kmem_is_active(memcg))
+       if (memcg && !memcg_kmem_online(memcg))
                return 0;
 
        if (nr_scanned == 0)
@@ -594,7 +594,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
                        /* synchronous write or broken a_ops? */
                        ClearPageReclaim(page);
                }
-               trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
+               trace_mm_vmscan_writepage(page);
                inc_zone_page_state(page, NR_VMSCAN_WRITE);
                return PAGE_SUCCESS;
        }
@@ -906,6 +906,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                int may_enter_fs;
                enum page_references references = PAGEREF_RECLAIM_CLEAN;
                bool dirty, writeback;
+               bool lazyfree = false;
+               int ret = SWAP_SUCCESS;
 
                cond_resched();
 
@@ -1049,6 +1051,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                                goto keep_locked;
                        if (!add_to_swap(page, page_list))
                                goto activate_locked;
+                       lazyfree = true;
                        may_enter_fs = 1;
 
                        /* Adding to swap updated mapping */
@@ -1060,14 +1063,17 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                 * processes. Try to unmap it here.
                 */
                if (page_mapped(page) && mapping) {
-                       switch (try_to_unmap(page,
-                                       ttu_flags|TTU_BATCH_FLUSH)) {
+                       switch (ret = try_to_unmap(page, lazyfree ?
+                               (ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) :
+                               (ttu_flags | TTU_BATCH_FLUSH))) {
                        case SWAP_FAIL:
                                goto activate_locked;
                        case SWAP_AGAIN:
                                goto keep_locked;
                        case SWAP_MLOCK:
                                goto cull_mlocked;
+                       case SWAP_LZFREE:
+                               goto lazyfree;
                        case SWAP_SUCCESS:
                                ; /* try to free the page below */
                        }
@@ -1174,6 +1180,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                        }
                }
 
+lazyfree:
                if (!mapping || !__remove_mapping(mapping, page, true))
                        goto keep_locked;
 
@@ -1184,8 +1191,11 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                 * we obviously don't have to worry about waking up a process
                 * waiting on the page lock, because there are no references.
                 */
-               __clear_page_locked(page);
+               __ClearPageLocked(page);
 free_it:
+               if (ret == SWAP_LZFREE)
+                       count_vm_event(PGLAZYFREED);
+
                nr_reclaimed++;
 
                /*
@@ -1426,6 +1436,7 @@ int isolate_lru_page(struct page *page)
        int ret = -EBUSY;
 
        VM_BUG_ON_PAGE(!page_count(page), page);
+       VM_BUG_ON_PAGE(PageTail(page), page);
 
        if (PageLRU(page)) {
                struct zone *zone = page_zone(page);
@@ -1691,11 +1702,8 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
            current_may_throttle())
                wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10);
 
-       trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id,
-               zone_idx(zone),
-               nr_scanned, nr_reclaimed,
-               sc->priority,
-               trace_shrink_flags(file));
+       trace_mm_vmscan_lru_shrink_inactive(zone, nr_scanned, nr_reclaimed,
+                       sc->priority, file);
        return nr_reclaimed;
 }
 
@@ -2046,10 +2054,16 @@ static void get_scan_count(struct lruvec *lruvec, int swappiness,
        }
 
        /*
-        * There is enough inactive page cache, do not reclaim
-        * anything from the anonymous working set right now.
+        * If there is enough inactive page cache, i.e. if the size of the
+        * inactive list is greater than that of the active list *and* the
+        * inactive list actually has some pages to scan on this priority, we
+        * do not reclaim anything from the anonymous working set right now.
+        * Without the second condition we could end up never scanning an
+        * lruvec even if it has plenty of old anonymous pages unless the
+        * system is under heavy pressure.
         */
-       if (!inactive_file_is_low(lruvec)) {
+       if (!inactive_file_is_low(lruvec) &&
+           get_lru_size(lruvec, LRU_INACTIVE_FILE) >> sc->priority) {
                scan_balance = SCAN_FILE;
                goto out;
        }
@@ -2393,6 +2407,7 @@ static bool shrink_zone(struct zone *zone, struct scan_control *sc,
                memcg = mem_cgroup_iter(root, NULL, &reclaim);
                do {
                        unsigned long lru_pages;
+                       unsigned long reclaimed;
                        unsigned long scanned;
                        struct lruvec *lruvec;
                        int swappiness;
@@ -2405,6 +2420,7 @@ static bool shrink_zone(struct zone *zone, struct scan_control *sc,
 
                        lruvec = mem_cgroup_zone_lruvec(zone, memcg);
                        swappiness = mem_cgroup_swappiness(memcg);
+                       reclaimed = sc->nr_reclaimed;
                        scanned = sc->nr_scanned;
 
                        shrink_lruvec(lruvec, swappiness, sc, &lru_pages);
@@ -2415,6 +2431,11 @@ static bool shrink_zone(struct zone *zone, struct scan_control *sc,
                                            memcg, sc->nr_scanned - scanned,
                                            lru_pages);
 
+                       /* Record the group's reclaim efficiency */
+                       vmpressure(sc->gfp_mask, memcg, false,
+                                  sc->nr_scanned - scanned,
+                                  sc->nr_reclaimed - reclaimed);
+
                        /*
                         * Direct reclaim and kswapd have to scan all memory
                         * cgroups to fulfill the overall scan target for the
@@ -2446,7 +2467,8 @@ static bool shrink_zone(struct zone *zone, struct scan_control *sc,
                        reclaim_state->reclaimed_slab = 0;
                }
 
-               vmpressure(sc->gfp_mask, sc->target_mem_cgroup,
+               /* Record the subtree's reclaim efficiency */
+               vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
                           sc->nr_scanned - nr_scanned,
                           sc->nr_reclaimed - nr_reclaimed);