mm: make lru_add_drain_all() selective

[cascardo/linux.git] / mm / swap.c

diff --git a/mm/swap.c b/mm/swap.c
index 62b78a6..759c3ca 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -31,6 +31,7 @@
 #include <linux/memcontrol.h>
 #include <linux/gfp.h>
 #include <linux/uio.h>
+#include <linux/hugetlb.h>
 
 #include "internal.h"
 
@@ -81,6 +82,19 @@ static void __put_compound_page(struct page *page)
 
 static void put_compound_page(struct page *page)
 {
+       /*
+        * hugetlbfs pages cannot be split from under us.  If this is a
+        * hugetlbfs page, check refcount on head page and release the page if
+        * the refcount becomes zero.
+        */
+       if (PageHuge(page)) {
+               page = compound_head(page);
+               if (put_page_testzero(page))
+                       __put_compound_page(page);
+
+               return;
+       }
+
        if (unlikely(PageTail(page))) {
                /* __split_huge_page_refcount can run under us */
                struct page *page_head = compound_trans_head(page);
@@ -184,38 +198,51 @@ bool __get_page_tail(struct page *page)
         * proper PT lock that already serializes against
         * split_huge_page().
         */
-       unsigned long flags;
        bool got = false;
-       struct page *page_head = compound_trans_head(page);
+       struct page *page_head;
 
-       if (likely(page != page_head && get_page_unless_zero(page_head))) {
+       /*
+        * If this is a hugetlbfs page it cannot be split under us.  Simply
+        * increment refcount for the head page.
+        */
+       if (PageHuge(page)) {
+               page_head = compound_head(page);
+               atomic_inc(&page_head->_count);
+               got = true;
+       } else {
+               unsigned long flags;
 
-               /* Ref to put_compound_page() comment. */
-               if (PageSlab(page_head)) {
+               page_head = compound_trans_head(page);
+               if (likely(page != page_head &&
+                                       get_page_unless_zero(page_head))) {
+
+                       /* Ref to put_compound_page() comment. */
+                       if (PageSlab(page_head)) {
+                               if (likely(PageTail(page))) {
+                                       __get_page_tail_foll(page, false);
+                                       return true;
+                               } else {
+                                       put_page(page_head);
+                                       return false;
+                               }
+                       }
+
+                       /*
+                        * page_head wasn't a dangling pointer but it
+                        * may not be a head page anymore by the time
+                        * we obtain the lock. That is ok as long as it
+                        * can't be freed from under us.
+                        */
+                       flags = compound_lock_irqsave(page_head);
+                       /* here __split_huge_page_refcount won't run anymore */
                        if (likely(PageTail(page))) {
                                __get_page_tail_foll(page, false);
-                               return true;
-                       } else {
-                               put_page(page_head);
-                               return false;
+                               got = true;
                        }
+                       compound_unlock_irqrestore(page_head, flags);
+                       if (unlikely(!got))
+                               put_page(page_head);
                }
-
-               /*
-                * page_head wasn't a dangling pointer but it
-                * may not be a head page anymore by the time
-                * we obtain the lock. That is ok as long as it
-                * can't be freed from under us.
-                */
-               flags = compound_lock_irqsave(page_head);
-               /* here __split_huge_page_refcount won't run anymore */
-               if (likely(PageTail(page))) {
-                       __get_page_tail_foll(page, false);
-                       got = true;
-               }
-               compound_unlock_irqrestore(page_head, flags);
-               if (unlikely(!got))
-                       put_page(page_head);
        }
        return got;
 }
@@ -405,6 +432,11 @@ static void activate_page_drain(int cpu)
                pagevec_lru_move_fn(pvec, __activate_page, NULL);
 }
 
+static bool need_activate_page_drain(int cpu)
+{
+       return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
+}
+
 void activate_page(struct page *page)
 {
        if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
@@ -422,6 +454,11 @@ static inline void activate_page_drain(int cpu)
 {
 }
 
+static bool need_activate_page_drain(int cpu)
+{
+       return false;
+}
+
 void activate_page(struct page *page)
 {
        struct zone *zone = page_zone(page);
@@ -674,12 +711,36 @@ static void lru_add_drain_per_cpu(struct work_struct *dummy)
        lru_add_drain();
 }
 
-/*
- * Returns 0 for success
- */
-int lru_add_drain_all(void)
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+void lru_add_drain_all(void)
 {
-       return schedule_on_each_cpu(lru_add_drain_per_cpu);
+       static DEFINE_MUTEX(lock);
+       static struct cpumask has_work;
+       int cpu;
+
+       mutex_lock(&lock);
+       get_online_cpus();
+       cpumask_clear(&has_work);
+
+       for_each_online_cpu(cpu) {
+               struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+
+               if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
+                   pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
+                   pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
+                   need_activate_page_drain(cpu)) {
+                       INIT_WORK(work, lru_add_drain_per_cpu);
+                       schedule_work_on(cpu, work);
+                       cpumask_set_cpu(cpu, &has_work);
+               }
+       }
+
+       for_each_cpu(cpu, &has_work)
+               flush_work(&per_cpu(lru_add_drain_work, cpu));
+
+       put_online_cpus();
+       mutex_unlock(&lock);
 }
 
 /*