writeback: trace global_dirty_state

[cascardo/linux.git] / mm / page-writeback.c

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 5f3e1b4..f9d9f54 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -36,6 +36,11 @@
 #include <linux/pagevec.h>
 #include <trace/events/writeback.h>
 
+/*
+ * Sleep at most 200ms at a time in balance_dirty_pages().
+ */
+#define MAX_PAUSE              max(HZ/5, 1)
+
 /*
  * Estimate write bandwidth at 200ms intervals.
  */
@@ -116,6 +121,7 @@ EXPORT_SYMBOL(laptop_mode);
 
 /* End of sysctl-exported parameters */
 
+unsigned long global_dirty_limit;
 
 /*
  * Scale the writeback cache size proportional to the relative writeout speeds.
@@ -398,6 +404,11 @@ unsigned long determine_dirtyable_memory(void)
        return x + 1;   /* Ensure that we never return 0 */
 }
 
+static unsigned long hard_dirty_limit(unsigned long thresh)
+{
+       return max(thresh, global_dirty_limit);
+}
+
 /*
  * global_dirty_limits - background-writeback and dirty-throttling thresholds
  *
@@ -436,6 +447,7 @@ void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
        }
        *pbackground = background;
        *pdirty = dirty;
+       trace_global_dirty_state(background, dirty);
 }
 
 /**
@@ -516,7 +528,67 @@ out:
        bdi->avg_write_bandwidth = avg;
 }
 
+/*
+ * The global dirtyable memory and dirty threshold could be suddenly knocked
+ * down by a large amount (eg. on the startup of KVM in a swapless system).
+ * This may throw the system into deep dirty exceeded state and throttle
+ * heavy/light dirtiers alike. To retain good responsiveness, maintain
+ * global_dirty_limit for tracking slowly down to the knocked down dirty
+ * threshold.
+ */
+static void update_dirty_limit(unsigned long thresh, unsigned long dirty)
+{
+       unsigned long limit = global_dirty_limit;
+
+       /*
+        * Follow up in one step.
+        */
+       if (limit < thresh) {
+               limit = thresh;
+               goto update;
+       }
+
+       /*
+        * Follow down slowly. Use the higher one as the target, because thresh
+        * may drop below dirty. This is exactly the reason to introduce
+        * global_dirty_limit which is guaranteed to lie above the dirty pages.
+        */
+       thresh = max(thresh, dirty);
+       if (limit > thresh) {
+               limit -= (limit - thresh) >> 5;
+               goto update;
+       }
+       return;
+update:
+       global_dirty_limit = limit;
+}
+
+static void global_update_bandwidth(unsigned long thresh,
+                                   unsigned long dirty,
+                                   unsigned long now)
+{
+       static DEFINE_SPINLOCK(dirty_lock);
+       static unsigned long update_time;
+
+       /*
+        * check locklessly first to optimize away locking for the most time
+        */
+       if (time_before(now, update_time + BANDWIDTH_INTERVAL))
+               return;
+
+       spin_lock(&dirty_lock);
+       if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) {
+               update_dirty_limit(thresh, dirty);
+               update_time = now;
+       }
+       spin_unlock(&dirty_lock);
+}
+
 void __bdi_update_bandwidth(struct backing_dev_info *bdi,
+                           unsigned long thresh,
+                           unsigned long dirty,
+                           unsigned long bdi_thresh,
+                           unsigned long bdi_dirty,
                            unsigned long start_time)
 {
        unsigned long now = jiffies;
@@ -538,6 +610,9 @@ void __bdi_update_bandwidth(struct backing_dev_info *bdi,
        if (elapsed > HZ && time_before(bdi->bw_time_stamp, start_time))
                goto snapshot;
 
+       if (thresh)
+               global_update_bandwidth(thresh, dirty, now);
+
        bdi_update_write_bandwidth(bdi, elapsed, written);
 
 snapshot:
@@ -546,12 +621,17 @@ snapshot:
 }
 
 static void bdi_update_bandwidth(struct backing_dev_info *bdi,
+                                unsigned long thresh,
+                                unsigned long dirty,
+                                unsigned long bdi_thresh,
+                                unsigned long bdi_dirty,
                                 unsigned long start_time)
 {
        if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
                return;
        spin_lock(&bdi->wb.list_lock);
-       __bdi_update_bandwidth(bdi, start_time);
+       __bdi_update_bandwidth(bdi, thresh, dirty, bdi_thresh, bdi_dirty,
+                              start_time);
        spin_unlock(&bdi->wb.list_lock);
 }
 
@@ -630,7 +710,8 @@ static void balance_dirty_pages(struct address_space *mapping,
                if (!bdi->dirty_exceeded)
                        bdi->dirty_exceeded = 1;
 
-               bdi_update_bandwidth(bdi, start_time);
+               bdi_update_bandwidth(bdi, dirty_thresh, nr_dirty,
+                                    bdi_thresh, bdi_dirty, start_time);
 
                /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
                 * Unstable writes are a feature of certain networked
@@ -653,6 +734,29 @@ static void balance_dirty_pages(struct address_space *mapping,
                io_schedule_timeout(pause);
                trace_balance_dirty_wait(bdi);
 
+               dirty_thresh = hard_dirty_limit(dirty_thresh);
+               /*
+                * max-pause area. If dirty exceeded but still within this
+                * area, no need to sleep for more than 200ms: (a) 8 pages per
+                * 200ms is typically more than enough to curb heavy dirtiers;
+                * (b) the pause time limit makes the dirtiers more responsive.
+                */
+               if (nr_dirty < dirty_thresh +
+                              dirty_thresh / DIRTY_MAXPAUSE_AREA &&
+                   time_after(jiffies, start_time + MAX_PAUSE))
+                       break;
+               /*
+                * pass-good area. When some bdi gets blocked (eg. NFS server
+                * not responding), or write bandwidth dropped dramatically due
+                * to concurrent reads, or dirty threshold suddenly dropped and
+                * the dirty pages cannot be brought down anytime soon (eg. on
+                * slow USB stick), at least let go of the good bdi's.
+                */
+               if (nr_dirty < dirty_thresh +
+                              dirty_thresh / DIRTY_PASSGOOD_AREA &&
+                   bdi_dirty < bdi_thresh)
+                       break;
+
                /*
                 * Increase the delay for each loop, up to our previous
                 * default of taking a 100ms nap.