nohz: Rename ts->idle_tick to ts->last_tick

[cascardo/linux.git] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 6a3a5b9..73cc490 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -271,10 +271,10 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
-static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
+static void tick_nohz_stop_sched_tick(struct tick_sched *ts, ktime_t now)
 {
        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
-       ktime_t last_update, expires, now;
+       ktime_t last_update, expires;
        struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
        u64 time_delta;
        int cpu;
@@ -282,8 +282,6 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
        cpu = smp_processor_id();
        ts = &per_cpu(tick_cpu_sched, cpu);
 
-       now = tick_nohz_start_idle(cpu, ts);
-
        /*
         * If this cpu is offline and it is the one which updates
         * jiffies, then give up the assignment and let it be taken by
@@ -402,9 +400,8 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
                if (!ts->tick_stopped) {
                        select_nohz_load_balancer(1);
 
-                       ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
+                       ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
                        ts->tick_stopped = 1;
-                       ts->idle_jiffies = last_jiffies;
                }
 
                ts->idle_sleeps++;
@@ -444,6 +441,18 @@ out:
        ts->sleep_length = ktime_sub(dev->next_event, now);
 }
 
+static void __tick_nohz_idle_enter(struct tick_sched *ts)
+{
+       ktime_t now;
+       int was_stopped = ts->tick_stopped;
+
+       now = tick_nohz_start_idle(smp_processor_id(), ts);
+       tick_nohz_stop_sched_tick(ts, now);
+
+       if (!was_stopped && ts->tick_stopped)
+               ts->idle_jiffies = ts->last_jiffies;
+}
+
 /**
  * tick_nohz_idle_enter - stop the idle tick from the idle task
  *
@@ -479,7 +488,7 @@ void tick_nohz_idle_enter(void)
         * update of the idle time accounting in tick_nohz_start_idle().
         */
        ts->inidle = 1;
-       tick_nohz_stop_sched_tick(ts);
+       __tick_nohz_idle_enter(ts);
 
        local_irq_enable();
 }
@@ -499,7 +508,7 @@ void tick_nohz_irq_exit(void)
        if (!ts->inidle)
                return;
 
-       tick_nohz_stop_sched_tick(ts);
+       __tick_nohz_idle_enter(ts);
 }
 
 /**
@@ -517,7 +526,7 @@ ktime_t tick_nohz_get_sleep_length(void)
 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 {
        hrtimer_cancel(&ts->sched_timer);
-       hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
+       hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 
        while (1) {
                /* Forward the time to expire in the future */
@@ -540,6 +549,41 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
        }
 }
 
+static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
+{
+       /* Update jiffies first */
+       select_nohz_load_balancer(0);
+       tick_do_update_jiffies64(now);
+       update_cpu_load_nohz();
+
+       touch_softlockup_watchdog();
+       /*
+        * Cancel the scheduled timer and restore the tick
+        */
+       ts->tick_stopped  = 0;
+       ts->idle_exittime = now;
+
+       tick_nohz_restart(ts, now);
+}
+
+static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
+{
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
+       unsigned long ticks;
+       /*
+        * We stopped the tick in idle. Update process times would miss the
+        * time we slept as update_process_times does only a 1 tick
+        * accounting. Enforce that this is accounted to idle !
+        */
+       ticks = jiffies - ts->idle_jiffies;
+       /*
+        * We might be one off. Do not randomly account a huge number of ticks!
+        */
+       if (ticks && ticks < LONG_MAX)
+               account_idle_ticks(ticks);
+#endif
+}
+
 /**
  * tick_nohz_idle_exit - restart the idle tick from the idle task
  *
@@ -551,9 +595,6 @@ void tick_nohz_idle_exit(void)
 {
        int cpu = smp_processor_id();
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-       unsigned long ticks;
-#endif
        ktime_t now;
 
        local_irq_disable();
@@ -568,38 +609,11 @@ void tick_nohz_idle_exit(void)
        if (ts->idle_active)
                tick_nohz_stop_idle(cpu, now);
 
-       if (!ts->tick_stopped) {
-               local_irq_enable();
-               return;
+       if (ts->tick_stopped) {
+               tick_nohz_restart_sched_tick(ts, now);
+               tick_nohz_account_idle_ticks(ts);
        }
 
-       /* Update jiffies first */
-       select_nohz_load_balancer(0);
-       tick_do_update_jiffies64(now);
-
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-       /*
-        * We stopped the tick in idle. Update process times would miss the
-        * time we slept as update_process_times does only a 1 tick
-        * accounting. Enforce that this is accounted to idle !
-        */
-       ticks = jiffies - ts->idle_jiffies;
-       /*
-        * We might be one off. Do not randomly account a huge number of ticks!
-        */
-       if (ticks && ticks < LONG_MAX)
-               account_idle_ticks(ticks);
-#endif
-
-       touch_softlockup_watchdog();
-       /*
-        * Cancel the scheduled timer and restore the tick
-        */
-       ts->tick_stopped  = 0;
-       ts->idle_exittime = now;
-
-       tick_nohz_restart(ts, now);
-
        local_irq_enable();
 }
 
@@ -803,7 +817,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
                 */
                if (ts->tick_stopped) {
                        touch_softlockup_watchdog();
-                       ts->idle_jiffies++;
+                       if (idle_cpu(cpu))
+                               ts->idle_jiffies++;
                }
                update_process_times(user_mode(regs));
                profile_tick(CPU_PROFILING);
@@ -814,6 +829,16 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
        return HRTIMER_RESTART;
 }
 
+static int sched_skew_tick;
+
+static int __init skew_tick(char *str)
+{
+       get_option(&str, &sched_skew_tick);
+
+       return 0;
+}
+early_param("skew_tick", skew_tick);
+
 /**
  * tick_setup_sched_timer - setup the tick emulation timer
  */
@@ -831,6 +856,14 @@ void tick_setup_sched_timer(void)
        /* Get the next period (per cpu) */
        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
 
+       /* Offset the tick to avert xtime_lock contention. */
+       if (sched_skew_tick) {
+               u64 offset = ktime_to_ns(tick_period) >> 1;
+               do_div(offset, num_possible_cpus());
+               offset *= smp_processor_id();
+               hrtimer_add_expires_ns(&ts->sched_timer, offset);
+       }
+
        for (;;) {
                hrtimer_forward(&ts->sched_timer, now, tick_period);
                hrtimer_start_expires(&ts->sched_timer,