timers: Prevent base clock rewind when forwarding clock

[cascardo/linux.git] / kernel / time / timer.c

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 32bf6f7..7c446fb 100644 (file)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -943,7 +943,14 @@ static struct timer_base *lock_timer_base(struct timer_list *timer,
 {
        for (;;) {
                struct timer_base *base;
-               u32 tf = timer->flags;
+               u32 tf;
+
+               /*
+                * We need to use READ_ONCE() here, otherwise the compiler
+                * might re-read @tf between the check for TIMER_MIGRATING
+                * and spin_lock().
+                */
+               tf = READ_ONCE(timer->flags);
 
                if (!(tf & TIMER_MIGRATING)) {
                        base = get_timer_base(tf);
@@ -964,6 +971,8 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
        unsigned long clk = 0, flags;
        int ret = 0;
 
+       BUG_ON(!timer->function);
+
        /*
         * This is a common optimization triggered by the networking code - if
         * the timer is re-modified to have the same timeout or ends up in the
@@ -972,13 +981,16 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
        if (timer_pending(timer)) {
                if (timer->expires == expires)
                        return 1;
+
                /*
-                * Take the current timer_jiffies of base, but without holding
-                * the lock!
+                * We lock timer base and calculate the bucket index right
+                * here. If the timer ends up in the same bucket, then we
+                * just update the expiry time and avoid the whole
+                * dequeue/enqueue dance.
                 */
-               base = get_timer_base(timer->flags);
-               clk = base->clk;
+               base = lock_timer_base(timer, &flags);
 
+               clk = base->clk;
                idx = calc_wheel_index(expires, clk);
 
                /*
@@ -988,14 +1000,14 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
                 */
                if (idx == timer_get_idx(timer)) {
                        timer->expires = expires;
-                       return 1;
+                       ret = 1;
+                       goto out_unlock;
                }
+       } else {
+               base = lock_timer_base(timer, &flags);
        }
 
        timer_stats_timer_set_start_info(timer);
-       BUG_ON(!timer->function);
-
-       base = lock_timer_base(timer, &flags);
 
        ret = detach_if_pending(timer, base, false);
        if (!ret && pending_only)
@@ -1028,9 +1040,10 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
        timer->expires = expires;
        /*
         * If 'idx' was calculated above and the base time did not advance
-        * between calculating 'idx' and taking the lock, only enqueue_timer()
-        * and trigger_dyntick_cpu() is required. Otherwise we need to
-        * (re)calculate the wheel index via internal_add_timer().
+        * between calculating 'idx' and possibly switching the base, only
+        * enqueue_timer() and trigger_dyntick_cpu() is required. Otherwise
+        * we need to (re)calculate the wheel index via
+        * internal_add_timer().
         */
        if (idx != UINT_MAX && clk == base->clk) {
                enqueue_timer(base, timer, idx);
@@ -1510,12 +1523,16 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
        is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
        base->next_expiry = nextevt;
        /*
-        * We have a fresh next event. Check whether we can forward the base:
+        * We have a fresh next event. Check whether we can forward the
+        * base. We can only do that when @basej is past base->clk
+        * otherwise we might rewind base->clk.
         */
-       if (time_after(nextevt, jiffies))
-               base->clk = jiffies;
-       else if (time_after(nextevt, base->clk))
-               base->clk = nextevt;
+       if (time_after(basej, base->clk)) {
+               if (time_after(nextevt, basej))
+                       base->clk = basej;
+               else if (time_after(nextevt, base->clk))
+                       base->clk = nextevt;
+       }
 
        if (time_before_eq(nextevt, basej)) {
                expires = basem;
@@ -1633,7 +1650,7 @@ static inline void __run_timers(struct timer_base *base)
 /*
  * This function runs timers and the timer-tq in bottom half context.
  */
-static void run_timer_softirq(struct softirq_action *h)
+static __latent_entropy void run_timer_softirq(struct softirq_action *h)
 {
        struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);