Merge tag 'trace-seq-buf-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/roste...

[cascardo/linux.git] / kernel / sched / deadline.c

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 28fa9d9..e5db8c6 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -563,11 +563,6 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se)
 {
        struct hrtimer *timer = &dl_se->dl_timer;
 
-       if (hrtimer_active(timer)) {
-               hrtimer_try_to_cancel(timer);
-               return;
-       }
-
        hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
        timer->function = dl_task_timer;
 }
@@ -633,7 +628,7 @@ static void update_curr_dl(struct rq *rq)
 
        sched_rt_avg_update(rq, delta_exec);
 
-       dl_se->runtime -= delta_exec;
+       dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec;
        if (dl_runtime_exceeded(rq, dl_se)) {
                __dequeue_task_dl(rq, curr, 0);
                if (likely(start_dl_timer(dl_se, curr->dl.dl_boosted)))
@@ -933,7 +928,7 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)
        struct task_struct *curr;
        struct rq *rq;
 
-       if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
+       if (sd_flag != SD_BALANCE_WAKE)
                goto out;
 
        rq = cpu_rq(cpu);
@@ -1018,6 +1013,10 @@ static void start_hrtick_dl(struct rq *rq, struct task_struct *p)
 {
        hrtick_start(rq, p->dl.runtime);
 }
+#else /* !CONFIG_SCHED_HRTICK */
+static void start_hrtick_dl(struct rq *rq, struct task_struct *p)
+{
+}
 #endif
 
 static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq,
@@ -1071,10 +1070,8 @@ struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev)
        /* Running task will never be pushed. */
        dequeue_pushable_dl_task(rq, p);
 
-#ifdef CONFIG_SCHED_HRTICK
        if (hrtick_enabled(rq))
                start_hrtick_dl(rq, p);
-#endif
 
        set_post_schedule(rq);
 
@@ -1093,10 +1090,8 @@ static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued)
 {
        update_curr_dl(rq);
 
-#ifdef CONFIG_SCHED_HRTICK
        if (hrtick_enabled(rq) && queued && p->dl.runtime > 0)
                start_hrtick_dl(rq, p);
-#endif
 }
 
 static void task_fork_dl(struct task_struct *p)
@@ -1333,6 +1328,7 @@ static int push_dl_task(struct rq *rq)
 {
        struct task_struct *next_task;
        struct rq *later_rq;
+       int ret = 0;
 
        if (!rq->dl.overloaded)
                return 0;
@@ -1378,7 +1374,6 @@ retry:
                         * The task is still there. We don't try
                         * again, some other cpu will pull it when ready.
                         */
-                       dequeue_pushable_dl_task(rq, next_task);
                        goto out;
                }
 
@@ -1394,6 +1389,7 @@ retry:
        deactivate_task(rq, next_task, 0);
        set_task_cpu(next_task, later_rq->cpu);
        activate_task(later_rq, next_task, 0);
+       ret = 1;
 
        resched_curr(later_rq);
 
@@ -1402,7 +1398,7 @@ retry:
 out:
        put_task_struct(next_task);
 
-       return 1;
+       return ret;
 }
 
 static void push_dl_tasks(struct rq *rq)
@@ -1508,7 +1504,7 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
            p->nr_cpus_allowed > 1 &&
            dl_task(rq->curr) &&
            (rq->curr->nr_cpus_allowed < 2 ||
-            dl_entity_preempt(&rq->curr->dl, &p->dl))) {
+            !dl_entity_preempt(&p->dl, &rq->curr->dl))) {
                push_dl_tasks(rq);
        }
 }
@@ -1517,10 +1513,33 @@ static void set_cpus_allowed_dl(struct task_struct *p,
                                const struct cpumask *new_mask)
 {
        struct rq *rq;
+       struct root_domain *src_rd;
        int weight;
 
        BUG_ON(!dl_task(p));
 
+       rq = task_rq(p);
+       src_rd = rq->rd;
+       /*
+        * Migrating a SCHED_DEADLINE task between exclusive
+        * cpusets (different root_domains) entails a bandwidth
+        * update. We already made space for us in the destination
+        * domain (see cpuset_can_attach()).
+        */
+       if (!cpumask_intersects(src_rd->span, new_mask)) {
+               struct dl_bw *src_dl_b;
+
+               src_dl_b = dl_bw_of(cpu_of(rq));
+               /*
+                * We now free resources of the root_domain we are migrating
+                * off. In the worst case, sched_setattr() may temporary fail
+                * until we complete the update.
+                */
+               raw_spin_lock(&src_dl_b->lock);
+               __dl_clear(src_dl_b, p->dl.dl_bw);
+               raw_spin_unlock(&src_dl_b->lock);
+       }
+
        /*
         * Update only if the task is actually running (i.e.,
         * it is on the rq AND it is not throttled).
@@ -1537,8 +1556,6 @@ static void set_cpus_allowed_dl(struct task_struct *p,
        if ((p->nr_cpus_allowed > 1) == (weight > 1))
                return;
 
-       rq = task_rq(p);
-
        /*
         * The process used to be able to migrate OR it can now migrate
         */
@@ -1586,22 +1603,48 @@ void init_sched_dl_class(void)
 
 #endif /* CONFIG_SMP */
 
+/*
+ *  Ensure p's dl_timer is cancelled. May drop rq->lock for a while.
+ */
+static void cancel_dl_timer(struct rq *rq, struct task_struct *p)
+{
+       struct hrtimer *dl_timer = &p->dl.dl_timer;
+
+       /* Nobody will change task's class if pi_lock is held */
+       lockdep_assert_held(&p->pi_lock);
+
+       if (hrtimer_active(dl_timer)) {
+               int ret = hrtimer_try_to_cancel(dl_timer);
+
+               if (unlikely(ret == -1)) {
+                       /*
+                        * Note, p may migrate OR new deadline tasks
+                        * may appear in rq when we are unlocking it.
+                        * A caller of us must be fine with that.
+                        */
+                       raw_spin_unlock(&rq->lock);
+                       hrtimer_cancel(dl_timer);
+                       raw_spin_lock(&rq->lock);
+               }
+       }
+}
+
 static void switched_from_dl(struct rq *rq, struct task_struct *p)
 {
-       if (hrtimer_active(&p->dl.dl_timer) && !dl_policy(p->policy))
-               hrtimer_try_to_cancel(&p->dl.dl_timer);
+       cancel_dl_timer(rq, p);
 
        __dl_clear_params(p);
 
-#ifdef CONFIG_SMP
        /*
         * Since this might be the only -deadline task on the rq,
         * this is the right place to try to pull some other one
         * from an overloaded cpu, if any.
         */
-       if (!rq->dl.dl_nr_running)
-               pull_dl_task(rq);
-#endif
+       if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)
+               return;
+
+       if (pull_dl_task(rq))
+               resched_curr(rq);
 }
 
 /*
@@ -1622,7 +1665,8 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)
 
        if (task_on_rq_queued(p) && rq->curr != p) {
 #ifdef CONFIG_SMP
-               if (rq->dl.overloaded && push_dl_task(rq) && rq != task_rq(p))
+               if (p->nr_cpus_allowed > 1 && rq->dl.overloaded &&
+                       push_dl_task(rq) && rq != task_rq(p))
                        /* Only reschedule if pushing failed */
                        check_resched = 0;
 #endif /* CONFIG_SMP */
@@ -1704,3 +1748,12 @@ const struct sched_class dl_sched_class = {
 
        .update_curr            = update_curr_dl,
 };
+
+#ifdef CONFIG_SCHED_DEBUG
+extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);
+
+void print_dl_stats(struct seq_file *m, int cpu)
+{
+       print_dl_rq(m, cpu, &cpu_rq(cpu)->dl);
+}
+#endif /* CONFIG_SCHED_DEBUG */