rcu: Shorten expedited_workdone* to exp_workdone*

[cascardo/linux.git] / kernel / rcu / tree.c

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 687d8a5..89f0287 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -370,6 +370,21 @@ void rcu_all_qs(void)
                rcu_momentary_dyntick_idle();
                local_irq_restore(flags);
        }
+       if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))) {
+               /*
+                * Yes, we just checked a per-CPU variable with preemption
+                * enabled, so we might be migrated to some other CPU at
+                * this point.  That is OK because in that case, the
+                * migration will supply the needed quiescent state.
+                * We might end up needlessly disabling preemption and
+                * invoking rcu_sched_qs() on the destination CPU, but
+                * the probability and cost are both quite low, so this
+                * should not be a problem in practice.
+                */
+               preempt_disable();
+               rcu_sched_qs();
+               preempt_enable();
+       }
        this_cpu_inc(rcu_qs_ctr);
        barrier(); /* Avoid RCU read-side critical sections leaking up. */
 }
@@ -3569,10 +3584,19 @@ static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp,
                               atomic_long_t *stat, unsigned long s)
 {
        if (rcu_exp_gp_seq_done(rsp, s)) {
-               if (rnp)
+               trace_rcu_exp_grace_period(rsp->name, s, TPS("done"));
+               if (rnp) {
+                       trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
+                                                 rnp->grplo, rnp->grphi,
+                                                 TPS("rel"));
                        mutex_unlock(&rnp->exp_funnel_mutex);
-               else if (rdp)
+               } else if (rdp) {
+                       trace_rcu_exp_funnel_lock(rsp->name,
+                                                 rdp->mynode->level + 1,
+                                                 rdp->cpu, rdp->cpu,
+                                                 TPS("rel"));
                        mutex_unlock(&rdp->exp_funnel_mutex);
+               }
                /* Ensure test happens before caller kfree(). */
                smp_mb__before_atomic(); /* ^^^ */
                atomic_long_inc(stat);
@@ -3592,22 +3616,6 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
        struct rcu_node *rnp0;
        struct rcu_node *rnp1 = NULL;
 
-       /*
-        * First try directly acquiring the root lock in order to reduce
-        * latency in the common case where expedited grace periods are
-        * rare.  We check mutex_is_locked() to avoid pathological levels of
-        * memory contention on ->exp_funnel_mutex in the heavy-load case.
-        */
-       rnp0 = rcu_get_root(rsp);
-       if (!mutex_is_locked(&rnp0->exp_funnel_mutex)) {
-               if (mutex_trylock(&rnp0->exp_funnel_mutex)) {
-                       if (sync_exp_work_done(rsp, rnp0, NULL,
-                                              &rdp->expedited_workdone0, s))
-                               return NULL;
-                       return rnp0;
-               }
-       }
-
        /*
         * Each pass through the following loop works its way
         * up the rcu_node tree, returning if others have done the
@@ -3616,23 +3624,33 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
         * can be inexact, as it is just promoting locality and is not
         * strictly needed for correctness.
         */
-       if (sync_exp_work_done(rsp, NULL, NULL, &rdp->expedited_workdone1, s))
+       if (sync_exp_work_done(rsp, NULL, NULL, &rdp->exp_workdone1, s))
                return NULL;
        mutex_lock(&rdp->exp_funnel_mutex);
+       trace_rcu_exp_funnel_lock(rsp->name, rdp->mynode->level + 1,
+                                 rdp->cpu, rdp->cpu, TPS("acq"));
        rnp0 = rdp->mynode;
        for (; rnp0 != NULL; rnp0 = rnp0->parent) {
-               if (sync_exp_work_done(rsp, rnp1, rdp,
-                                      &rdp->expedited_workdone2, s))
+               if (sync_exp_work_done(rsp, rnp1, rdp, &rdp->exp_workdone2, s))
                        return NULL;
                mutex_lock(&rnp0->exp_funnel_mutex);
-               if (rnp1)
+               trace_rcu_exp_funnel_lock(rsp->name, rnp0->level,
+                                         rnp0->grplo, rnp0->grphi, TPS("acq"));
+               if (rnp1) {
+                       trace_rcu_exp_funnel_lock(rsp->name, rnp1->level,
+                                                 rnp1->grplo, rnp1->grphi,
+                                                 TPS("rel"));
                        mutex_unlock(&rnp1->exp_funnel_mutex);
-               else
+               } else {
+                       trace_rcu_exp_funnel_lock(rsp->name,
+                                                 rdp->mynode->level + 1,
+                                                 rdp->cpu, rdp->cpu,
+                                                 TPS("rel"));
                        mutex_unlock(&rdp->exp_funnel_mutex);
+               }
                rnp1 = rnp0;
        }
-       if (sync_exp_work_done(rsp, rnp1, rdp,
-                              &rdp->expedited_workdone3, s))
+       if (sync_exp_work_done(rsp, rnp1, rdp, &rdp->exp_workdone3, s))
                return NULL;
        return rnp1;
 }
@@ -3788,7 +3806,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
                                ndetected++;
                                rdp = per_cpu_ptr(rsp->rda, cpu);
                                pr_cont(" %d-%c%c%c", cpu,
-                                       "O."[cpu_online(cpu)],
+                                       "O."[!!cpu_online(cpu)],
                                        "o."[!!(rdp->grpmask & rnp->expmaskinit)],
                                        "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);
                        }
@@ -3857,16 +3875,21 @@ void synchronize_sched_expedited(void)
 
        /* Take a snapshot of the sequence number.  */
        s = rcu_exp_gp_seq_snap(rsp);
+       trace_rcu_exp_grace_period(rsp->name, s, TPS("snap"));
 
        rnp = exp_funnel_lock(rsp, s);
        if (rnp == NULL)
                return;  /* Someone else did our work for us. */
 
        rcu_exp_gp_seq_start(rsp);
+       trace_rcu_exp_grace_period(rsp->name, s, TPS("start"));
        sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);
        synchronize_sched_expedited_wait(rsp);
 
        rcu_exp_gp_seq_end(rsp);
+       trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));
+       trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
+                                 rnp->grplo, rnp->grphi, TPS("rel"));
        mutex_unlock(&rnp->exp_funnel_mutex);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);