rcu_momentary_dyntick_idle();
local_irq_restore(flags);
}
+ if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))) {
+ /*
+ * Yes, we just checked a per-CPU variable with preemption
+ * enabled, so we might be migrated to some other CPU at
+ * this point. That is OK because in that case, the
+ * migration will supply the needed quiescent state.
+ * We might end up needlessly disabling preemption and
+ * invoking rcu_sched_qs() on the destination CPU, but
+ * the probability and cost are both quite low, so this
+ * should not be a problem in practice.
+ */
+ preempt_disable();
+ rcu_sched_qs();
+ preempt_enable();
+ }
this_cpu_inc(rcu_qs_ctr);
barrier(); /* Avoid RCU read-side critical sections leaking up. */
}
idle_task(smp_processor_id());
trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0);
- ftrace_dump(DUMP_ORIG);
+ rcu_ftrace_dump(DUMP_ORIG);
WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
current->pid, current->comm,
idle->pid, idle->comm); /* must be idle task! */
trace_rcu_dyntick(TPS("Error on exit: not idle task"),
oldval, rdtp->dynticks_nesting);
- ftrace_dump(DUMP_ORIG);
+ rcu_ftrace_dump(DUMP_ORIG);
WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
current->pid, current->comm,
idle->pid, idle->comm); /* must be idle task! */
atomic_long_t *stat, unsigned long s)
{
if (rcu_exp_gp_seq_done(rsp, s)) {
- if (rnp)
+ trace_rcu_exp_grace_period(rsp->name, s, TPS("done"));
+ if (rnp) {
+ trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
+ rnp->grplo, rnp->grphi,
+ TPS("rel"));
mutex_unlock(&rnp->exp_funnel_mutex);
- else if (rdp)
+ } else if (rdp) {
+ trace_rcu_exp_funnel_lock(rsp->name,
+ rdp->mynode->level + 1,
+ rdp->cpu, rdp->cpu,
+ TPS("rel"));
mutex_unlock(&rdp->exp_funnel_mutex);
+ }
/* Ensure test happens before caller kfree(). */
smp_mb__before_atomic(); /* ^^^ */
atomic_long_inc(stat);
struct rcu_node *rnp0;
struct rcu_node *rnp1 = NULL;
- /*
- * First try directly acquiring the root lock in order to reduce
- * latency in the common case where expedited grace periods are
- * rare. We check mutex_is_locked() to avoid pathological levels of
- * memory contention on ->exp_funnel_mutex in the heavy-load case.
- */
- rnp0 = rcu_get_root(rsp);
- if (!mutex_is_locked(&rnp0->exp_funnel_mutex)) {
- if (mutex_trylock(&rnp0->exp_funnel_mutex)) {
- if (sync_exp_work_done(rsp, rnp0, NULL,
- &rdp->expedited_workdone0, s))
- return NULL;
- return rnp0;
- }
- }
-
/*
* Each pass through the following loop works its way
* up the rcu_node tree, returning if others have done the
* can be inexact, as it is just promoting locality and is not
* strictly needed for correctness.
*/
- if (sync_exp_work_done(rsp, NULL, NULL, &rdp->expedited_workdone1, s))
+ if (sync_exp_work_done(rsp, NULL, NULL, &rdp->exp_workdone1, s))
return NULL;
mutex_lock(&rdp->exp_funnel_mutex);
+ trace_rcu_exp_funnel_lock(rsp->name, rdp->mynode->level + 1,
+ rdp->cpu, rdp->cpu, TPS("acq"));
rnp0 = rdp->mynode;
for (; rnp0 != NULL; rnp0 = rnp0->parent) {
- if (sync_exp_work_done(rsp, rnp1, rdp,
- &rdp->expedited_workdone2, s))
+ if (sync_exp_work_done(rsp, rnp1, rdp, &rdp->exp_workdone2, s))
return NULL;
mutex_lock(&rnp0->exp_funnel_mutex);
- if (rnp1)
+ trace_rcu_exp_funnel_lock(rsp->name, rnp0->level,
+ rnp0->grplo, rnp0->grphi, TPS("acq"));
+ if (rnp1) {
+ trace_rcu_exp_funnel_lock(rsp->name, rnp1->level,
+ rnp1->grplo, rnp1->grphi,
+ TPS("rel"));
mutex_unlock(&rnp1->exp_funnel_mutex);
- else
+ } else {
+ trace_rcu_exp_funnel_lock(rsp->name,
+ rdp->mynode->level + 1,
+ rdp->cpu, rdp->cpu,
+ TPS("rel"));
mutex_unlock(&rdp->exp_funnel_mutex);
+ }
rnp1 = rnp0;
}
- if (sync_exp_work_done(rsp, rnp1, rdp,
- &rdp->expedited_workdone3, s))
+ if (sync_exp_work_done(rsp, rnp1, rdp, &rdp->exp_workdone3, s))
return NULL;
return rnp1;
}
if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
return;
+ if (rcu_is_cpu_rrupt_from_idle()) {
+ rcu_report_exp_rdp(&rcu_sched_state,
+ this_cpu_ptr(&rcu_sched_data), true);
+ return;
+ }
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, true);
resched_cpu(smp_processor_id());
}
rsp->name);
ndetected = 0;
rcu_for_each_leaf_node(rsp, rnp) {
- ndetected = rcu_print_task_exp_stall(rnp);
+ ndetected += rcu_print_task_exp_stall(rnp);
mask = 1;
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
struct rcu_data *rdp;
ndetected++;
rdp = per_cpu_ptr(rsp->rda, cpu);
pr_cont(" %d-%c%c%c", cpu,
- "O."[cpu_online(cpu)],
+ "O."[!!cpu_online(cpu)],
"o."[!!(rdp->grpmask & rnp->expmaskinit)],
"N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);
}
pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
jiffies - jiffies_start, rsp->expedited_sequence,
rnp_root->expmask, ".T"[!!rnp_root->exp_tasks]);
- if (!ndetected) {
+ if (ndetected) {
pr_err("blocking rcu_node structures:");
rcu_for_each_node_breadth_first(rsp, rnp) {
if (rnp == rnp_root)
/* Take a snapshot of the sequence number. */
s = rcu_exp_gp_seq_snap(rsp);
+ trace_rcu_exp_grace_period(rsp->name, s, TPS("snap"));
rnp = exp_funnel_lock(rsp, s);
if (rnp == NULL)
return; /* Someone else did our work for us. */
rcu_exp_gp_seq_start(rsp);
+ trace_rcu_exp_grace_period(rsp->name, s, TPS("start"));
sync_rcu_exp_select_cpus(rsp, sync_sched_exp_handler);
synchronize_sched_expedited_wait(rsp);
rcu_exp_gp_seq_end(rsp);
+ trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));
+ trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
+ rnp->grplo, rnp->grphi, TPS("rel"));
mutex_unlock(&rnp->exp_funnel_mutex);
}
EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
projects / cascardo / linux.git / blobdiff