Merge branches 'pm-cpufreq-fixes' and 'pm-sleep-fixes'

[cascardo/linux.git] / drivers / cpufreq / intel_pstate.c

diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 1c7b91c..4737520 100644 (file)
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -179,6 +179,7 @@ struct _pid {
 /**
  * struct cpudata -    Per CPU instance data storage
  * @cpu:               CPU number for this instance data
+ * @policy:            CPUFreq policy value
  * @update_util:       CPUFreq utility callback information
  * @update_util_set:   CPUFreq utility callback is set
  * @iowait_boost:      iowait-related boost fraction
@@ -201,6 +202,7 @@ struct _pid {
 struct cpudata {
        int cpu;
 
+       unsigned int policy;
        struct update_util_data update_util;
        bool   update_util_set;
 
@@ -225,7 +227,7 @@ struct cpudata {
 static struct cpudata **all_cpu_data;
 
 /**
- * struct pid_adjust_policy - Stores static PID configuration data
+ * struct pstate_adjust_policy - Stores static PID configuration data
  * @sample_rate_ms:    PID calculation sample rate in ms
  * @sample_rate_ns:    Sample rate calculation in ns
  * @deadband:          PID deadband
@@ -1142,10 +1144,8 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
        *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
 }
 
-static void intel_pstate_set_min_pstate(struct cpudata *cpu)
+static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
 {
-       int pstate = cpu->pstate.min_pstate;
-
        trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
        cpu->pstate.current_pstate = pstate;
        /*
@@ -1157,6 +1157,20 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu)
                      pstate_funcs.get_val(cpu, pstate));
 }
 
+static void intel_pstate_set_min_pstate(struct cpudata *cpu)
+{
+       intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
+}
+
+static void intel_pstate_max_within_limits(struct cpudata *cpu)
+{
+       int min_pstate, max_pstate;
+
+       update_turbo_state();
+       intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate);
+       intel_pstate_set_pstate(cpu, max_pstate);
+}
+
 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 {
        cpu->pstate.min_pstate = pstate_funcs.get_min();
@@ -1232,6 +1246,7 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
 {
        struct sample *sample = &cpu->sample;
        int32_t busy_frac, boost;
+       int target, avg_pstate;
 
        busy_frac = div_fp(sample->mperf, sample->tsc);
 
@@ -1242,7 +1257,26 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
                busy_frac = boost;
 
        sample->busy_scaled = busy_frac * 100;
-       return get_avg_pstate(cpu) - pid_calc(&cpu->pid, sample->busy_scaled);
+
+       target = limits->no_turbo || limits->turbo_disabled ?
+                       cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
+       target += target >> 2;
+       target = mul_fp(target, busy_frac);
+       if (target < cpu->pstate.min_pstate)
+               target = cpu->pstate.min_pstate;
+
+       /*
+        * If the average P-state during the previous cycle was higher than the
+        * current target, add 50% of the difference to the target to reduce
+        * possible performance oscillations and offset possible performance
+        * loss related to moving the workload from one CPU to another within
+        * a package/module.
+        */
+       avg_pstate = get_avg_pstate(cpu);
+       if (avg_pstate > target)
+               target += (avg_pstate - target) >> 1;
+
+       return target;
 }
 
 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
@@ -1305,7 +1339,8 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 
        from = cpu->pstate.current_pstate;
 
-       target_pstate = pstate_funcs.get_target_pstate(cpu);
+       target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
+               cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
 
        intel_pstate_update_pstate(cpu, target_pstate);
 
@@ -1471,7 +1506,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
        pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
                 policy->cpuinfo.max_freq, policy->max);
 
-       cpu = all_cpu_data[0];
+       cpu = all_cpu_data[policy->cpu];
+       cpu->policy = policy->policy;
+
        if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
            policy->max < policy->cpuinfo.max_freq &&
            policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
@@ -1479,7 +1516,7 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
                policy->max = policy->cpuinfo.max_freq;
        }
 
-       if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
+       if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
                limits = &performance_limits;
                if (policy->max >= policy->cpuinfo.max_freq) {
                        pr_debug("set performance\n");
@@ -1515,6 +1552,15 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
        limits->max_perf = round_up(limits->max_perf, FRAC_BITS);
 
  out:
+       if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
+               /*
+                * NOHZ_FULL CPUs need this as the governor callback may not
+                * be invoked on them.
+                */
+               intel_pstate_clear_update_util_hook(policy->cpu);
+               intel_pstate_max_within_limits(cpu);
+       }
+
        intel_pstate_set_update_util_hook(policy->cpu);
 
        intel_pstate_hwp_set_policy(policy);