drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #include <linux/kernel.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/module.h>
  16 #include <linux/ktime.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/tick.h>
  19 #include <linux/slab.h>
  20 #include <linux/sched.h>
  21 #include <linux/list.h>
  22 #include <linux/cpu.h>
  23 #include <linux/cpufreq.h>
  24 #include <linux/sysfs.h>
  25 #include <linux/types.h>
  26 #include <linux/fs.h>
  27 #include <linux/debugfs.h>
  28 #include <linux/acpi.h>
  29 #include <linux/vmalloc.h>
  30 #include <trace/events/power.h>
  31
  32 #include <asm/div64.h>
  33 #include <asm/msr.h>
  34 #include <asm/cpu_device_id.h>
  35 #include <asm/cpufeature.h>
  36
  37 #if IS_ENABLED(CONFIG_ACPI)
  38 #include <acpi/processor.h>
  39 #endif
  40
  41 #define BYT_RATIOS              0x66a
  42 #define BYT_VIDS                0x66b
  43 #define BYT_TURBO_RATIOS        0x66c
  44 #define BYT_TURBO_VIDS          0x66d
  45
  46 #define FRAC_BITS 8
  47 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  48 #define fp_toint(X) ((X) >> FRAC_BITS)
  49
  50 static inline int32_t mul_fp(int32_t x, int32_t y)
  51 {
  52         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  53 }
  54
  55 static inline int32_t div_fp(s64 x, s64 y)
  56 {
  57         return div64_s64((int64_t)x << FRAC_BITS, y);
  58 }
  59
  60 static inline int ceiling_fp(int32_t x)
  61 {
  62         int mask, ret;
  63
  64         ret = fp_toint(x);
  65         mask = (1 << FRAC_BITS) - 1;
  66         if (x & mask)
  67                 ret += 1;
  68         return ret;
  69 }
  70
  71 struct sample {
  72         int32_t core_pct_busy;
  73         u64 aperf;
  74         u64 mperf;
  75         u64 tsc;
  76         int freq;
  77         ktime_t time;
  78 };
  79
  80 struct pstate_data {
  81         int     current_pstate;
  82         int     min_pstate;
  83         int     max_pstate;
  84         int     max_pstate_physical;
  85         int     scaling;
  86         int     turbo_pstate;
  87 };
  88
  89 struct vid_data {
  90         int min;
  91         int max;
  92         int turbo;
  93         int32_t ratio;
  94 };
  95
  96 struct _pid {
  97         int setpoint;
  98         int32_t integral;
  99         int32_t p_gain;
 100         int32_t i_gain;
 101         int32_t d_gain;
 102         int deadband;
 103         int32_t last_err;
 104 };
 105
 106 struct cpudata {
 107         int cpu;
 108
 109         struct timer_list timer;
 110
 111         struct pstate_data pstate;
 112         struct vid_data vid;
 113         struct _pid pid;
 114
 115         ktime_t last_sample_time;
 116         u64     prev_aperf;
 117         u64     prev_mperf;
 118         u64     prev_tsc;
 119         struct sample sample;
 120 #if IS_ENABLED(CONFIG_ACPI)
 121         struct acpi_processor_performance acpi_perf_data;
 122 #endif
 123 };
 124
 125 static struct cpudata **all_cpu_data;
 126 struct pstate_adjust_policy {
 127         int sample_rate_ms;
 128         int deadband;
 129         int setpoint;
 130         int p_gain_pct;
 131         int d_gain_pct;
 132         int i_gain_pct;
 133 };
 134
 135 struct pstate_funcs {
 136         int (*get_max)(void);
 137         int (*get_max_physical)(void);
 138         int (*get_min)(void);
 139         int (*get_turbo)(void);
 140         int (*get_scaling)(void);
 141         void (*set)(struct cpudata*, int pstate);
 142         void (*get_vid)(struct cpudata *);
 143 };
 144
 145 struct cpu_defaults {
 146         struct pstate_adjust_policy pid_policy;
 147         struct pstate_funcs funcs;
 148 };
 149
 150 static struct pstate_adjust_policy pid_params;
 151 static struct pstate_funcs pstate_funcs;
 152 static int hwp_active;
 153 static int no_acpi_perf;
 154
 155 struct perf_limits {
 156         int no_turbo;
 157         int turbo_disabled;
 158         int max_perf_pct;
 159         int min_perf_pct;
 160         int32_t max_perf;
 161         int32_t min_perf;
 162         int max_policy_pct;
 163         int max_sysfs_pct;
 164         int min_policy_pct;
 165         int min_sysfs_pct;
 166         int max_perf_ctl;
 167         int min_perf_ctl;
 168 };
 169
 170 static struct perf_limits performance_limits = {
 171         .no_turbo = 0,
 172         .turbo_disabled = 0,
 173         .max_perf_pct = 100,
 174         .max_perf = int_tofp(1),
 175         .min_perf_pct = 100,
 176         .min_perf = int_tofp(1),
 177         .max_policy_pct = 100,
 178         .max_sysfs_pct = 100,
 179         .min_policy_pct = 0,
 180         .min_sysfs_pct = 0,
 181 };
 182
 183 static struct perf_limits powersave_limits = {
 184         .no_turbo = 0,
 185         .turbo_disabled = 0,
 186         .max_perf_pct = 100,
 187         .max_perf = int_tofp(1),
 188         .min_perf_pct = 0,
 189         .min_perf = 0,
 190         .max_policy_pct = 100,
 191         .max_sysfs_pct = 100,
 192         .min_policy_pct = 0,
 193         .min_sysfs_pct = 0,
 194         .max_perf_ctl = 0,
 195         .min_perf_ctl = 0,
 196 };
 197
 198 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
 199 static struct perf_limits *limits = &performance_limits;
 200 #else
 201 static struct perf_limits *limits = &powersave_limits;
 202 #endif
 203
 204 #if IS_ENABLED(CONFIG_ACPI)
 205 /*
 206  * The max target pstate ratio is a 8 bit value in both PLATFORM_INFO MSR and
 207  * in TURBO_RATIO_LIMIT MSR, which pstate driver stores in max_pstate and
 208  * max_turbo_pstate fields. The PERF_CTL MSR contains 16 bit value for P state
 209  * ratio, out of it only high 8 bits are used. For example 0x1700 is setting
 210  * target ratio 0x17. The _PSS control value stores in a format which can be
 211  * directly written to PERF_CTL MSR. But in intel_pstate driver this shift
 212  * occurs during write to PERF_CTL (E.g. for cores core_set_pstate()).
 213  * This function converts the _PSS control value to intel pstate driver format
 214  * for comparison and assignment.
 215  */
 216 static int convert_to_native_pstate_format(struct cpudata *cpu, int index)
 217 {
 218         return cpu->acpi_perf_data.states[index].control >> 8;
 219 }
 220
 221 static int intel_pstate_init_perf_limits(struct cpufreq_policy *policy)
 222 {
 223         struct cpudata *cpu;
 224         int ret;
 225         bool turbo_absent = false;
 226         int max_pstate_index;
 227         int min_pss_ctl, max_pss_ctl, turbo_pss_ctl;
 228         int i;
 229
 230         cpu = all_cpu_data[policy->cpu];
 231
 232         pr_debug("intel_pstate: default limits 0x%x 0x%x 0x%x\n",
 233                  cpu->pstate.min_pstate, cpu->pstate.max_pstate,
 234                  cpu->pstate.turbo_pstate);
 235
 236         if (!cpu->acpi_perf_data.shared_cpu_map &&
 237             zalloc_cpumask_var_node(&cpu->acpi_perf_data.shared_cpu_map,
 238                                     GFP_KERNEL, cpu_to_node(policy->cpu))) {
 239                 return -ENOMEM;
 240         }
 241
 242         ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
 243                                                   policy->cpu);
 244         if (ret)
 245                 return ret;
 246
 247         /*
 248          * Check if the control value in _PSS is for PERF_CTL MSR, which should
 249          * guarantee that the states returned by it map to the states in our
 250          * list directly.
 251          */
 252         if (cpu->acpi_perf_data.control_register.space_id !=
 253                                                 ACPI_ADR_SPACE_FIXED_HARDWARE)
 254                 return -EIO;
 255
 256         pr_debug("intel_pstate: CPU%u - ACPI _PSS perf data\n", policy->cpu);
 257         for (i = 0; i < cpu->acpi_perf_data.state_count; i++)
 258                 pr_debug("     %cP%d: %u MHz, %u mW, 0x%x\n",
 259                          (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
 260                          (u32) cpu->acpi_perf_data.states[i].core_frequency,
 261                          (u32) cpu->acpi_perf_data.states[i].power,
 262                          (u32) cpu->acpi_perf_data.states[i].control);
 263
 264         /*
 265          * If there is only one entry _PSS, simply ignore _PSS and continue as
 266          * usual without taking _PSS into account
 267          */
 268         if (cpu->acpi_perf_data.state_count < 2)
 269                 return 0;
 270
 271         turbo_pss_ctl = convert_to_native_pstate_format(cpu, 0);
 272         min_pss_ctl = convert_to_native_pstate_format(cpu,
 273                                         cpu->acpi_perf_data.state_count - 1);
 274         /* Check if there is a turbo freq in _PSS */
 275         if (turbo_pss_ctl <= cpu->pstate.max_pstate &&
 276             turbo_pss_ctl > cpu->pstate.min_pstate) {
 277                 pr_debug("intel_pstate: no turbo range exists in _PSS\n");
 278                 limits->no_turbo = limits->turbo_disabled = 1;
 279                 cpu->pstate.turbo_pstate = cpu->pstate.max_pstate;
 280                 turbo_absent = true;
 281         }
 282
 283         /* Check if the max non turbo p state < Intel P state max */
 284         max_pstate_index = turbo_absent ? 0 : 1;
 285         max_pss_ctl = convert_to_native_pstate_format(cpu, max_pstate_index);
 286         if (max_pss_ctl < cpu->pstate.max_pstate &&
 287             max_pss_ctl > cpu->pstate.min_pstate)
 288                 cpu->pstate.max_pstate = max_pss_ctl;
 289
 290         /* check If min perf > Intel P State min */
 291         if (min_pss_ctl > cpu->pstate.min_pstate &&
 292             min_pss_ctl < cpu->pstate.max_pstate) {
 293                 cpu->pstate.min_pstate = min_pss_ctl;
 294                 policy->cpuinfo.min_freq = min_pss_ctl * cpu->pstate.scaling;
 295         }
 296
 297         if (turbo_absent)
 298                 policy->cpuinfo.max_freq = cpu->pstate.max_pstate *
 299                                                 cpu->pstate.scaling;
 300         else {
 301                 policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate *
 302                                                 cpu->pstate.scaling;
 303                 /*
 304                  * The _PSS table doesn't contain whole turbo frequency range.
 305                  * This just contains +1 MHZ above the max non turbo frequency,
 306                  * with control value corresponding to max turbo ratio. But
 307                  * when cpufreq set policy is called, it will call with this
 308                  * max frequency, which will cause a reduced performance as
 309                  * this driver uses real max turbo frequency as the max
 310                  * frequeny. So correct this frequency in _PSS table to
 311                  * correct max turbo frequency based on the turbo ratio.
 312                  * Also need to convert to MHz as _PSS freq is in MHz.
 313                  */
 314                 cpu->acpi_perf_data.states[0].core_frequency =
 315                                                 turbo_pss_ctl * 100;
 316         }
 317
 318         pr_debug("intel_pstate: Updated limits using _PSS 0x%x 0x%x 0x%x\n",
 319                  cpu->pstate.min_pstate, cpu->pstate.max_pstate,
 320                  cpu->pstate.turbo_pstate);
 321         pr_debug("intel_pstate: policy max_freq=%d Khz min_freq = %d KHz\n",
 322                  policy->cpuinfo.max_freq, policy->cpuinfo.min_freq);
 323
 324         return 0;
 325 }
 326
 327 static int intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 328 {
 329         struct cpudata *cpu;
 330
 331         if (!no_acpi_perf)
 332                 return 0;
 333
 334         cpu = all_cpu_data[policy->cpu];
 335         acpi_processor_unregister_performance(policy->cpu);
 336         return 0;
 337 }
 338
 339 #else
 340 static int intel_pstate_init_perf_limits(struct cpufreq_policy *policy)
 341 {
 342         return 0;
 343 }
 344
 345 static int intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 346 {
 347         return 0;
 348 }
 349 #endif
 350
 351 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
 352                              int deadband, int integral) {
 353         pid->setpoint = setpoint;
 354         pid->deadband  = deadband;
 355         pid->integral  = int_tofp(integral);
 356         pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
 357 }
 358
 359 static inline void pid_p_gain_set(struct _pid *pid, int percent)
 360 {
 361         pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
 362 }
 363
 364 static inline void pid_i_gain_set(struct _pid *pid, int percent)
 365 {
 366         pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
 367 }
 368
 369 static inline void pid_d_gain_set(struct _pid *pid, int percent)
 370 {
 371         pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
 372 }
 373
 374 static signed int pid_calc(struct _pid *pid, int32_t busy)
 375 {
 376         signed int result;
 377         int32_t pterm, dterm, fp_error;
 378         int32_t integral_limit;
 379
 380         fp_error = int_tofp(pid->setpoint) - busy;
 381
 382         if (abs(fp_error) <= int_tofp(pid->deadband))
 383                 return 0;
 384
 385         pterm = mul_fp(pid->p_gain, fp_error);
 386
 387         pid->integral += fp_error;
 388
 389         /*
 390          * We limit the integral here so that it will never
 391          * get higher than 30.  This prevents it from becoming
 392          * too large an input over long periods of time and allows
 393          * it to get factored out sooner.
 394          *
 395          * The value of 30 was chosen through experimentation.
 396          */
 397         integral_limit = int_tofp(30);
 398         if (pid->integral > integral_limit)
 399                 pid->integral = integral_limit;
 400         if (pid->integral < -integral_limit)
 401                 pid->integral = -integral_limit;
 402
 403         dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
 404         pid->last_err = fp_error;
 405
 406         result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
 407         result = result + (1 << (FRAC_BITS-1));
 408         return (signed int)fp_toint(result);
 409 }
 410
 411 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
 412 {
 413         pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
 414         pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
 415         pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
 416
 417         pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
 418 }
 419
 420 static inline void intel_pstate_reset_all_pid(void)
 421 {
 422         unsigned int cpu;
 423
 424         for_each_online_cpu(cpu) {
 425                 if (all_cpu_data[cpu])
 426                         intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
 427         }
 428 }
 429
 430 static inline void update_turbo_state(void)
 431 {
 432         u64 misc_en;
 433         struct cpudata *cpu;
 434
 435         cpu = all_cpu_data[0];
 436         rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
 437         limits->turbo_disabled =
 438                 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
 439                  cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
 440 }
 441
 442 static void intel_pstate_hwp_set(void)
 443 {
 444         int min, hw_min, max, hw_max, cpu, range, adj_range;
 445         u64 value, cap;
 446
 447         rdmsrl(MSR_HWP_CAPABILITIES, cap);
 448         hw_min = HWP_LOWEST_PERF(cap);
 449         hw_max = HWP_HIGHEST_PERF(cap);
 450         range = hw_max - hw_min;
 451
 452         get_online_cpus();
 453
 454         for_each_online_cpu(cpu) {
 455                 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
 456                 adj_range = limits->min_perf_pct * range / 100;
 457                 min = hw_min + adj_range;
 458                 value &= ~HWP_MIN_PERF(~0L);
 459                 value |= HWP_MIN_PERF(min);
 460
 461                 adj_range = limits->max_perf_pct * range / 100;
 462                 max = hw_min + adj_range;
 463                 if (limits->no_turbo) {
 464                         hw_max = HWP_GUARANTEED_PERF(cap);
 465                         if (hw_max < max)
 466                                 max = hw_max;
 467                 }
 468
 469                 value &= ~HWP_MAX_PERF(~0L);
 470                 value |= HWP_MAX_PERF(max);
 471                 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
 472         }
 473
 474         put_online_cpus();
 475 }
 476
 477 /************************** debugfs begin ************************/
 478 static int pid_param_set(void *data, u64 val)
 479 {
 480         *(u32 *)data = val;
 481         intel_pstate_reset_all_pid();
 482         return 0;
 483 }
 484
 485 static int pid_param_get(void *data, u64 *val)
 486 {
 487         *val = *(u32 *)data;
 488         return 0;
 489 }
 490 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
 491
 492 struct pid_param {
 493         char *name;
 494         void *value;
 495 };
 496
 497 static struct pid_param pid_files[] = {
 498         {"sample_rate_ms", &pid_params.sample_rate_ms},
 499         {"d_gain_pct", &pid_params.d_gain_pct},
 500         {"i_gain_pct", &pid_params.i_gain_pct},
 501         {"deadband", &pid_params.deadband},
 502         {"setpoint", &pid_params.setpoint},
 503         {"p_gain_pct", &pid_params.p_gain_pct},
 504         {NULL, NULL}
 505 };
 506
 507 static void __init intel_pstate_debug_expose_params(void)
 508 {
 509         struct dentry *debugfs_parent;
 510         int i = 0;
 511
 512         if (hwp_active)
 513                 return;
 514         debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
 515         if (IS_ERR_OR_NULL(debugfs_parent))
 516                 return;
 517         while (pid_files[i].name) {
 518                 debugfs_create_file(pid_files[i].name, 0660,
 519                                     debugfs_parent, pid_files[i].value,
 520                                     &fops_pid_param);
 521                 i++;
 522         }
 523 }
 524
 525 /************************** debugfs end ************************/
 526
 527 /************************** sysfs begin ************************/
 528 #define show_one(file_name, object)                                     \
 529         static ssize_t show_##file_name                                 \
 530         (struct kobject *kobj, struct attribute *attr, char *buf)       \
 531         {                                                               \
 532                 return sprintf(buf, "%u\n", limits->object);            \
 533         }
 534
 535 static ssize_t show_turbo_pct(struct kobject *kobj,
 536                                 struct attribute *attr, char *buf)
 537 {
 538         struct cpudata *cpu;
 539         int total, no_turbo, turbo_pct;
 540         uint32_t turbo_fp;
 541
 542         cpu = all_cpu_data[0];
 543
 544         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
 545         no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
 546         turbo_fp = div_fp(int_tofp(no_turbo), int_tofp(total));
 547         turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
 548         return sprintf(buf, "%u\n", turbo_pct);
 549 }
 550
 551 static ssize_t show_num_pstates(struct kobject *kobj,
 552                                 struct attribute *attr, char *buf)
 553 {
 554         struct cpudata *cpu;
 555         int total;
 556
 557         cpu = all_cpu_data[0];
 558         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
 559         return sprintf(buf, "%u\n", total);
 560 }
 561
 562 static ssize_t show_no_turbo(struct kobject *kobj,
 563                              struct attribute *attr, char *buf)
 564 {
 565         ssize_t ret;
 566
 567         update_turbo_state();
 568         if (limits->turbo_disabled)
 569                 ret = sprintf(buf, "%u\n", limits->turbo_disabled);
 570         else
 571                 ret = sprintf(buf, "%u\n", limits->no_turbo);
 572
 573         return ret;
 574 }
 575
 576 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 577                               const char *buf, size_t count)
 578 {
 579         unsigned int input;
 580         int ret;
 581
 582         ret = sscanf(buf, "%u", &input);
 583         if (ret != 1)
 584                 return -EINVAL;
 585
 586         update_turbo_state();
 587         if (limits->turbo_disabled) {
 588                 pr_warn("intel_pstate: Turbo disabled by BIOS or unavailable on processor\n");
 589                 return -EPERM;
 590         }
 591
 592         limits->no_turbo = clamp_t(int, input, 0, 1);
 593
 594         if (hwp_active)
 595                 intel_pstate_hwp_set();
 596
 597         return count;
 598 }
 599
 600 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 601                                   const char *buf, size_t count)
 602 {
 603         unsigned int input;
 604         int ret;
 605
 606         ret = sscanf(buf, "%u", &input);
 607         if (ret != 1)
 608                 return -EINVAL;
 609
 610         limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
 611         limits->max_perf_pct = min(limits->max_policy_pct,
 612                                    limits->max_sysfs_pct);
 613         limits->max_perf_pct = max(limits->min_policy_pct,
 614                                    limits->max_perf_pct);
 615         limits->max_perf_pct = max(limits->min_perf_pct,
 616                                    limits->max_perf_pct);
 617         limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
 618                                   int_tofp(100));
 619
 620         if (hwp_active)
 621                 intel_pstate_hwp_set();
 622         return count;
 623 }
 624
 625 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 626                                   const char *buf, size_t count)
 627 {
 628         unsigned int input;
 629         int ret;
 630
 631         ret = sscanf(buf, "%u", &input);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
 636         limits->min_perf_pct = max(limits->min_policy_pct,
 637                                    limits->min_sysfs_pct);
 638         limits->min_perf_pct = min(limits->max_policy_pct,
 639                                    limits->min_perf_pct);
 640         limits->min_perf_pct = min(limits->max_perf_pct,
 641                                    limits->min_perf_pct);
 642         limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
 643                                   int_tofp(100));
 644
 645         if (hwp_active)
 646                 intel_pstate_hwp_set();
 647         return count;
 648 }
 649
 650 show_one(max_perf_pct, max_perf_pct);
 651 show_one(min_perf_pct, min_perf_pct);
 652
 653 define_one_global_rw(no_turbo);
 654 define_one_global_rw(max_perf_pct);
 655 define_one_global_rw(min_perf_pct);
 656 define_one_global_ro(turbo_pct);
 657 define_one_global_ro(num_pstates);
 658
 659 static struct attribute *intel_pstate_attributes[] = {
 660         &no_turbo.attr,
 661         &max_perf_pct.attr,
 662         &min_perf_pct.attr,
 663         &turbo_pct.attr,
 664         &num_pstates.attr,
 665         NULL
 666 };
 667
 668 static struct attribute_group intel_pstate_attr_group = {
 669         .attrs = intel_pstate_attributes,
 670 };
 671
 672 static void __init intel_pstate_sysfs_expose_params(void)
 673 {
 674         struct kobject *intel_pstate_kobject;
 675         int rc;
 676
 677         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
 678                                                 &cpu_subsys.dev_root->kobj);
 679         BUG_ON(!intel_pstate_kobject);
 680         rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
 681         BUG_ON(rc);
 682 }
 683 /************************** sysfs end ************************/
 684
 685 static void intel_pstate_hwp_enable(struct cpudata *cpudata)
 686 {
 687         pr_info("intel_pstate: HWP enabled\n");
 688
 689         wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
 690 }
 691
 692 static int byt_get_min_pstate(void)
 693 {
 694         u64 value;
 695
 696         rdmsrl(BYT_RATIOS, value);
 697         return (value >> 8) & 0x7F;
 698 }
 699
 700 static int byt_get_max_pstate(void)
 701 {
 702         u64 value;
 703
 704         rdmsrl(BYT_RATIOS, value);
 705         return (value >> 16) & 0x7F;
 706 }
 707
 708 static int byt_get_turbo_pstate(void)
 709 {
 710         u64 value;
 711
 712         rdmsrl(BYT_TURBO_RATIOS, value);
 713         return value & 0x7F;
 714 }
 715
 716 static void byt_set_pstate(struct cpudata *cpudata, int pstate)
 717 {
 718         u64 val;
 719         int32_t vid_fp;
 720         u32 vid;
 721
 722         val = (u64)pstate << 8;
 723         if (limits->no_turbo && !limits->turbo_disabled)
 724                 val |= (u64)1 << 32;
 725
 726         vid_fp = cpudata->vid.min + mul_fp(
 727                 int_tofp(pstate - cpudata->pstate.min_pstate),
 728                 cpudata->vid.ratio);
 729
 730         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
 731         vid = ceiling_fp(vid_fp);
 732
 733         if (pstate > cpudata->pstate.max_pstate)
 734                 vid = cpudata->vid.turbo;
 735
 736         val |= vid;
 737
 738         wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
 739 }
 740
 741 #define BYT_BCLK_FREQS 5
 742 static int byt_freq_table[BYT_BCLK_FREQS] = { 833, 1000, 1333, 1167, 800};
 743
 744 static int byt_get_scaling(void)
 745 {
 746         u64 value;
 747         int i;
 748
 749         rdmsrl(MSR_FSB_FREQ, value);
 750         i = value & 0x3;
 751
 752         BUG_ON(i > BYT_BCLK_FREQS);
 753
 754         return byt_freq_table[i] * 100;
 755 }
 756
 757 static void byt_get_vid(struct cpudata *cpudata)
 758 {
 759         u64 value;
 760
 761         rdmsrl(BYT_VIDS, value);
 762         cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
 763         cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
 764         cpudata->vid.ratio = div_fp(
 765                 cpudata->vid.max - cpudata->vid.min,
 766                 int_tofp(cpudata->pstate.max_pstate -
 767                         cpudata->pstate.min_pstate));
 768
 769         rdmsrl(BYT_TURBO_VIDS, value);
 770         cpudata->vid.turbo = value & 0x7f;
 771 }
 772
 773 static int core_get_min_pstate(void)
 774 {
 775         u64 value;
 776
 777         rdmsrl(MSR_PLATFORM_INFO, value);
 778         return (value >> 40) & 0xFF;
 779 }
 780
 781 static int core_get_max_pstate_physical(void)
 782 {
 783         u64 value;
 784
 785         rdmsrl(MSR_PLATFORM_INFO, value);
 786         return (value >> 8) & 0xFF;
 787 }
 788
 789 static int core_get_max_pstate(void)
 790 {
 791         u64 tar;
 792         u64 plat_info;
 793         int max_pstate;
 794         int err;
 795
 796         rdmsrl(MSR_PLATFORM_INFO, plat_info);
 797         max_pstate = (plat_info >> 8) & 0xFF;
 798
 799         err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
 800         if (!err) {
 801                 /* Do some sanity checking for safety */
 802                 if (plat_info & 0x600000000) {
 803                         u64 tdp_ctrl;
 804                         u64 tdp_ratio;
 805                         int tdp_msr;
 806
 807                         err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
 808                         if (err)
 809                                 goto skip_tar;
 810
 811                         tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl;
 812                         err = rdmsrl_safe(tdp_msr, &tdp_ratio);
 813                         if (err)
 814                                 goto skip_tar;
 815
 816                         if (tdp_ratio - 1 == tar) {
 817                                 max_pstate = tar;
 818                                 pr_debug("max_pstate=TAC %x\n", max_pstate);
 819                         } else {
 820                                 goto skip_tar;
 821                         }
 822                 }
 823         }
 824
 825 skip_tar:
 826         return max_pstate;
 827 }
 828
 829 static int core_get_turbo_pstate(void)
 830 {
 831         u64 value;
 832         int nont, ret;
 833
 834         rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
 835         nont = core_get_max_pstate();
 836         ret = (value) & 255;
 837         if (ret <= nont)
 838                 ret = nont;
 839         return ret;
 840 }
 841
 842 static inline int core_get_scaling(void)
 843 {
 844         return 100000;
 845 }
 846
 847 static void core_set_pstate(struct cpudata *cpudata, int pstate)
 848 {
 849         u64 val;
 850
 851         val = (u64)pstate << 8;
 852         if (limits->no_turbo && !limits->turbo_disabled)
 853                 val |= (u64)1 << 32;
 854
 855         wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
 856 }
 857
 858 static int knl_get_turbo_pstate(void)
 859 {
 860         u64 value;
 861         int nont, ret;
 862
 863         rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
 864         nont = core_get_max_pstate();
 865         ret = (((value) >> 8) & 0xFF);
 866         if (ret <= nont)
 867                 ret = nont;
 868         return ret;
 869 }
 870
 871 static struct cpu_defaults core_params = {
 872         .pid_policy = {
 873                 .sample_rate_ms = 10,
 874                 .deadband = 0,
 875                 .setpoint = 97,
 876                 .p_gain_pct = 20,
 877                 .d_gain_pct = 0,
 878                 .i_gain_pct = 0,
 879         },
 880         .funcs = {
 881                 .get_max = core_get_max_pstate,
 882                 .get_max_physical = core_get_max_pstate_physical,
 883                 .get_min = core_get_min_pstate,
 884                 .get_turbo = core_get_turbo_pstate,
 885                 .get_scaling = core_get_scaling,
 886                 .set = core_set_pstate,
 887         },
 888 };
 889
 890 static struct cpu_defaults byt_params = {
 891         .pid_policy = {
 892                 .sample_rate_ms = 10,
 893                 .deadband = 0,
 894                 .setpoint = 60,
 895                 .p_gain_pct = 14,
 896                 .d_gain_pct = 0,
 897                 .i_gain_pct = 4,
 898         },
 899         .funcs = {
 900                 .get_max = byt_get_max_pstate,
 901                 .get_max_physical = byt_get_max_pstate,
 902                 .get_min = byt_get_min_pstate,
 903                 .get_turbo = byt_get_turbo_pstate,
 904                 .set = byt_set_pstate,
 905                 .get_scaling = byt_get_scaling,
 906                 .get_vid = byt_get_vid,
 907         },
 908 };
 909
 910 static struct cpu_defaults knl_params = {
 911         .pid_policy = {
 912                 .sample_rate_ms = 10,
 913                 .deadband = 0,
 914                 .setpoint = 97,
 915                 .p_gain_pct = 20,
 916                 .d_gain_pct = 0,
 917                 .i_gain_pct = 0,
 918         },
 919         .funcs = {
 920                 .get_max = core_get_max_pstate,
 921                 .get_max_physical = core_get_max_pstate_physical,
 922                 .get_min = core_get_min_pstate,
 923                 .get_turbo = knl_get_turbo_pstate,
 924                 .get_scaling = core_get_scaling,
 925                 .set = core_set_pstate,
 926         },
 927 };
 928
 929 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
 930 {
 931         int max_perf = cpu->pstate.turbo_pstate;
 932         int max_perf_adj;
 933         int min_perf;
 934
 935         if (limits->no_turbo || limits->turbo_disabled)
 936                 max_perf = cpu->pstate.max_pstate;
 937
 938         /*
 939          * performance can be limited by user through sysfs, by cpufreq
 940          * policy, or by cpu specific default values determined through
 941          * experimentation.
 942          */
 943         if (limits->max_perf_ctl && limits->max_sysfs_pct >=
 944                                                 limits->max_policy_pct) {
 945                 *max = limits->max_perf_ctl;
 946         } else {
 947                 max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf),
 948                                         limits->max_perf));
 949                 *max = clamp_t(int, max_perf_adj, cpu->pstate.min_pstate,
 950                                cpu->pstate.turbo_pstate);
 951         }
 952
 953         if (limits->min_perf_ctl) {
 954                 *min = limits->min_perf_ctl;
 955         } else {
 956                 min_perf = fp_toint(mul_fp(int_tofp(max_perf),
 957                                     limits->min_perf));
 958                 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
 959         }
 960 }
 961
 962 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate, bool force)
 963 {
 964         int max_perf, min_perf;
 965
 966         if (force) {
 967                 update_turbo_state();
 968
 969                 intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
 970
 971                 pstate = clamp_t(int, pstate, min_perf, max_perf);
 972
 973                 if (pstate == cpu->pstate.current_pstate)
 974                         return;
 975         }
 976         trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
 977
 978         cpu->pstate.current_pstate = pstate;
 979
 980         pstate_funcs.set(cpu, pstate);
 981 }
 982
 983 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 984 {
 985         cpu->pstate.min_pstate = pstate_funcs.get_min();
 986         cpu->pstate.max_pstate = pstate_funcs.get_max();
 987         cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
 988         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
 989         cpu->pstate.scaling = pstate_funcs.get_scaling();
 990
 991         if (pstate_funcs.get_vid)
 992                 pstate_funcs.get_vid(cpu);
 993         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
 994 }
 995
 996 static inline void intel_pstate_calc_busy(struct cpudata *cpu)
 997 {
 998         struct sample *sample = &cpu->sample;
 999         int64_t core_pct;
1000
1001         core_pct = int_tofp(sample->aperf) * int_tofp(100);
1002         core_pct = div64_u64(core_pct, int_tofp(sample->mperf));
1003
1004         sample->freq = fp_toint(
1005                 mul_fp(int_tofp(
1006                         cpu->pstate.max_pstate_physical *
1007                         cpu->pstate.scaling / 100),
1008                         core_pct));
1009
1010         sample->core_pct_busy = (int32_t)core_pct;
1011 }
1012
1013 static inline void intel_pstate_sample(struct cpudata *cpu)
1014 {
1015         u64 aperf, mperf;
1016         unsigned long flags;
1017         u64 tsc;
1018
1019         local_irq_save(flags);
1020         rdmsrl(MSR_IA32_APERF, aperf);
1021         rdmsrl(MSR_IA32_MPERF, mperf);
1022         if (cpu->prev_mperf == mperf) {
1023                 local_irq_restore(flags);
1024                 return;
1025         }
1026
1027         tsc = rdtsc();
1028         local_irq_restore(flags);
1029
1030         cpu->last_sample_time = cpu->sample.time;
1031         cpu->sample.time = ktime_get();
1032         cpu->sample.aperf = aperf;
1033         cpu->sample.mperf = mperf;
1034         cpu->sample.tsc =  tsc;
1035         cpu->sample.aperf -= cpu->prev_aperf;
1036         cpu->sample.mperf -= cpu->prev_mperf;
1037         cpu->sample.tsc -= cpu->prev_tsc;
1038
1039         intel_pstate_calc_busy(cpu);
1040
1041         cpu->prev_aperf = aperf;
1042         cpu->prev_mperf = mperf;
1043         cpu->prev_tsc = tsc;
1044 }
1045
1046 static inline void intel_hwp_set_sample_time(struct cpudata *cpu)
1047 {
1048         int delay;
1049
1050         delay = msecs_to_jiffies(50);
1051         mod_timer_pinned(&cpu->timer, jiffies + delay);
1052 }
1053
1054 static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
1055 {
1056         int delay;
1057
1058         delay = msecs_to_jiffies(pid_params.sample_rate_ms);
1059         mod_timer_pinned(&cpu->timer, jiffies + delay);
1060 }
1061
1062 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
1063 {
1064         int32_t core_busy, max_pstate, current_pstate, sample_ratio;
1065         s64 duration_us;
1066         u32 sample_time;
1067
1068         /*
1069          * core_busy is the ratio of actual performance to max
1070          * max_pstate is the max non turbo pstate available
1071          * current_pstate was the pstate that was requested during
1072          *      the last sample period.
1073          *
1074          * We normalize core_busy, which was our actual percent
1075          * performance to what we requested during the last sample
1076          * period. The result will be a percentage of busy at a
1077          * specified pstate.
1078          */
1079         core_busy = cpu->sample.core_pct_busy;
1080         max_pstate = int_tofp(cpu->pstate.max_pstate_physical);
1081         current_pstate = int_tofp(cpu->pstate.current_pstate);
1082         core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
1083
1084         /*
1085          * Since we have a deferred timer, it will not fire unless
1086          * we are in C0.  So, determine if the actual elapsed time
1087          * is significantly greater (3x) than our sample interval.  If it
1088          * is, then we were idle for a long enough period of time
1089          * to adjust our busyness.
1090          */
1091         sample_time = pid_params.sample_rate_ms  * USEC_PER_MSEC;
1092         duration_us = ktime_us_delta(cpu->sample.time,
1093                                      cpu->last_sample_time);
1094         if (duration_us > sample_time * 3) {
1095                 sample_ratio = div_fp(int_tofp(sample_time),
1096                                       int_tofp(duration_us));
1097                 core_busy = mul_fp(core_busy, sample_ratio);
1098         }
1099
1100         return core_busy;
1101 }
1102
1103 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1104 {
1105         int32_t busy_scaled;
1106         struct _pid *pid;
1107         signed int ctl;
1108         int from;
1109         struct sample *sample;
1110
1111         from = cpu->pstate.current_pstate;
1112
1113         pid = &cpu->pid;
1114         busy_scaled = intel_pstate_get_scaled_busy(cpu);
1115
1116         ctl = pid_calc(pid, busy_scaled);
1117
1118         /* Negative values of ctl increase the pstate and vice versa */
1119         intel_pstate_set_pstate(cpu, cpu->pstate.current_pstate - ctl, true);
1120
1121         sample = &cpu->sample;
1122         trace_pstate_sample(fp_toint(sample->core_pct_busy),
1123                 fp_toint(busy_scaled),
1124                 from,
1125                 cpu->pstate.current_pstate,
1126                 sample->mperf,
1127                 sample->aperf,
1128                 sample->tsc,
1129                 sample->freq);
1130 }
1131
1132 static void intel_hwp_timer_func(unsigned long __data)
1133 {
1134         struct cpudata *cpu = (struct cpudata *) __data;
1135
1136         intel_pstate_sample(cpu);
1137         intel_hwp_set_sample_time(cpu);
1138 }
1139
1140 static void intel_pstate_timer_func(unsigned long __data)
1141 {
1142         struct cpudata *cpu = (struct cpudata *) __data;
1143
1144         intel_pstate_sample(cpu);
1145
1146         intel_pstate_adjust_busy_pstate(cpu);
1147
1148         intel_pstate_set_sample_time(cpu);
1149 }
1150
1151 #define ICPU(model, policy) \
1152         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1153                         (unsigned long)&policy }
1154
1155 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1156         ICPU(0x2a, core_params),
1157         ICPU(0x2d, core_params),
1158         ICPU(0x37, byt_params),
1159         ICPU(0x3a, core_params),
1160         ICPU(0x3c, core_params),
1161         ICPU(0x3d, core_params),
1162         ICPU(0x3e, core_params),
1163         ICPU(0x3f, core_params),
1164         ICPU(0x45, core_params),
1165         ICPU(0x46, core_params),
1166         ICPU(0x47, core_params),
1167         ICPU(0x4c, byt_params),
1168         ICPU(0x4e, core_params),
1169         ICPU(0x4f, core_params),
1170         ICPU(0x5e, core_params),
1171         ICPU(0x56, core_params),
1172         ICPU(0x57, knl_params),
1173         {}
1174 };
1175 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1176
1177 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = {
1178         ICPU(0x56, core_params),
1179         {}
1180 };
1181
1182 static int intel_pstate_init_cpu(unsigned int cpunum)
1183 {
1184         struct cpudata *cpu;
1185
1186         if (!all_cpu_data[cpunum])
1187                 all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata),
1188                                                GFP_KERNEL);
1189         if (!all_cpu_data[cpunum])
1190                 return -ENOMEM;
1191
1192         cpu = all_cpu_data[cpunum];
1193
1194         cpu->cpu = cpunum;
1195
1196         if (hwp_active)
1197                 intel_pstate_hwp_enable(cpu);
1198
1199         intel_pstate_get_cpu_pstates(cpu);
1200
1201         init_timer_deferrable(&cpu->timer);
1202         cpu->timer.data = (unsigned long)cpu;
1203         cpu->timer.expires = jiffies + HZ/100;
1204
1205         if (!hwp_active)
1206                 cpu->timer.function = intel_pstate_timer_func;
1207         else
1208                 cpu->timer.function = intel_hwp_timer_func;
1209
1210         intel_pstate_busy_pid_reset(cpu);
1211         intel_pstate_sample(cpu);
1212
1213         add_timer_on(&cpu->timer, cpunum);
1214
1215         pr_debug("intel_pstate: controlling: cpu %d\n", cpunum);
1216
1217         return 0;
1218 }
1219
1220 static unsigned int intel_pstate_get(unsigned int cpu_num)
1221 {
1222         struct sample *sample;
1223         struct cpudata *cpu;
1224
1225         cpu = all_cpu_data[cpu_num];
1226         if (!cpu)
1227                 return 0;
1228         sample = &cpu->sample;
1229         return sample->freq;
1230 }
1231
1232 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
1233 {
1234 #if IS_ENABLED(CONFIG_ACPI)
1235         struct cpudata *cpu;
1236         int i;
1237 #endif
1238         pr_debug("intel_pstate: %s max %u policy->max %u\n", __func__,
1239                  policy->cpuinfo.max_freq, policy->max);
1240         if (!policy->cpuinfo.max_freq)
1241                 return -ENODEV;
1242
1243         if (policy->policy == CPUFREQ_POLICY_PERFORMANCE &&
1244             policy->max >= policy->cpuinfo.max_freq) {
1245                 pr_debug("intel_pstate: set performance\n");
1246                 limits = &performance_limits;
1247                 return 0;
1248         }
1249
1250         pr_debug("intel_pstate: set powersave\n");
1251         limits = &powersave_limits;
1252         limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
1253         limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100);
1254         limits->max_policy_pct = (policy->max * 100) / policy->cpuinfo.max_freq;
1255         limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100);
1256
1257         /* Normalize user input to [min_policy_pct, max_policy_pct] */
1258         limits->min_perf_pct = max(limits->min_policy_pct,
1259                                    limits->min_sysfs_pct);
1260         limits->min_perf_pct = min(limits->max_policy_pct,
1261                                    limits->min_perf_pct);
1262         limits->max_perf_pct = min(limits->max_policy_pct,
1263                                    limits->max_sysfs_pct);
1264         limits->max_perf_pct = max(limits->min_policy_pct,
1265                                    limits->max_perf_pct);
1266
1267         /* Make sure min_perf_pct <= max_perf_pct */
1268         limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
1269
1270         limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
1271                                   int_tofp(100));
1272         limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
1273                                   int_tofp(100));
1274
1275 #if IS_ENABLED(CONFIG_ACPI)
1276         cpu = all_cpu_data[policy->cpu];
1277         for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
1278                 int control;
1279
1280                 control = convert_to_native_pstate_format(cpu, i);
1281                 if (control * cpu->pstate.scaling == policy->max)
1282                         limits->max_perf_ctl = control;
1283                 if (control * cpu->pstate.scaling == policy->min)
1284                         limits->min_perf_ctl = control;
1285         }
1286
1287         pr_debug("intel_pstate: max %u policy_max %u perf_ctl [0x%x-0x%x]\n",
1288                  policy->cpuinfo.max_freq, policy->max, limits->min_perf_ctl,
1289                  limits->max_perf_ctl);
1290 #endif
1291
1292         if (hwp_active)
1293                 intel_pstate_hwp_set();
1294
1295         return 0;
1296 }
1297
1298 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
1299 {
1300         cpufreq_verify_within_cpu_limits(policy);
1301
1302         if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
1303             policy->policy != CPUFREQ_POLICY_PERFORMANCE)
1304                 return -EINVAL;
1305
1306         return 0;
1307 }
1308
1309 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1310 {
1311         int cpu_num = policy->cpu;
1312         struct cpudata *cpu = all_cpu_data[cpu_num];
1313
1314         pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
1315
1316         del_timer_sync(&all_cpu_data[cpu_num]->timer);
1317         if (hwp_active)
1318                 return;
1319
1320         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
1321 }
1322
1323 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
1324 {
1325         struct cpudata *cpu;
1326         int rc;
1327
1328         rc = intel_pstate_init_cpu(policy->cpu);
1329         if (rc)
1330                 return rc;
1331
1332         cpu = all_cpu_data[policy->cpu];
1333
1334         if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
1335                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1336         else
1337                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
1338
1339         policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
1340         policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1341
1342         /* cpuinfo and default policy values */
1343         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
1344         policy->cpuinfo.max_freq =
1345                 cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1346         if (!no_acpi_perf)
1347                 intel_pstate_init_perf_limits(policy);
1348         /*
1349          * If there is no acpi perf data or error, we ignore and use Intel P
1350          * state calculated limits, So this is not fatal error.
1351          */
1352         policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
1353         cpumask_set_cpu(policy->cpu, policy->cpus);
1354
1355         return 0;
1356 }
1357
1358 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
1359 {
1360         return intel_pstate_exit_perf_limits(policy);
1361 }
1362
1363 static struct cpufreq_driver intel_pstate_driver = {
1364         .flags          = CPUFREQ_CONST_LOOPS,
1365         .verify         = intel_pstate_verify_policy,
1366         .setpolicy      = intel_pstate_set_policy,
1367         .get            = intel_pstate_get,
1368         .init           = intel_pstate_cpu_init,
1369         .exit           = intel_pstate_cpu_exit,
1370         .stop_cpu       = intel_pstate_stop_cpu,
1371         .name           = "intel_pstate",
1372 };
1373
1374 static int __initdata no_load;
1375 static int __initdata no_hwp;
1376 static int __initdata hwp_only;
1377 static unsigned int force_load;
1378
1379 static int intel_pstate_msrs_not_valid(void)
1380 {
1381         if (!pstate_funcs.get_max() ||
1382             !pstate_funcs.get_min() ||
1383             !pstate_funcs.get_turbo())
1384                 return -ENODEV;
1385
1386         return 0;
1387 }
1388
1389 static void copy_pid_params(struct pstate_adjust_policy *policy)
1390 {
1391         pid_params.sample_rate_ms = policy->sample_rate_ms;
1392         pid_params.p_gain_pct = policy->p_gain_pct;
1393         pid_params.i_gain_pct = policy->i_gain_pct;
1394         pid_params.d_gain_pct = policy->d_gain_pct;
1395         pid_params.deadband = policy->deadband;
1396         pid_params.setpoint = policy->setpoint;
1397 }
1398
1399 static void copy_cpu_funcs(struct pstate_funcs *funcs)
1400 {
1401         pstate_funcs.get_max   = funcs->get_max;
1402         pstate_funcs.get_max_physical = funcs->get_max_physical;
1403         pstate_funcs.get_min   = funcs->get_min;
1404         pstate_funcs.get_turbo = funcs->get_turbo;
1405         pstate_funcs.get_scaling = funcs->get_scaling;
1406         pstate_funcs.set       = funcs->set;
1407         pstate_funcs.get_vid   = funcs->get_vid;
1408 }
1409
1410 #if IS_ENABLED(CONFIG_ACPI)
1411
1412 static bool intel_pstate_no_acpi_pss(void)
1413 {
1414         int i;
1415
1416         for_each_possible_cpu(i) {
1417                 acpi_status status;
1418                 union acpi_object *pss;
1419                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
1420                 struct acpi_processor *pr = per_cpu(processors, i);
1421
1422                 if (!pr)
1423                         continue;
1424
1425                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
1426                 if (ACPI_FAILURE(status))
1427                         continue;
1428
1429                 pss = buffer.pointer;
1430                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
1431                         kfree(pss);
1432                         return false;
1433                 }
1434
1435                 kfree(pss);
1436         }
1437
1438         return true;
1439 }
1440
1441 static bool intel_pstate_has_acpi_ppc(void)
1442 {
1443         int i;
1444
1445         for_each_possible_cpu(i) {
1446                 struct acpi_processor *pr = per_cpu(processors, i);
1447
1448                 if (!pr)
1449                         continue;
1450                 if (acpi_has_method(pr->handle, "_PPC"))
1451                         return true;
1452         }
1453         return false;
1454 }
1455
1456 enum {
1457         PSS,
1458         PPC,
1459 };
1460
1461 struct hw_vendor_info {
1462         u16  valid;
1463         char oem_id[ACPI_OEM_ID_SIZE];
1464         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
1465         int  oem_pwr_table;
1466 };
1467
1468 /* Hardware vendor-specific info that has its own power management modes */
1469 static struct hw_vendor_info vendor_info[] = {
1470         {1, "HP    ", "ProLiant", PSS},
1471         {1, "ORACLE", "X4-2    ", PPC},
1472         {1, "ORACLE", "X4-2L   ", PPC},
1473         {1, "ORACLE", "X4-2B   ", PPC},
1474         {1, "ORACLE", "X3-2    ", PPC},
1475         {1, "ORACLE", "X3-2L   ", PPC},
1476         {1, "ORACLE", "X3-2B   ", PPC},
1477         {1, "ORACLE", "X4470M2 ", PPC},
1478         {1, "ORACLE", "X4270M3 ", PPC},
1479         {1, "ORACLE", "X4270M2 ", PPC},
1480         {1, "ORACLE", "X4170M2 ", PPC},
1481         {1, "ORACLE", "X4170 M3", PPC},
1482         {1, "ORACLE", "X4275 M3", PPC},
1483         {1, "ORACLE", "X6-2    ", PPC},
1484         {1, "ORACLE", "Sudbury ", PPC},
1485         {0, "", ""},
1486 };
1487
1488 static bool intel_pstate_platform_pwr_mgmt_exists(void)
1489 {
1490         struct acpi_table_header hdr;
1491         struct hw_vendor_info *v_info;
1492         const struct x86_cpu_id *id;
1493         u64 misc_pwr;
1494
1495         id = x86_match_cpu(intel_pstate_cpu_oob_ids);
1496         if (id) {
1497                 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
1498                 if ( misc_pwr & (1 << 8))
1499                         return true;
1500         }
1501
1502         if (acpi_disabled ||
1503             ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
1504                 return false;
1505
1506         for (v_info = vendor_info; v_info->valid; v_info++) {
1507                 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
1508                         !strncmp(hdr.oem_table_id, v_info->oem_table_id,
1509                                                 ACPI_OEM_TABLE_ID_SIZE))
1510                         switch (v_info->oem_pwr_table) {
1511                         case PSS:
1512                                 return intel_pstate_no_acpi_pss();
1513                         case PPC:
1514                                 return intel_pstate_has_acpi_ppc() &&
1515                                         (!force_load);
1516                         }
1517         }
1518
1519         return false;
1520 }
1521 #else /* CONFIG_ACPI not enabled */
1522 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
1523 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
1524 #endif /* CONFIG_ACPI */
1525
1526 static int __init intel_pstate_init(void)
1527 {
1528         int cpu, rc = 0;
1529         const struct x86_cpu_id *id;
1530         struct cpu_defaults *cpu_def;
1531
1532         if (no_load)
1533                 return -ENODEV;
1534
1535         id = x86_match_cpu(intel_pstate_cpu_ids);
1536         if (!id)
1537                 return -ENODEV;
1538
1539         /*
1540          * The Intel pstate driver will be ignored if the platform
1541          * firmware has its own power management modes.
1542          */
1543         if (intel_pstate_platform_pwr_mgmt_exists())
1544                 return -ENODEV;
1545
1546         cpu_def = (struct cpu_defaults *)id->driver_data;
1547
1548         copy_pid_params(&cpu_def->pid_policy);
1549         copy_cpu_funcs(&cpu_def->funcs);
1550
1551         if (intel_pstate_msrs_not_valid())
1552                 return -ENODEV;
1553
1554         pr_info("Intel P-state driver initializing.\n");
1555
1556         all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
1557         if (!all_cpu_data)
1558                 return -ENOMEM;
1559
1560         if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp)
1561                 hwp_active++;
1562
1563         if (!hwp_active && hwp_only)
1564                 goto out;
1565
1566         rc = cpufreq_register_driver(&intel_pstate_driver);
1567         if (rc)
1568                 goto out;
1569
1570         intel_pstate_debug_expose_params();
1571         intel_pstate_sysfs_expose_params();
1572
1573         return rc;
1574 out:
1575         get_online_cpus();
1576         for_each_online_cpu(cpu) {
1577                 if (all_cpu_data[cpu]) {
1578                         del_timer_sync(&all_cpu_data[cpu]->timer);
1579                         kfree(all_cpu_data[cpu]);
1580                 }
1581         }
1582
1583         put_online_cpus();
1584         vfree(all_cpu_data);
1585         return -ENODEV;
1586 }
1587 device_initcall(intel_pstate_init);
1588
1589 static int __init intel_pstate_setup(char *str)
1590 {
1591         if (!str)
1592                 return -EINVAL;
1593
1594         if (!strcmp(str, "disable"))
1595                 no_load = 1;
1596         if (!strcmp(str, "no_hwp"))
1597                 no_hwp = 1;
1598         if (!strcmp(str, "force"))
1599                 force_load = 1;
1600         if (!strcmp(str, "hwp_only"))
1601                 hwp_only = 1;
1602         if (!strcmp(str, "no_acpi"))
1603                 no_acpi_perf = 1;
1604
1605         return 0;
1606 }
1607 early_param("intel_pstate", intel_pstate_setup);
1608
1609 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1610 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1611 MODULE_LICENSE("GPL");