drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #include <linux/kernel.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/module.h>
  16 #include <linux/ktime.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/tick.h>
  19 #include <linux/slab.h>
  20 #include <linux/sched.h>
  21 #include <linux/list.h>
  22 #include <linux/cpu.h>
  23 #include <linux/cpufreq.h>
  24 #include <linux/sysfs.h>
  25 #include <linux/types.h>
  26 #include <linux/fs.h>
  27 #include <linux/debugfs.h>
  28 #include <linux/acpi.h>
  29 #include <trace/events/power.h>
  30
  31 #include <asm/div64.h>
  32 #include <asm/msr.h>
  33 #include <asm/cpu_device_id.h>
  34
  35 #define SAMPLE_COUNT            3
  36
  37 #define BYT_RATIOS      0x66a
  38 #define BYT_VIDS        0x66b
  39
  40 #define FRAC_BITS 8
  41 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  42 #define fp_toint(X) ((X) >> FRAC_BITS)
  43
  44 static inline int32_t mul_fp(int32_t x, int32_t y)
  45 {
  46         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  47 }
  48
  49 static inline int32_t div_fp(int32_t x, int32_t y)
  50 {
  51         return div_s64((int64_t)x << FRAC_BITS, (int64_t)y);
  52 }
  53
  54 static u64 energy_divisor;
  55
  56 struct sample {
  57         int32_t core_pct_busy;
  58         u64 aperf;
  59         u64 mperf;
  60         int freq;
  61 };
  62
  63 struct pstate_data {
  64         int     current_pstate;
  65         int     min_pstate;
  66         int     max_pstate;
  67         int     turbo_pstate;
  68 };
  69
  70 struct vid_data {
  71         int32_t min;
  72         int32_t max;
  73         int32_t ratio;
  74 };
  75
  76 struct _pid {
  77         int setpoint;
  78         int32_t integral;
  79         int32_t p_gain;
  80         int32_t i_gain;
  81         int32_t d_gain;
  82         int deadband;
  83         int32_t last_err;
  84 };
  85
  86 struct cpudata {
  87         int cpu;
  88
  89         char name[64];
  90
  91         struct timer_list timer;
  92
  93         struct pstate_data pstate;
  94         struct vid_data vid;
  95         struct _pid pid;
  96
  97         u64     prev_aperf;
  98         u64     prev_mperf;
  99         int     sample_ptr;
 100         struct sample samples[SAMPLE_COUNT];
 101 };
 102
 103 static struct cpudata **all_cpu_data;
 104 struct pstate_adjust_policy {
 105         int sample_rate_ms;
 106         int deadband;
 107         int setpoint;
 108         int p_gain_pct;
 109         int d_gain_pct;
 110         int i_gain_pct;
 111 };
 112
 113 struct pstate_funcs {
 114         int (*get_max)(void);
 115         int (*get_min)(void);
 116         int (*get_turbo)(void);
 117         void (*set)(struct cpudata*, int pstate);
 118         void (*get_vid)(struct cpudata *);
 119 };
 120
 121 struct cpu_defaults {
 122         struct pstate_adjust_policy pid_policy;
 123         struct pstate_funcs funcs;
 124 };
 125
 126 static struct pstate_adjust_policy pid_params;
 127 static struct pstate_funcs pstate_funcs;
 128
 129 struct perf_limits {
 130         int no_turbo;
 131         int max_perf_pct;
 132         int min_perf_pct;
 133         int32_t max_perf;
 134         int32_t min_perf;
 135         int max_policy_pct;
 136         int max_sysfs_pct;
 137 };
 138
 139 static struct perf_limits limits = {
 140         .no_turbo = 0,
 141         .max_perf_pct = 100,
 142         .max_perf = int_tofp(1),
 143         .min_perf_pct = 0,
 144         .min_perf = 0,
 145         .max_policy_pct = 100,
 146         .max_sysfs_pct = 100,
 147 };
 148
 149 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
 150                         int deadband, int integral) {
 151         pid->setpoint = setpoint;
 152         pid->deadband  = deadband;
 153         pid->integral  = int_tofp(integral);
 154         pid->last_err  = setpoint - busy;
 155 }
 156
 157 static inline void pid_p_gain_set(struct _pid *pid, int percent)
 158 {
 159         pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
 160 }
 161
 162 static inline void pid_i_gain_set(struct _pid *pid, int percent)
 163 {
 164         pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
 165 }
 166
 167 static inline void pid_d_gain_set(struct _pid *pid, int percent)
 168 {
 169
 170         pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
 171 }
 172
 173 static signed int pid_calc(struct _pid *pid, int32_t busy)
 174 {
 175         signed int result;
 176         int32_t pterm, dterm, fp_error;
 177         int32_t integral_limit;
 178
 179         fp_error = int_tofp(pid->setpoint) - busy;
 180
 181         if (abs(fp_error) <= int_tofp(pid->deadband))
 182                 return 0;
 183
 184         pterm = mul_fp(pid->p_gain, fp_error);
 185
 186         pid->integral += fp_error;
 187
 188         /* limit the integral term */
 189         integral_limit = int_tofp(30);
 190         if (pid->integral > integral_limit)
 191                 pid->integral = integral_limit;
 192         if (pid->integral < -integral_limit)
 193                 pid->integral = -integral_limit;
 194
 195         dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
 196         pid->last_err = fp_error;
 197
 198         result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
 199
 200         return (signed int)fp_toint(result);
 201 }
 202
 203 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
 204 {
 205         pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
 206         pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
 207         pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
 208
 209         pid_reset(&cpu->pid,
 210                 pid_params.setpoint,
 211                 100,
 212                 pid_params.deadband,
 213                 0);
 214 }
 215
 216 static inline void intel_pstate_reset_all_pid(void)
 217 {
 218         unsigned int cpu;
 219         for_each_online_cpu(cpu) {
 220                 if (all_cpu_data[cpu])
 221                         intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
 222         }
 223 }
 224
 225 /************************** debugfs begin ************************/
 226 static int pid_param_set(void *data, u64 val)
 227 {
 228         *(u32 *)data = val;
 229         intel_pstate_reset_all_pid();
 230         return 0;
 231 }
 232 static int pid_param_get(void *data, u64 *val)
 233 {
 234         *val = *(u32 *)data;
 235         return 0;
 236 }
 237 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get,
 238                         pid_param_set, "%llu\n");
 239
 240 struct pid_param {
 241         char *name;
 242         void *value;
 243 };
 244
 245 static struct pid_param pid_files[] = {
 246         {"sample_rate_ms", &pid_params.sample_rate_ms},
 247         {"d_gain_pct", &pid_params.d_gain_pct},
 248         {"i_gain_pct", &pid_params.i_gain_pct},
 249         {"deadband", &pid_params.deadband},
 250         {"setpoint", &pid_params.setpoint},
 251         {"p_gain_pct", &pid_params.p_gain_pct},
 252         {NULL, NULL}
 253 };
 254
 255 static struct dentry *debugfs_parent;
 256 static void intel_pstate_debug_expose_params(void)
 257 {
 258         int i = 0;
 259
 260         debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
 261         if (IS_ERR_OR_NULL(debugfs_parent))
 262                 return;
 263         while (pid_files[i].name) {
 264                 debugfs_create_file(pid_files[i].name, 0660,
 265                                 debugfs_parent, pid_files[i].value,
 266                                 &fops_pid_param);
 267                 i++;
 268         }
 269 }
 270
 271 /************************** debugfs end ************************/
 272
 273 /************************** sysfs begin ************************/
 274 #define show_one(file_name, object)                                     \
 275         static ssize_t show_##file_name                                 \
 276         (struct kobject *kobj, struct attribute *attr, char *buf)       \
 277         {                                                               \
 278                 return sprintf(buf, "%u\n", limits.object);             \
 279         }
 280
 281 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 282                                 const char *buf, size_t count)
 283 {
 284         unsigned int input;
 285         int ret;
 286         ret = sscanf(buf, "%u", &input);
 287         if (ret != 1)
 288                 return -EINVAL;
 289         limits.no_turbo = clamp_t(int, input, 0 , 1);
 290
 291         return count;
 292 }
 293
 294 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 295                                 const char *buf, size_t count)
 296 {
 297         unsigned int input;
 298         int ret;
 299         ret = sscanf(buf, "%u", &input);
 300         if (ret != 1)
 301                 return -EINVAL;
 302
 303         limits.max_sysfs_pct = clamp_t(int, input, 0 , 100);
 304         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 305         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 306         return count;
 307 }
 308
 309 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 310                                 const char *buf, size_t count)
 311 {
 312         unsigned int input;
 313         int ret;
 314         ret = sscanf(buf, "%u", &input);
 315         if (ret != 1)
 316                 return -EINVAL;
 317         limits.min_perf_pct = clamp_t(int, input, 0 , 100);
 318         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 319
 320         return count;
 321 }
 322
 323 show_one(no_turbo, no_turbo);
 324 show_one(max_perf_pct, max_perf_pct);
 325 show_one(min_perf_pct, min_perf_pct);
 326
 327 define_one_global_rw(no_turbo);
 328 define_one_global_rw(max_perf_pct);
 329 define_one_global_rw(min_perf_pct);
 330
 331 static struct attribute *intel_pstate_attributes[] = {
 332         &no_turbo.attr,
 333         &max_perf_pct.attr,
 334         &min_perf_pct.attr,
 335         NULL
 336 };
 337
 338 static struct attribute_group intel_pstate_attr_group = {
 339         .attrs = intel_pstate_attributes,
 340 };
 341 static struct kobject *intel_pstate_kobject;
 342
 343 static void intel_pstate_sysfs_expose_params(void)
 344 {
 345         int rc;
 346
 347         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
 348                                                 &cpu_subsys.dev_root->kobj);
 349         BUG_ON(!intel_pstate_kobject);
 350         rc = sysfs_create_group(intel_pstate_kobject,
 351                                 &intel_pstate_attr_group);
 352         BUG_ON(rc);
 353 }
 354
 355 /************************** sysfs end ************************/
 356 static int byt_get_min_pstate(void)
 357 {
 358         u64 value;
 359         rdmsrl(BYT_RATIOS, value);
 360         return value & 0xFF;
 361 }
 362
 363 static int byt_get_max_pstate(void)
 364 {
 365         u64 value;
 366         rdmsrl(BYT_RATIOS, value);
 367         return (value >> 16) & 0xFF;
 368 }
 369
 370 static void byt_set_pstate(struct cpudata *cpudata, int pstate)
 371 {
 372         u64 val;
 373         int32_t vid_fp;
 374         u32 vid;
 375
 376         val = pstate << 8;
 377         if (limits.no_turbo)
 378                 val |= (u64)1 << 32;
 379
 380         vid_fp = cpudata->vid.min + mul_fp(
 381                 int_tofp(pstate - cpudata->pstate.min_pstate),
 382                 cpudata->vid.ratio);
 383
 384         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
 385         vid = fp_toint(vid_fp);
 386
 387         val |= vid;
 388
 389         wrmsrl(MSR_IA32_PERF_CTL, val);
 390 }
 391
 392 static void byt_get_vid(struct cpudata *cpudata)
 393 {
 394         u64 value;
 395
 396         rdmsrl(BYT_VIDS, value);
 397         cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
 398         cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
 399         cpudata->vid.ratio = div_fp(
 400                 cpudata->vid.max - cpudata->vid.min,
 401                 int_tofp(cpudata->pstate.max_pstate -
 402                         cpudata->pstate.min_pstate));
 403 }
 404
 405
 406 static int core_get_min_pstate(void)
 407 {
 408         u64 value;
 409         rdmsrl(MSR_PLATFORM_INFO, value);
 410         return (value >> 40) & 0xFF;
 411 }
 412
 413 static int core_get_max_pstate(void)
 414 {
 415         u64 value;
 416         rdmsrl(MSR_PLATFORM_INFO, value);
 417         return (value >> 8) & 0xFF;
 418 }
 419
 420 static int core_get_turbo_pstate(void)
 421 {
 422         u64 value;
 423         int nont, ret;
 424         rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
 425         nont = core_get_max_pstate();
 426         ret = ((value) & 255);
 427         if (ret <= nont)
 428                 ret = nont;
 429         return ret;
 430 }
 431
 432 static void core_set_pstate(struct cpudata *cpudata, int pstate)
 433 {
 434         u64 val;
 435
 436         val = pstate << 8;
 437         if (limits.no_turbo)
 438                 val |= (u64)1 << 32;
 439
 440         wrmsrl(MSR_IA32_PERF_CTL, val);
 441 }
 442
 443 static struct cpu_defaults core_params = {
 444         .pid_policy = {
 445                 .sample_rate_ms = 10,
 446                 .deadband = 0,
 447                 .setpoint = 97,
 448                 .p_gain_pct = 20,
 449                 .d_gain_pct = 0,
 450                 .i_gain_pct = 0,
 451         },
 452         .funcs = {
 453                 .get_max = core_get_max_pstate,
 454                 .get_min = core_get_min_pstate,
 455                 .get_turbo = core_get_turbo_pstate,
 456                 .set = core_set_pstate,
 457         },
 458 };
 459
 460 static struct cpu_defaults byt_params = {
 461         .pid_policy = {
 462                 .sample_rate_ms = 10,
 463                 .deadband = 0,
 464                 .setpoint = 97,
 465                 .p_gain_pct = 14,
 466                 .d_gain_pct = 0,
 467                 .i_gain_pct = 4,
 468         },
 469         .funcs = {
 470                 .get_max = byt_get_max_pstate,
 471                 .get_min = byt_get_min_pstate,
 472                 .get_turbo = byt_get_max_pstate,
 473                 .set = byt_set_pstate,
 474                 .get_vid = byt_get_vid,
 475         },
 476 };
 477
 478
 479 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
 480 {
 481         int max_perf = cpu->pstate.turbo_pstate;
 482         int max_perf_adj;
 483         int min_perf;
 484         if (limits.no_turbo)
 485                 max_perf = cpu->pstate.max_pstate;
 486
 487         max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf));
 488         *max = clamp_t(int, max_perf_adj,
 489                         cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
 490
 491         min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf));
 492         *min = clamp_t(int, min_perf,
 493                         cpu->pstate.min_pstate, max_perf);
 494 }
 495
 496 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
 497 {
 498         int max_perf, min_perf;
 499
 500         intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
 501
 502         pstate = clamp_t(int, pstate, min_perf, max_perf);
 503
 504         if (pstate == cpu->pstate.current_pstate)
 505                 return;
 506
 507         trace_cpu_frequency(pstate * 100000, cpu->cpu);
 508
 509         cpu->pstate.current_pstate = pstate;
 510
 511         pstate_funcs.set(cpu, pstate);
 512 }
 513
 514 static inline void intel_pstate_pstate_increase(struct cpudata *cpu, int steps)
 515 {
 516         int target;
 517         target = cpu->pstate.current_pstate + steps;
 518
 519         intel_pstate_set_pstate(cpu, target);
 520 }
 521
 522 static inline void intel_pstate_pstate_decrease(struct cpudata *cpu, int steps)
 523 {
 524         int target;
 525         target = cpu->pstate.current_pstate - steps;
 526         intel_pstate_set_pstate(cpu, target);
 527 }
 528
 529 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 530 {
 531         sprintf(cpu->name, "Intel 2nd generation core");
 532
 533         cpu->pstate.min_pstate = pstate_funcs.get_min();
 534         cpu->pstate.max_pstate = pstate_funcs.get_max();
 535         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
 536
 537         if (pstate_funcs.get_vid)
 538                 pstate_funcs.get_vid(cpu);
 539
 540         /*
 541          * goto max pstate so we don't slow up boot if we are built-in if we are
 542          * a module we will take care of it during normal operation
 543          */
 544         intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate);
 545 }
 546
 547 static inline void intel_pstate_calc_busy(struct cpudata *cpu,
 548                                         struct sample *sample)
 549 {
 550         u64 core_pct;
 551         core_pct = div64_u64(int_tofp(sample->aperf * 100),
 552                              sample->mperf);
 553         sample->freq = fp_toint(cpu->pstate.max_pstate * core_pct * 1000);
 554
 555         sample->core_pct_busy = core_pct;
 556 }
 557
 558 static inline void intel_pstate_sample(struct cpudata *cpu)
 559 {
 560         u64 aperf, mperf;
 561
 562         rdmsrl(MSR_IA32_APERF, aperf);
 563         rdmsrl(MSR_IA32_MPERF, mperf);
 564
 565         cpu->sample_ptr = (cpu->sample_ptr + 1) % SAMPLE_COUNT;
 566         cpu->samples[cpu->sample_ptr].aperf = aperf;
 567         cpu->samples[cpu->sample_ptr].mperf = mperf;
 568         cpu->samples[cpu->sample_ptr].aperf -= cpu->prev_aperf;
 569         cpu->samples[cpu->sample_ptr].mperf -= cpu->prev_mperf;
 570
 571         intel_pstate_calc_busy(cpu, &cpu->samples[cpu->sample_ptr]);
 572
 573         cpu->prev_aperf = aperf;
 574         cpu->prev_mperf = mperf;
 575 }
 576
 577 static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
 578 {
 579         int sample_time, delay;
 580
 581         sample_time = pid_params.sample_rate_ms;
 582         delay = msecs_to_jiffies(sample_time);
 583         mod_timer_pinned(&cpu->timer, jiffies + delay);
 584 }
 585
 586 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
 587 {
 588         int32_t core_busy, max_pstate, current_pstate;
 589
 590         core_busy = cpu->samples[cpu->sample_ptr].core_pct_busy;
 591         max_pstate = int_tofp(cpu->pstate.max_pstate);
 592         current_pstate = int_tofp(cpu->pstate.current_pstate);
 593         return mul_fp(core_busy, div_fp(max_pstate, current_pstate));
 594 }
 595
 596 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 597 {
 598         int32_t busy_scaled;
 599         struct _pid *pid;
 600         signed int ctl = 0;
 601         int steps;
 602
 603         pid = &cpu->pid;
 604         busy_scaled = intel_pstate_get_scaled_busy(cpu);
 605
 606         ctl = pid_calc(pid, busy_scaled);
 607
 608         steps = abs(ctl);
 609
 610         if (ctl < 0)
 611                 intel_pstate_pstate_increase(cpu, steps);
 612         else
 613                 intel_pstate_pstate_decrease(cpu, steps);
 614 }
 615
 616 static void intel_pstate_timer_func(unsigned long __data)
 617 {
 618         struct cpudata *cpu = (struct cpudata *) __data;
 619         struct sample *sample;
 620         u64 energy;
 621
 622         intel_pstate_sample(cpu);
 623
 624         sample = &cpu->samples[cpu->sample_ptr];
 625         rdmsrl(MSR_PKG_ENERGY_STATUS, energy);
 626
 627         intel_pstate_adjust_busy_pstate(cpu);
 628
 629         trace_pstate_sample(fp_toint(sample->core_pct_busy),
 630                         fp_toint(intel_pstate_get_scaled_busy(cpu)),
 631                         cpu->pstate.current_pstate,
 632                         sample->mperf,
 633                         sample->aperf,
 634                         div64_u64(energy, energy_divisor),
 635                         sample->freq);
 636
 637         intel_pstate_set_sample_time(cpu);
 638 }
 639
 640 #define ICPU(model, policy) \
 641         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
 642                         (unsigned long)&policy }
 643
 644 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 645         ICPU(0x2a, core_params),
 646         ICPU(0x2d, core_params),
 647         ICPU(0x37, byt_params),
 648         ICPU(0x3a, core_params),
 649         ICPU(0x3c, core_params),
 650         ICPU(0x3e, core_params),
 651         ICPU(0x3f, core_params),
 652         ICPU(0x45, core_params),
 653         ICPU(0x46, core_params),
 654         {}
 655 };
 656 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
 657
 658 static int intel_pstate_init_cpu(unsigned int cpunum)
 659 {
 660
 661         const struct x86_cpu_id *id;
 662         struct cpudata *cpu;
 663
 664         id = x86_match_cpu(intel_pstate_cpu_ids);
 665         if (!id)
 666                 return -ENODEV;
 667
 668         all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL);
 669         if (!all_cpu_data[cpunum])
 670                 return -ENOMEM;
 671
 672         cpu = all_cpu_data[cpunum];
 673
 674         intel_pstate_get_cpu_pstates(cpu);
 675         if (!cpu->pstate.current_pstate) {
 676                 all_cpu_data[cpunum] = NULL;
 677                 kfree(cpu);
 678                 return -ENODATA;
 679         }
 680
 681         cpu->cpu = cpunum;
 682
 683         init_timer_deferrable(&cpu->timer);
 684         cpu->timer.function = intel_pstate_timer_func;
 685         cpu->timer.data =
 686                 (unsigned long)cpu;
 687         cpu->timer.expires = jiffies + HZ/100;
 688         intel_pstate_busy_pid_reset(cpu);
 689         intel_pstate_sample(cpu);
 690         intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate);
 691
 692         add_timer_on(&cpu->timer, cpunum);
 693
 694         pr_info("Intel pstate controlling: cpu %d\n", cpunum);
 695
 696         return 0;
 697 }
 698
 699 static unsigned int intel_pstate_get(unsigned int cpu_num)
 700 {
 701         struct sample *sample;
 702         struct cpudata *cpu;
 703
 704         cpu = all_cpu_data[cpu_num];
 705         if (!cpu)
 706                 return 0;
 707         sample = &cpu->samples[cpu->sample_ptr];
 708         return sample->freq;
 709 }
 710
 711 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 712 {
 713         struct cpudata *cpu;
 714
 715         cpu = all_cpu_data[policy->cpu];
 716
 717         if (!policy->cpuinfo.max_freq)
 718                 return -ENODEV;
 719
 720         if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
 721                 limits.min_perf_pct = 100;
 722                 limits.min_perf = int_tofp(1);
 723                 limits.max_perf_pct = 100;
 724                 limits.max_perf = int_tofp(1);
 725                 limits.no_turbo = 0;
 726                 return 0;
 727         }
 728         limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
 729         limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100);
 730         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 731
 732         limits.max_policy_pct = policy->max * 100 / policy->cpuinfo.max_freq;
 733         limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100);
 734         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 735         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 736
 737         return 0;
 738 }
 739
 740 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
 741 {
 742         cpufreq_verify_within_cpu_limits(policy);
 743
 744         if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) &&
 745                 (policy->policy != CPUFREQ_POLICY_PERFORMANCE))
 746                 return -EINVAL;
 747
 748         return 0;
 749 }
 750
 751 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
 752 {
 753         int cpu = policy->cpu;
 754
 755         del_timer(&all_cpu_data[cpu]->timer);
 756         kfree(all_cpu_data[cpu]);
 757         all_cpu_data[cpu] = NULL;
 758         return 0;
 759 }
 760
 761 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 762 {
 763         struct cpudata *cpu;
 764         int rc;
 765
 766         rc = intel_pstate_init_cpu(policy->cpu);
 767         if (rc)
 768                 return rc;
 769
 770         cpu = all_cpu_data[policy->cpu];
 771
 772         if (!limits.no_turbo &&
 773                 limits.min_perf_pct == 100 && limits.max_perf_pct == 100)
 774                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 775         else
 776                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
 777
 778         policy->min = cpu->pstate.min_pstate * 100000;
 779         policy->max = cpu->pstate.turbo_pstate * 100000;
 780
 781         /* cpuinfo and default policy values */
 782         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000;
 783         policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000;
 784         policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 785         cpumask_set_cpu(policy->cpu, policy->cpus);
 786
 787         return 0;
 788 }
 789
 790 static struct cpufreq_driver intel_pstate_driver = {
 791         .flags          = CPUFREQ_CONST_LOOPS,
 792         .verify         = intel_pstate_verify_policy,
 793         .setpolicy      = intel_pstate_set_policy,
 794         .get            = intel_pstate_get,
 795         .init           = intel_pstate_cpu_init,
 796         .exit           = intel_pstate_cpu_exit,
 797         .name           = "intel_pstate",
 798 };
 799
 800 static int __initdata no_load;
 801
 802 static int intel_pstate_msrs_not_valid(void)
 803 {
 804         /* Check that all the msr's we are using are valid. */
 805         u64 aperf, mperf, tmp;
 806
 807         rdmsrl(MSR_IA32_APERF, aperf);
 808         rdmsrl(MSR_IA32_MPERF, mperf);
 809
 810         if (!pstate_funcs.get_max() ||
 811                 !pstate_funcs.get_min() ||
 812                 !pstate_funcs.get_turbo())
 813                 return -ENODEV;
 814
 815         rdmsrl(MSR_IA32_APERF, tmp);
 816         if (!(tmp - aperf))
 817                 return -ENODEV;
 818
 819         rdmsrl(MSR_IA32_MPERF, tmp);
 820         if (!(tmp - mperf))
 821                 return -ENODEV;
 822
 823         return 0;
 824 }
 825
 826 static void copy_pid_params(struct pstate_adjust_policy *policy)
 827 {
 828         pid_params.sample_rate_ms = policy->sample_rate_ms;
 829         pid_params.p_gain_pct = policy->p_gain_pct;
 830         pid_params.i_gain_pct = policy->i_gain_pct;
 831         pid_params.d_gain_pct = policy->d_gain_pct;
 832         pid_params.deadband = policy->deadband;
 833         pid_params.setpoint = policy->setpoint;
 834 }
 835
 836 static void copy_cpu_funcs(struct pstate_funcs *funcs)
 837 {
 838         pstate_funcs.get_max   = funcs->get_max;
 839         pstate_funcs.get_min   = funcs->get_min;
 840         pstate_funcs.get_turbo = funcs->get_turbo;
 841         pstate_funcs.set       = funcs->set;
 842         pstate_funcs.get_vid   = funcs->get_vid;
 843 }
 844
 845 #if IS_ENABLED(CONFIG_ACPI)
 846 #include <acpi/processor.h>
 847
 848 static bool intel_pstate_no_acpi_pss(void)
 849 {
 850         int i;
 851
 852         for_each_possible_cpu(i) {
 853                 acpi_status status;
 854                 union acpi_object *pss;
 855                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 856                 struct acpi_processor *pr = per_cpu(processors, i);
 857
 858                 if (!pr)
 859                         continue;
 860
 861                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
 862                 if (ACPI_FAILURE(status))
 863                         continue;
 864
 865                 pss = buffer.pointer;
 866                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
 867                         kfree(pss);
 868                         return false;
 869                 }
 870
 871                 kfree(pss);
 872         }
 873
 874         return true;
 875 }
 876
 877 struct hw_vendor_info {
 878         u16  valid;
 879         char oem_id[ACPI_OEM_ID_SIZE];
 880         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
 881 };
 882
 883 /* Hardware vendor-specific info that has its own power management modes */
 884 static struct hw_vendor_info vendor_info[] = {
 885         {1, "HP    ", "ProLiant"},
 886         {0, "", ""},
 887 };
 888
 889 static bool intel_pstate_platform_pwr_mgmt_exists(void)
 890 {
 891         struct acpi_table_header hdr;
 892         struct hw_vendor_info *v_info;
 893
 894         if (acpi_disabled
 895             || ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
 896                 return false;
 897
 898         for (v_info = vendor_info; v_info->valid; v_info++) {
 899                 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE)
 900                     && !strncmp(hdr.oem_table_id, v_info->oem_table_id, ACPI_OEM_TABLE_ID_SIZE)
 901                     && intel_pstate_no_acpi_pss())
 902                         return true;
 903         }
 904
 905         return false;
 906 }
 907 #else /* CONFIG_ACPI not enabled */
 908 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
 909 #endif /* CONFIG_ACPI */
 910
 911 static int __init intel_pstate_init(void)
 912 {
 913         int cpu, rc = 0;
 914         const struct x86_cpu_id *id;
 915         struct cpu_defaults *cpu_info;
 916         u64 units;
 917
 918         if (no_load)
 919                 return -ENODEV;
 920
 921         id = x86_match_cpu(intel_pstate_cpu_ids);
 922         if (!id)
 923                 return -ENODEV;
 924
 925         /*
 926          * The Intel pstate driver will be ignored if the platform
 927          * firmware has its own power management modes.
 928          */
 929         if (intel_pstate_platform_pwr_mgmt_exists())
 930                 return -ENODEV;
 931
 932         cpu_info = (struct cpu_defaults *)id->driver_data;
 933
 934         copy_pid_params(&cpu_info->pid_policy);
 935         copy_cpu_funcs(&cpu_info->funcs);
 936
 937         if (intel_pstate_msrs_not_valid())
 938                 return -ENODEV;
 939
 940         pr_info("Intel P-state driver initializing.\n");
 941
 942         all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
 943         if (!all_cpu_data)
 944                 return -ENOMEM;
 945
 946         rc = cpufreq_register_driver(&intel_pstate_driver);
 947         if (rc)
 948                 goto out;
 949
 950         rdmsrl(MSR_RAPL_POWER_UNIT, units);
 951         energy_divisor = 1 << ((units >> 8) & 0x1f); /* bits{12:8} */
 952
 953         intel_pstate_debug_expose_params();
 954         intel_pstate_sysfs_expose_params();
 955
 956         return rc;
 957 out:
 958         get_online_cpus();
 959         for_each_online_cpu(cpu) {
 960                 if (all_cpu_data[cpu]) {
 961                         del_timer_sync(&all_cpu_data[cpu]->timer);
 962                         kfree(all_cpu_data[cpu]);
 963                 }
 964         }
 965
 966         put_online_cpus();
 967         vfree(all_cpu_data);
 968         return -ENODEV;
 969 }
 970 device_initcall(intel_pstate_init);
 971
 972 static int __init intel_pstate_setup(char *str)
 973 {
 974         if (!str)
 975                 return -EINVAL;
 976
 977         if (!strcmp(str, "disable"))
 978                 no_load = 1;
 979         return 0;
 980 }
 981 early_param("intel_pstate", intel_pstate_setup);
 982
 983 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
 984 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
 985 MODULE_LICENSE("GPL");