drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #include <linux/kernel.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/module.h>
  16 #include <linux/ktime.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/tick.h>
  19 #include <linux/slab.h>
  20 #include <linux/sched.h>
  21 #include <linux/list.h>
  22 #include <linux/cpu.h>
  23 #include <linux/cpufreq.h>
  24 #include <linux/sysfs.h>
  25 #include <linux/types.h>
  26 #include <linux/fs.h>
  27 #include <linux/debugfs.h>
  28 #include <linux/acpi.h>
  29 #include <trace/events/power.h>
  30
  31 #include <asm/div64.h>
  32 #include <asm/msr.h>
  33 #include <asm/cpu_device_id.h>
  34
  35 #define BYT_RATIOS              0x66a
  36 #define BYT_VIDS                0x66b
  37 #define BYT_TURBO_RATIOS        0x66c
  38 #define BYT_TURBO_VIDS          0x66d
  39
  40
  41 #define FRAC_BITS 8
  42 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  43 #define fp_toint(X) ((X) >> FRAC_BITS)
  44
  45
  46 static inline int32_t mul_fp(int32_t x, int32_t y)
  47 {
  48         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  49 }
  50
  51 static inline int32_t div_fp(int32_t x, int32_t y)
  52 {
  53         return div_s64((int64_t)x << FRAC_BITS, (int64_t)y);
  54 }
  55
  56 struct sample {
  57         int32_t core_pct_busy;
  58         u64 aperf;
  59         u64 mperf;
  60         int freq;
  61         ktime_t time;
  62 };
  63
  64 struct pstate_data {
  65         int     current_pstate;
  66         int     min_pstate;
  67         int     max_pstate;
  68         int     turbo_pstate;
  69 };
  70
  71 struct vid_data {
  72         int min;
  73         int max;
  74         int turbo;
  75         int32_t ratio;
  76 };
  77
  78 struct _pid {
  79         int setpoint;
  80         int32_t integral;
  81         int32_t p_gain;
  82         int32_t i_gain;
  83         int32_t d_gain;
  84         int deadband;
  85         int32_t last_err;
  86 };
  87
  88 struct cpudata {
  89         int cpu;
  90
  91         struct timer_list timer;
  92
  93         struct pstate_data pstate;
  94         struct vid_data vid;
  95         struct _pid pid;
  96
  97         ktime_t last_sample_time;
  98         u64     prev_aperf;
  99         u64     prev_mperf;
 100         struct sample sample;
 101 };
 102
 103 static struct cpudata **all_cpu_data;
 104 struct pstate_adjust_policy {
 105         int sample_rate_ms;
 106         int deadband;
 107         int setpoint;
 108         int p_gain_pct;
 109         int d_gain_pct;
 110         int i_gain_pct;
 111 };
 112
 113 struct pstate_funcs {
 114         int (*get_max)(void);
 115         int (*get_min)(void);
 116         int (*get_turbo)(void);
 117         void (*set)(struct cpudata*, int pstate);
 118         void (*get_vid)(struct cpudata *);
 119 };
 120
 121 struct cpu_defaults {
 122         struct pstate_adjust_policy pid_policy;
 123         struct pstate_funcs funcs;
 124 };
 125
 126 static struct pstate_adjust_policy pid_params;
 127 static struct pstate_funcs pstate_funcs;
 128
 129 struct perf_limits {
 130         int no_turbo;
 131         int max_perf_pct;
 132         int min_perf_pct;
 133         int32_t max_perf;
 134         int32_t min_perf;
 135         int max_policy_pct;
 136         int max_sysfs_pct;
 137 };
 138
 139 static struct perf_limits limits = {
 140         .no_turbo = 0,
 141         .max_perf_pct = 100,
 142         .max_perf = int_tofp(1),
 143         .min_perf_pct = 0,
 144         .min_perf = 0,
 145         .max_policy_pct = 100,
 146         .max_sysfs_pct = 100,
 147 };
 148
 149 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
 150                         int deadband, int integral) {
 151         pid->setpoint = setpoint;
 152         pid->deadband  = deadband;
 153         pid->integral  = int_tofp(integral);
 154         pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
 155 }
 156
 157 static inline void pid_p_gain_set(struct _pid *pid, int percent)
 158 {
 159         pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
 160 }
 161
 162 static inline void pid_i_gain_set(struct _pid *pid, int percent)
 163 {
 164         pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
 165 }
 166
 167 static inline void pid_d_gain_set(struct _pid *pid, int percent)
 168 {
 169
 170         pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
 171 }
 172
 173 static signed int pid_calc(struct _pid *pid, int32_t busy)
 174 {
 175         signed int result;
 176         int32_t pterm, dterm, fp_error;
 177         int32_t integral_limit;
 178
 179         fp_error = int_tofp(pid->setpoint) - busy;
 180
 181         if (abs(fp_error) <= int_tofp(pid->deadband))
 182                 return 0;
 183
 184         pterm = mul_fp(pid->p_gain, fp_error);
 185
 186         pid->integral += fp_error;
 187
 188         /* limit the integral term */
 189         integral_limit = int_tofp(30);
 190         if (pid->integral > integral_limit)
 191                 pid->integral = integral_limit;
 192         if (pid->integral < -integral_limit)
 193                 pid->integral = -integral_limit;
 194
 195         dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
 196         pid->last_err = fp_error;
 197
 198         result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
 199         if (result >= 0)
 200                 result = result + (1 << (FRAC_BITS-1));
 201         else
 202                 result = result - (1 << (FRAC_BITS-1));
 203         return (signed int)fp_toint(result);
 204 }
 205
 206 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
 207 {
 208         pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
 209         pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
 210         pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
 211
 212         pid_reset(&cpu->pid,
 213                 pid_params.setpoint,
 214                 100,
 215                 pid_params.deadband,
 216                 0);
 217 }
 218
 219 static inline void intel_pstate_reset_all_pid(void)
 220 {
 221         unsigned int cpu;
 222         for_each_online_cpu(cpu) {
 223                 if (all_cpu_data[cpu])
 224                         intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
 225         }
 226 }
 227
 228 /************************** debugfs begin ************************/
 229 static int pid_param_set(void *data, u64 val)
 230 {
 231         *(u32 *)data = val;
 232         intel_pstate_reset_all_pid();
 233         return 0;
 234 }
 235 static int pid_param_get(void *data, u64 *val)
 236 {
 237         *val = *(u32 *)data;
 238         return 0;
 239 }
 240 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get,
 241                         pid_param_set, "%llu\n");
 242
 243 struct pid_param {
 244         char *name;
 245         void *value;
 246 };
 247
 248 static struct pid_param pid_files[] = {
 249         {"sample_rate_ms", &pid_params.sample_rate_ms},
 250         {"d_gain_pct", &pid_params.d_gain_pct},
 251         {"i_gain_pct", &pid_params.i_gain_pct},
 252         {"deadband", &pid_params.deadband},
 253         {"setpoint", &pid_params.setpoint},
 254         {"p_gain_pct", &pid_params.p_gain_pct},
 255         {NULL, NULL}
 256 };
 257
 258 static struct dentry *debugfs_parent;
 259 static void intel_pstate_debug_expose_params(void)
 260 {
 261         int i = 0;
 262
 263         debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
 264         if (IS_ERR_OR_NULL(debugfs_parent))
 265                 return;
 266         while (pid_files[i].name) {
 267                 debugfs_create_file(pid_files[i].name, 0660,
 268                                 debugfs_parent, pid_files[i].value,
 269                                 &fops_pid_param);
 270                 i++;
 271         }
 272 }
 273
 274 /************************** debugfs end ************************/
 275
 276 /************************** sysfs begin ************************/
 277 #define show_one(file_name, object)                                     \
 278         static ssize_t show_##file_name                                 \
 279         (struct kobject *kobj, struct attribute *attr, char *buf)       \
 280         {                                                               \
 281                 return sprintf(buf, "%u\n", limits.object);             \
 282         }
 283
 284 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 285                                 const char *buf, size_t count)
 286 {
 287         unsigned int input;
 288         int ret;
 289         ret = sscanf(buf, "%u", &input);
 290         if (ret != 1)
 291                 return -EINVAL;
 292         limits.no_turbo = clamp_t(int, input, 0 , 1);
 293
 294         return count;
 295 }
 296
 297 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 298                                 const char *buf, size_t count)
 299 {
 300         unsigned int input;
 301         int ret;
 302         ret = sscanf(buf, "%u", &input);
 303         if (ret != 1)
 304                 return -EINVAL;
 305
 306         limits.max_sysfs_pct = clamp_t(int, input, 0 , 100);
 307         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 308         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 309         return count;
 310 }
 311
 312 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 313                                 const char *buf, size_t count)
 314 {
 315         unsigned int input;
 316         int ret;
 317         ret = sscanf(buf, "%u", &input);
 318         if (ret != 1)
 319                 return -EINVAL;
 320         limits.min_perf_pct = clamp_t(int, input, 0 , 100);
 321         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 322
 323         return count;
 324 }
 325
 326 show_one(no_turbo, no_turbo);
 327 show_one(max_perf_pct, max_perf_pct);
 328 show_one(min_perf_pct, min_perf_pct);
 329
 330 define_one_global_rw(no_turbo);
 331 define_one_global_rw(max_perf_pct);
 332 define_one_global_rw(min_perf_pct);
 333
 334 static struct attribute *intel_pstate_attributes[] = {
 335         &no_turbo.attr,
 336         &max_perf_pct.attr,
 337         &min_perf_pct.attr,
 338         NULL
 339 };
 340
 341 static struct attribute_group intel_pstate_attr_group = {
 342         .attrs = intel_pstate_attributes,
 343 };
 344 static struct kobject *intel_pstate_kobject;
 345
 346 static void intel_pstate_sysfs_expose_params(void)
 347 {
 348         int rc;
 349
 350         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
 351                                                 &cpu_subsys.dev_root->kobj);
 352         BUG_ON(!intel_pstate_kobject);
 353         rc = sysfs_create_group(intel_pstate_kobject,
 354                                 &intel_pstate_attr_group);
 355         BUG_ON(rc);
 356 }
 357
 358 /************************** sysfs end ************************/
 359 static int byt_get_min_pstate(void)
 360 {
 361         u64 value;
 362         rdmsrl(BYT_RATIOS, value);
 363         return (value >> 8) & 0x3F;
 364 }
 365
 366 static int byt_get_max_pstate(void)
 367 {
 368         u64 value;
 369         rdmsrl(BYT_RATIOS, value);
 370         return (value >> 16) & 0x3F;
 371 }
 372
 373 static int byt_get_turbo_pstate(void)
 374 {
 375         u64 value;
 376         rdmsrl(BYT_TURBO_RATIOS, value);
 377         return value & 0x3F;
 378 }
 379
 380 static void byt_set_pstate(struct cpudata *cpudata, int pstate)
 381 {
 382         u64 val;
 383         int32_t vid_fp;
 384         u32 vid;
 385
 386         val = pstate << 8;
 387         if (limits.no_turbo)
 388                 val |= (u64)1 << 32;
 389
 390         vid_fp = cpudata->vid.min + mul_fp(
 391                 int_tofp(pstate - cpudata->pstate.min_pstate),
 392                 cpudata->vid.ratio);
 393
 394         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
 395         vid = fp_toint(vid_fp);
 396
 397         if (pstate > cpudata->pstate.max_pstate)
 398                 vid = cpudata->vid.turbo;
 399
 400         val |= vid;
 401
 402         wrmsrl(MSR_IA32_PERF_CTL, val);
 403 }
 404
 405 static void byt_get_vid(struct cpudata *cpudata)
 406 {
 407         u64 value;
 408
 409
 410         rdmsrl(BYT_VIDS, value);
 411         cpudata->vid.min = int_tofp((value >> 8) & 0x3f);
 412         cpudata->vid.max = int_tofp((value >> 16) & 0x3f);
 413         cpudata->vid.ratio = div_fp(
 414                 cpudata->vid.max - cpudata->vid.min,
 415                 int_tofp(cpudata->pstate.max_pstate -
 416                         cpudata->pstate.min_pstate));
 417
 418         rdmsrl(BYT_TURBO_VIDS, value);
 419         cpudata->vid.turbo = value & 0x7f;
 420 }
 421
 422
 423 static int core_get_min_pstate(void)
 424 {
 425         u64 value;
 426         rdmsrl(MSR_PLATFORM_INFO, value);
 427         return (value >> 40) & 0xFF;
 428 }
 429
 430 static int core_get_max_pstate(void)
 431 {
 432         u64 value;
 433         rdmsrl(MSR_PLATFORM_INFO, value);
 434         return (value >> 8) & 0xFF;
 435 }
 436
 437 static int core_get_turbo_pstate(void)
 438 {
 439         u64 value;
 440         int nont, ret;
 441         rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
 442         nont = core_get_max_pstate();
 443         ret = ((value) & 255);
 444         if (ret <= nont)
 445                 ret = nont;
 446         return ret;
 447 }
 448
 449 static void core_set_pstate(struct cpudata *cpudata, int pstate)
 450 {
 451         u64 val;
 452
 453         val = pstate << 8;
 454         if (limits.no_turbo)
 455                 val |= (u64)1 << 32;
 456
 457         wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
 458 }
 459
 460 static struct cpu_defaults core_params = {
 461         .pid_policy = {
 462                 .sample_rate_ms = 10,
 463                 .deadband = 0,
 464                 .setpoint = 97,
 465                 .p_gain_pct = 20,
 466                 .d_gain_pct = 0,
 467                 .i_gain_pct = 0,
 468         },
 469         .funcs = {
 470                 .get_max = core_get_max_pstate,
 471                 .get_min = core_get_min_pstate,
 472                 .get_turbo = core_get_turbo_pstate,
 473                 .set = core_set_pstate,
 474         },
 475 };
 476
 477 static struct cpu_defaults byt_params = {
 478         .pid_policy = {
 479                 .sample_rate_ms = 10,
 480                 .deadband = 0,
 481                 .setpoint = 97,
 482                 .p_gain_pct = 14,
 483                 .d_gain_pct = 0,
 484                 .i_gain_pct = 4,
 485         },
 486         .funcs = {
 487                 .get_max = byt_get_max_pstate,
 488                 .get_min = byt_get_min_pstate,
 489                 .get_turbo = byt_get_turbo_pstate,
 490                 .set = byt_set_pstate,
 491                 .get_vid = byt_get_vid,
 492         },
 493 };
 494
 495
 496 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
 497 {
 498         int max_perf = cpu->pstate.turbo_pstate;
 499         int max_perf_adj;
 500         int min_perf;
 501         if (limits.no_turbo)
 502                 max_perf = cpu->pstate.max_pstate;
 503
 504         max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf));
 505         *max = clamp_t(int, max_perf_adj,
 506                         cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
 507
 508         min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf));
 509         *min = clamp_t(int, min_perf,
 510                         cpu->pstate.min_pstate, max_perf);
 511 }
 512
 513 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
 514 {
 515         int max_perf, min_perf;
 516
 517         intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
 518
 519         pstate = clamp_t(int, pstate, min_perf, max_perf);
 520
 521         if (pstate == cpu->pstate.current_pstate)
 522                 return;
 523
 524         trace_cpu_frequency(pstate * 100000, cpu->cpu);
 525
 526         cpu->pstate.current_pstate = pstate;
 527
 528         pstate_funcs.set(cpu, pstate);
 529 }
 530
 531 static inline void intel_pstate_pstate_increase(struct cpudata *cpu, int steps)
 532 {
 533         int target;
 534         target = cpu->pstate.current_pstate + steps;
 535
 536         intel_pstate_set_pstate(cpu, target);
 537 }
 538
 539 static inline void intel_pstate_pstate_decrease(struct cpudata *cpu, int steps)
 540 {
 541         int target;
 542         target = cpu->pstate.current_pstate - steps;
 543         intel_pstate_set_pstate(cpu, target);
 544 }
 545
 546 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 547 {
 548         cpu->pstate.min_pstate = pstate_funcs.get_min();
 549         cpu->pstate.max_pstate = pstate_funcs.get_max();
 550         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
 551
 552         if (pstate_funcs.get_vid)
 553                 pstate_funcs.get_vid(cpu);
 554         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
 555 }
 556
 557 static inline void intel_pstate_calc_busy(struct cpudata *cpu)
 558 {
 559         struct sample *sample = &cpu->sample;
 560         int64_t core_pct;
 561         int32_t rem;
 562
 563         core_pct = int_tofp(sample->aperf) * int_tofp(100);
 564         core_pct = div_u64_rem(core_pct, int_tofp(sample->mperf), &rem);
 565
 566         if ((rem << 1) >= int_tofp(sample->mperf))
 567                 core_pct += 1;
 568
 569         sample->freq = fp_toint(
 570                 mul_fp(int_tofp(cpu->pstate.max_pstate * 1000), core_pct));
 571
 572         sample->core_pct_busy = (int32_t)core_pct;
 573 }
 574
 575 static inline void intel_pstate_sample(struct cpudata *cpu)
 576 {
 577         u64 aperf, mperf;
 578
 579         rdmsrl(MSR_IA32_APERF, aperf);
 580         rdmsrl(MSR_IA32_MPERF, mperf);
 581
 582         aperf = aperf >> FRAC_BITS;
 583         mperf = mperf >> FRAC_BITS;
 584
 585         cpu->last_sample_time = cpu->sample.time;
 586         cpu->sample.time = ktime_get();
 587         cpu->sample.aperf = aperf;
 588         cpu->sample.mperf = mperf;
 589         cpu->sample.aperf -= cpu->prev_aperf;
 590         cpu->sample.mperf -= cpu->prev_mperf;
 591
 592         intel_pstate_calc_busy(cpu);
 593
 594         cpu->prev_aperf = aperf;
 595         cpu->prev_mperf = mperf;
 596 }
 597
 598 static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
 599 {
 600         int sample_time, delay;
 601
 602         sample_time = pid_params.sample_rate_ms;
 603         delay = msecs_to_jiffies(sample_time);
 604         mod_timer_pinned(&cpu->timer, jiffies + delay);
 605 }
 606
 607 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
 608 {
 609         int32_t core_busy, max_pstate, current_pstate, sample_ratio;
 610         u32 duration_us;
 611         u32 sample_time;
 612
 613         core_busy = cpu->sample.core_pct_busy;
 614         max_pstate = int_tofp(cpu->pstate.max_pstate);
 615         current_pstate = int_tofp(cpu->pstate.current_pstate);
 616         core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
 617
 618         sample_time = (pid_params.sample_rate_ms  * USEC_PER_MSEC);
 619         duration_us = (u32) ktime_us_delta(cpu->sample.time,
 620                                         cpu->last_sample_time);
 621         if (duration_us > sample_time * 3) {
 622                 sample_ratio = div_fp(int_tofp(sample_time),
 623                                 int_tofp(duration_us));
 624                 core_busy = mul_fp(core_busy, sample_ratio);
 625         }
 626
 627         return core_busy;
 628 }
 629
 630 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 631 {
 632         int32_t busy_scaled;
 633         struct _pid *pid;
 634         signed int ctl = 0;
 635         int steps;
 636
 637         pid = &cpu->pid;
 638         busy_scaled = intel_pstate_get_scaled_busy(cpu);
 639
 640         ctl = pid_calc(pid, busy_scaled);
 641
 642         steps = abs(ctl);
 643
 644         if (ctl < 0)
 645                 intel_pstate_pstate_increase(cpu, steps);
 646         else
 647                 intel_pstate_pstate_decrease(cpu, steps);
 648 }
 649
 650 static void intel_pstate_timer_func(unsigned long __data)
 651 {
 652         struct cpudata *cpu = (struct cpudata *) __data;
 653         struct sample *sample;
 654
 655         intel_pstate_sample(cpu);
 656
 657         sample = &cpu->sample;
 658
 659         intel_pstate_adjust_busy_pstate(cpu);
 660
 661         trace_pstate_sample(fp_toint(sample->core_pct_busy),
 662                         fp_toint(intel_pstate_get_scaled_busy(cpu)),
 663                         cpu->pstate.current_pstate,
 664                         sample->mperf,
 665                         sample->aperf,
 666                         sample->freq);
 667
 668         intel_pstate_set_sample_time(cpu);
 669 }
 670
 671 #define ICPU(model, policy) \
 672         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
 673                         (unsigned long)&policy }
 674
 675 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 676         ICPU(0x2a, core_params),
 677         ICPU(0x2d, core_params),
 678         ICPU(0x37, byt_params),
 679         ICPU(0x3a, core_params),
 680         ICPU(0x3c, core_params),
 681         ICPU(0x3d, core_params),
 682         ICPU(0x3e, core_params),
 683         ICPU(0x3f, core_params),
 684         ICPU(0x45, core_params),
 685         ICPU(0x46, core_params),
 686         ICPU(0x4f, core_params),
 687         ICPU(0x56, core_params),
 688         {}
 689 };
 690 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
 691
 692 static int intel_pstate_init_cpu(unsigned int cpunum)
 693 {
 694         struct cpudata *cpu;
 695
 696         all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL);
 697         if (!all_cpu_data[cpunum])
 698                 return -ENOMEM;
 699
 700         cpu = all_cpu_data[cpunum];
 701
 702         intel_pstate_get_cpu_pstates(cpu);
 703
 704         cpu->cpu = cpunum;
 705
 706         init_timer_deferrable(&cpu->timer);
 707         cpu->timer.function = intel_pstate_timer_func;
 708         cpu->timer.data =
 709                 (unsigned long)cpu;
 710         cpu->timer.expires = jiffies + HZ/100;
 711         intel_pstate_busy_pid_reset(cpu);
 712         intel_pstate_sample(cpu);
 713
 714         add_timer_on(&cpu->timer, cpunum);
 715
 716         pr_info("Intel pstate controlling: cpu %d\n", cpunum);
 717
 718         return 0;
 719 }
 720
 721 static unsigned int intel_pstate_get(unsigned int cpu_num)
 722 {
 723         struct sample *sample;
 724         struct cpudata *cpu;
 725
 726         cpu = all_cpu_data[cpu_num];
 727         if (!cpu)
 728                 return 0;
 729         sample = &cpu->sample;
 730         return sample->freq;
 731 }
 732
 733 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 734 {
 735         struct cpudata *cpu;
 736
 737         cpu = all_cpu_data[policy->cpu];
 738
 739         if (!policy->cpuinfo.max_freq)
 740                 return -ENODEV;
 741
 742         if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
 743                 limits.min_perf_pct = 100;
 744                 limits.min_perf = int_tofp(1);
 745                 limits.max_perf_pct = 100;
 746                 limits.max_perf = int_tofp(1);
 747                 limits.no_turbo = 0;
 748                 return 0;
 749         }
 750         limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
 751         limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100);
 752         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 753
 754         limits.max_policy_pct = policy->max * 100 / policy->cpuinfo.max_freq;
 755         limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100);
 756         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 757         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 758
 759         return 0;
 760 }
 761
 762 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
 763 {
 764         cpufreq_verify_within_cpu_limits(policy);
 765
 766         if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) &&
 767                 (policy->policy != CPUFREQ_POLICY_PERFORMANCE))
 768                 return -EINVAL;
 769
 770         return 0;
 771 }
 772
 773 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
 774 {
 775         int cpu_num = policy->cpu;
 776         struct cpudata *cpu = all_cpu_data[cpu_num];
 777
 778         pr_info("intel_pstate CPU %d exiting\n", cpu_num);
 779
 780         del_timer_sync(&all_cpu_data[cpu_num]->timer);
 781         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
 782         kfree(all_cpu_data[cpu_num]);
 783         all_cpu_data[cpu_num] = NULL;
 784 }
 785
 786 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 787 {
 788         struct cpudata *cpu;
 789         int rc;
 790
 791         rc = intel_pstate_init_cpu(policy->cpu);
 792         if (rc)
 793                 return rc;
 794
 795         cpu = all_cpu_data[policy->cpu];
 796
 797         if (!limits.no_turbo &&
 798                 limits.min_perf_pct == 100 && limits.max_perf_pct == 100)
 799                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 800         else
 801                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
 802
 803         policy->min = cpu->pstate.min_pstate * 100000;
 804         policy->max = cpu->pstate.turbo_pstate * 100000;
 805
 806         /* cpuinfo and default policy values */
 807         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000;
 808         policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000;
 809         policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 810         cpumask_set_cpu(policy->cpu, policy->cpus);
 811
 812         return 0;
 813 }
 814
 815 static struct cpufreq_driver intel_pstate_driver = {
 816         .flags          = CPUFREQ_CONST_LOOPS,
 817         .verify         = intel_pstate_verify_policy,
 818         .setpolicy      = intel_pstate_set_policy,
 819         .get            = intel_pstate_get,
 820         .init           = intel_pstate_cpu_init,
 821         .stop_cpu       = intel_pstate_stop_cpu,
 822         .name           = "intel_pstate",
 823 };
 824
 825 static int __initdata no_load;
 826
 827 static int intel_pstate_msrs_not_valid(void)
 828 {
 829         /* Check that all the msr's we are using are valid. */
 830         u64 aperf, mperf, tmp;
 831
 832         rdmsrl(MSR_IA32_APERF, aperf);
 833         rdmsrl(MSR_IA32_MPERF, mperf);
 834
 835         if (!pstate_funcs.get_max() ||
 836                 !pstate_funcs.get_min() ||
 837                 !pstate_funcs.get_turbo())
 838                 return -ENODEV;
 839
 840         rdmsrl(MSR_IA32_APERF, tmp);
 841         if (!(tmp - aperf))
 842                 return -ENODEV;
 843
 844         rdmsrl(MSR_IA32_MPERF, tmp);
 845         if (!(tmp - mperf))
 846                 return -ENODEV;
 847
 848         return 0;
 849 }
 850
 851 static void copy_pid_params(struct pstate_adjust_policy *policy)
 852 {
 853         pid_params.sample_rate_ms = policy->sample_rate_ms;
 854         pid_params.p_gain_pct = policy->p_gain_pct;
 855         pid_params.i_gain_pct = policy->i_gain_pct;
 856         pid_params.d_gain_pct = policy->d_gain_pct;
 857         pid_params.deadband = policy->deadband;
 858         pid_params.setpoint = policy->setpoint;
 859 }
 860
 861 static void copy_cpu_funcs(struct pstate_funcs *funcs)
 862 {
 863         pstate_funcs.get_max   = funcs->get_max;
 864         pstate_funcs.get_min   = funcs->get_min;
 865         pstate_funcs.get_turbo = funcs->get_turbo;
 866         pstate_funcs.set       = funcs->set;
 867         pstate_funcs.get_vid   = funcs->get_vid;
 868 }
 869
 870 #if IS_ENABLED(CONFIG_ACPI)
 871 #include <acpi/processor.h>
 872
 873 static bool intel_pstate_no_acpi_pss(void)
 874 {
 875         int i;
 876
 877         for_each_possible_cpu(i) {
 878                 acpi_status status;
 879                 union acpi_object *pss;
 880                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 881                 struct acpi_processor *pr = per_cpu(processors, i);
 882
 883                 if (!pr)
 884                         continue;
 885
 886                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
 887                 if (ACPI_FAILURE(status))
 888                         continue;
 889
 890                 pss = buffer.pointer;
 891                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
 892                         kfree(pss);
 893                         return false;
 894                 }
 895
 896                 kfree(pss);
 897         }
 898
 899         return true;
 900 }
 901
 902 struct hw_vendor_info {
 903         u16  valid;
 904         char oem_id[ACPI_OEM_ID_SIZE];
 905         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
 906 };
 907
 908 /* Hardware vendor-specific info that has its own power management modes */
 909 static struct hw_vendor_info vendor_info[] = {
 910         {1, "HP    ", "ProLiant"},
 911         {0, "", ""},
 912 };
 913
 914 static bool intel_pstate_platform_pwr_mgmt_exists(void)
 915 {
 916         struct acpi_table_header hdr;
 917         struct hw_vendor_info *v_info;
 918
 919         if (acpi_disabled
 920             || ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
 921                 return false;
 922
 923         for (v_info = vendor_info; v_info->valid; v_info++) {
 924                 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE)
 925                     && !strncmp(hdr.oem_table_id, v_info->oem_table_id, ACPI_OEM_TABLE_ID_SIZE)
 926                     && intel_pstate_no_acpi_pss())
 927                         return true;
 928         }
 929
 930         return false;
 931 }
 932 #else /* CONFIG_ACPI not enabled */
 933 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
 934 #endif /* CONFIG_ACPI */
 935
 936 static int __init intel_pstate_init(void)
 937 {
 938         int cpu, rc = 0;
 939         const struct x86_cpu_id *id;
 940         struct cpu_defaults *cpu_info;
 941
 942         if (no_load)
 943                 return -ENODEV;
 944
 945         id = x86_match_cpu(intel_pstate_cpu_ids);
 946         if (!id)
 947                 return -ENODEV;
 948
 949         /*
 950          * The Intel pstate driver will be ignored if the platform
 951          * firmware has its own power management modes.
 952          */
 953         if (intel_pstate_platform_pwr_mgmt_exists())
 954                 return -ENODEV;
 955
 956         cpu_info = (struct cpu_defaults *)id->driver_data;
 957
 958         copy_pid_params(&cpu_info->pid_policy);
 959         copy_cpu_funcs(&cpu_info->funcs);
 960
 961         if (intel_pstate_msrs_not_valid())
 962                 return -ENODEV;
 963
 964         pr_info("Intel P-state driver initializing.\n");
 965
 966         all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
 967         if (!all_cpu_data)
 968                 return -ENOMEM;
 969
 970         rc = cpufreq_register_driver(&intel_pstate_driver);
 971         if (rc)
 972                 goto out;
 973
 974         intel_pstate_debug_expose_params();
 975         intel_pstate_sysfs_expose_params();
 976
 977         return rc;
 978 out:
 979         get_online_cpus();
 980         for_each_online_cpu(cpu) {
 981                 if (all_cpu_data[cpu]) {
 982                         del_timer_sync(&all_cpu_data[cpu]->timer);
 983                         kfree(all_cpu_data[cpu]);
 984                 }
 985         }
 986
 987         put_online_cpus();
 988         vfree(all_cpu_data);
 989         return -ENODEV;
 990 }
 991 device_initcall(intel_pstate_init);
 992
 993 static int __init intel_pstate_setup(char *str)
 994 {
 995         if (!str)
 996                 return -EINVAL;
 997
 998         if (!strcmp(str, "disable"))
 999                 no_load = 1;
1000         return 0;
1001 }
1002 early_param("intel_pstate", intel_pstate_setup);
1003
1004 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1005 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1006 MODULE_LICENSE("GPL");