drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 /**
  35  * The "cpufreq driver" - the arch- or hardware-dependent low
  36  * level driver of CPUFreq support, and its spinlock. This lock
  37  * also protects the cpufreq_cpu_data array.
  38  */
  39 static struct cpufreq_driver *cpufreq_driver;
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  42 static DEFINE_RWLOCK(cpufreq_driver_lock);
  43 DEFINE_MUTEX(cpufreq_governor_lock);
  44 static LIST_HEAD(cpufreq_policy_list);
  45
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48
  49 /* Flag to suspend/resume CPUFreq governors */
  50 static bool cpufreq_suspended;
  51
  52 static inline bool has_target(void)
  53 {
  54         return cpufreq_driver->target_index || cpufreq_driver->target;
  55 }
  56
  57 /*
  58  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  59  * sections
  60  */
  61 static DECLARE_RWSEM(cpufreq_rwsem);
  62
  63 /* internal prototypes */
  64 static int __cpufreq_governor(struct cpufreq_policy *policy,
  65                 unsigned int event);
  66 static unsigned int __cpufreq_get(unsigned int cpu);
  67 static void handle_update(struct work_struct *work);
  68
  69 /**
  70  * Two notifier lists: the "policy" list is involved in the
  71  * validation process for a new CPU frequency policy; the
  72  * "transition" list for kernel code that needs to handle
  73  * changes to devices when the CPU clock speed changes.
  74  * The mutex locks both lists.
  75  */
  76 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  77 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  78
  79 static bool init_cpufreq_transition_notifier_list_called;
  80 static int __init init_cpufreq_transition_notifier_list(void)
  81 {
  82         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  83         init_cpufreq_transition_notifier_list_called = true;
  84         return 0;
  85 }
  86 pure_initcall(init_cpufreq_transition_notifier_list);
  87
  88 static int off __read_mostly;
  89 static int cpufreq_disabled(void)
  90 {
  91         return off;
  92 }
  93 void disable_cpufreq(void)
  94 {
  95         off = 1;
  96 }
  97 static LIST_HEAD(cpufreq_governor_list);
  98 static DEFINE_MUTEX(cpufreq_governor_mutex);
  99
 100 bool have_governor_per_policy(void)
 101 {
 102         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 103 }
 104 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 105
 106 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 107 {
 108         if (have_governor_per_policy())
 109                 return &policy->kobj;
 110         else
 111                 return cpufreq_global_kobject;
 112 }
 113 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 114
 115 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 116 {
 117         u64 idle_time;
 118         u64 cur_wall_time;
 119         u64 busy_time;
 120
 121         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 122
 123         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 128         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 129
 130         idle_time = cur_wall_time - busy_time;
 131         if (wall)
 132                 *wall = cputime_to_usecs(cur_wall_time);
 133
 134         return cputime_to_usecs(idle_time);
 135 }
 136
 137 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 138 {
 139         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 140
 141         if (idle_time == -1ULL)
 142                 return get_cpu_idle_time_jiffy(cpu, wall);
 143         else if (!io_busy)
 144                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 145
 146         return idle_time;
 147 }
 148 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 149
 150 /*
 151  * This is a generic cpufreq init() routine which can be used by cpufreq
 152  * drivers of SMP systems. It will do following:
 153  * - validate & show freq table passed
 154  * - set policies transition latency
 155  * - policy->cpus with all possible CPUs
 156  */
 157 int cpufreq_generic_init(struct cpufreq_policy *policy,
 158                 struct cpufreq_frequency_table *table,
 159                 unsigned int transition_latency)
 160 {
 161         int ret;
 162
 163         ret = cpufreq_table_validate_and_show(policy, table);
 164         if (ret) {
 165                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 166                 return ret;
 167         }
 168
 169         policy->cpuinfo.transition_latency = transition_latency;
 170
 171         /*
 172          * The driver only supports the SMP configuartion where all processors
 173          * share the clock and voltage and clock.
 174          */
 175         cpumask_setall(policy->cpus);
 176
 177         return 0;
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 180
 181 unsigned int cpufreq_generic_get(unsigned int cpu)
 182 {
 183         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 184
 185         if (!policy || IS_ERR(policy->clk)) {
 186                 pr_err("%s: No %s associated to cpu: %d\n",
 187                        __func__, policy ? "clk" : "policy", cpu);
 188                 return 0;
 189         }
 190
 191         return clk_get_rate(policy->clk) / 1000;
 192 }
 193 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 194
 195 /* Only for cpufreq core internal use */
 196 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 197 {
 198         return per_cpu(cpufreq_cpu_data, cpu);
 199 }
 200
 201 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 202 {
 203         struct cpufreq_policy *policy = NULL;
 204         unsigned long flags;
 205
 206         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 207                 return NULL;
 208
 209         if (!down_read_trylock(&cpufreq_rwsem))
 210                 return NULL;
 211
 212         /* get the cpufreq driver */
 213         read_lock_irqsave(&cpufreq_driver_lock, flags);
 214
 215         if (cpufreq_driver) {
 216                 /* get the CPU */
 217                 policy = per_cpu(cpufreq_cpu_data, cpu);
 218                 if (policy)
 219                         kobject_get(&policy->kobj);
 220         }
 221
 222         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 223
 224         if (!policy)
 225                 up_read(&cpufreq_rwsem);
 226
 227         return policy;
 228 }
 229 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 230
 231 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 232 {
 233         if (cpufreq_disabled())
 234                 return;
 235
 236         kobject_put(&policy->kobj);
 237         up_read(&cpufreq_rwsem);
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 240
 241 /*********************************************************************
 242  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 243  *********************************************************************/
 244
 245 /**
 246  * adjust_jiffies - adjust the system "loops_per_jiffy"
 247  *
 248  * This function alters the system "loops_per_jiffy" for the clock
 249  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 250  * systems as each CPU might be scaled differently. So, use the arch
 251  * per-CPU loops_per_jiffy value wherever possible.
 252  */
 253 #ifndef CONFIG_SMP
 254 static unsigned long l_p_j_ref;
 255 static unsigned int l_p_j_ref_freq;
 256
 257 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 258 {
 259         if (ci->flags & CPUFREQ_CONST_LOOPS)
 260                 return;
 261
 262         if (!l_p_j_ref_freq) {
 263                 l_p_j_ref = loops_per_jiffy;
 264                 l_p_j_ref_freq = ci->old;
 265                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 266                          l_p_j_ref, l_p_j_ref_freq);
 267         }
 268         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 269                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 270                                                                 ci->new);
 271                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 272                          loops_per_jiffy, ci->new);
 273         }
 274 }
 275 #else
 276 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 277 {
 278         return;
 279 }
 280 #endif
 281
 282 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 283                 struct cpufreq_freqs *freqs, unsigned int state)
 284 {
 285         BUG_ON(irqs_disabled());
 286
 287         if (cpufreq_disabled())
 288                 return;
 289
 290         freqs->flags = cpufreq_driver->flags;
 291         pr_debug("notification %u of frequency transition to %u kHz\n",
 292                  state, freqs->new);
 293
 294         switch (state) {
 295
 296         case CPUFREQ_PRECHANGE:
 297                 /* detect if the driver reported a value as "old frequency"
 298                  * which is not equal to what the cpufreq core thinks is
 299                  * "old frequency".
 300                  */
 301                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 302                         if ((policy) && (policy->cpu == freqs->cpu) &&
 303                             (policy->cur) && (policy->cur != freqs->old)) {
 304                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 305                                          freqs->old, policy->cur);
 306                                 freqs->old = policy->cur;
 307                         }
 308                 }
 309                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 310                                 CPUFREQ_PRECHANGE, freqs);
 311                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 312                 break;
 313
 314         case CPUFREQ_POSTCHANGE:
 315                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 316                 pr_debug("FREQ: %lu - CPU: %lu\n",
 317                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 318                 trace_cpu_frequency(freqs->new, freqs->cpu);
 319                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 320                                 CPUFREQ_POSTCHANGE, freqs);
 321                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 322                         policy->cur = freqs->new;
 323                 break;
 324         }
 325 }
 326
 327 /**
 328  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 329  * on frequency transition.
 330  *
 331  * This function calls the transition notifiers and the "adjust_jiffies"
 332  * function. It is called twice on all CPU frequency changes that have
 333  * external effects.
 334  */
 335 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 336                 struct cpufreq_freqs *freqs, unsigned int state)
 337 {
 338         for_each_cpu(freqs->cpu, policy->cpus)
 339                 __cpufreq_notify_transition(policy, freqs, state);
 340 }
 341
 342 /* Do post notifications when there are chances that transition has failed */
 343 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 344                 struct cpufreq_freqs *freqs, int transition_failed)
 345 {
 346         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 347         if (!transition_failed)
 348                 return;
 349
 350         swap(freqs->old, freqs->new);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 352         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 353 }
 354
 355 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 356                 struct cpufreq_freqs *freqs)
 357 {
 358
 359         /*
 360          * Catch double invocations of _begin() which lead to self-deadlock.
 361          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 362          * doesn't invoke _begin() on their behalf, and hence the chances of
 363          * double invocations are very low. Moreover, there are scenarios
 364          * where these checks can emit false-positive warnings in these
 365          * drivers; so we avoid that by skipping them altogether.
 366          */
 367         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 368                                 && current == policy->transition_task);
 369
 370 wait:
 371         wait_event(policy->transition_wait, !policy->transition_ongoing);
 372
 373         spin_lock(&policy->transition_lock);
 374
 375         if (unlikely(policy->transition_ongoing)) {
 376                 spin_unlock(&policy->transition_lock);
 377                 goto wait;
 378         }
 379
 380         policy->transition_ongoing = true;
 381         policy->transition_task = current;
 382
 383         spin_unlock(&policy->transition_lock);
 384
 385         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 386 }
 387 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 388
 389 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 390                 struct cpufreq_freqs *freqs, int transition_failed)
 391 {
 392         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 393                 return;
 394
 395         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 396
 397         policy->transition_ongoing = false;
 398         policy->transition_task = NULL;
 399
 400         wake_up(&policy->transition_wait);
 401 }
 402 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 403
 404
 405 /*********************************************************************
 406  *                          SYSFS INTERFACE                          *
 407  *********************************************************************/
 408 static ssize_t show_boost(struct kobject *kobj,
 409                                  struct attribute *attr, char *buf)
 410 {
 411         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 412 }
 413
 414 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 415                                   const char *buf, size_t count)
 416 {
 417         int ret, enable;
 418
 419         ret = sscanf(buf, "%d", &enable);
 420         if (ret != 1 || enable < 0 || enable > 1)
 421                 return -EINVAL;
 422
 423         if (cpufreq_boost_trigger_state(enable)) {
 424                 pr_err("%s: Cannot %s BOOST!\n",
 425                        __func__, enable ? "enable" : "disable");
 426                 return -EINVAL;
 427         }
 428
 429         pr_debug("%s: cpufreq BOOST %s\n",
 430                  __func__, enable ? "enabled" : "disabled");
 431
 432         return count;
 433 }
 434 define_one_global_rw(boost);
 435
 436 static struct cpufreq_governor *__find_governor(const char *str_governor)
 437 {
 438         struct cpufreq_governor *t;
 439
 440         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 441                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 442                         return t;
 443
 444         return NULL;
 445 }
 446
 447 /**
 448  * cpufreq_parse_governor - parse a governor string
 449  */
 450 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 451                                 struct cpufreq_governor **governor)
 452 {
 453         int err = -EINVAL;
 454
 455         if (!cpufreq_driver)
 456                 goto out;
 457
 458         if (cpufreq_driver->setpolicy) {
 459                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 460                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 461                         err = 0;
 462                 } else if (!strncasecmp(str_governor, "powersave",
 463                                                 CPUFREQ_NAME_LEN)) {
 464                         *policy = CPUFREQ_POLICY_POWERSAVE;
 465                         err = 0;
 466                 }
 467         } else if (has_target()) {
 468                 struct cpufreq_governor *t;
 469
 470                 mutex_lock(&cpufreq_governor_mutex);
 471
 472                 t = __find_governor(str_governor);
 473
 474                 if (t == NULL) {
 475                         int ret;
 476
 477                         mutex_unlock(&cpufreq_governor_mutex);
 478                         ret = request_module("cpufreq_%s", str_governor);
 479                         mutex_lock(&cpufreq_governor_mutex);
 480
 481                         if (ret == 0)
 482                                 t = __find_governor(str_governor);
 483                 }
 484
 485                 if (t != NULL) {
 486                         *governor = t;
 487                         err = 0;
 488                 }
 489
 490                 mutex_unlock(&cpufreq_governor_mutex);
 491         }
 492 out:
 493         return err;
 494 }
 495
 496 /**
 497  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 498  * print out cpufreq information
 499  *
 500  * Write out information from cpufreq_driver->policy[cpu]; object must be
 501  * "unsigned int".
 502  */
 503
 504 #define show_one(file_name, object)                     \
 505 static ssize_t show_##file_name                         \
 506 (struct cpufreq_policy *policy, char *buf)              \
 507 {                                                       \
 508         return sprintf(buf, "%u\n", policy->object);    \
 509 }
 510
 511 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 512 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 513 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 514 show_one(scaling_min_freq, min);
 515 show_one(scaling_max_freq, max);
 516
 517 static ssize_t show_scaling_cur_freq(
 518         struct cpufreq_policy *policy, char *buf)
 519 {
 520         ssize_t ret;
 521
 522         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 523                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 524         else
 525                 ret = sprintf(buf, "%u\n", policy->cur);
 526         return ret;
 527 }
 528
 529 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 530                                 struct cpufreq_policy *new_policy);
 531
 532 /**
 533  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 534  */
 535 #define store_one(file_name, object)                    \
 536 static ssize_t store_##file_name                                        \
 537 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 538 {                                                                       \
 539         int ret, temp;                                                  \
 540         struct cpufreq_policy new_policy;                               \
 541                                                                         \
 542         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 543         if (ret)                                                        \
 544                 return -EINVAL;                                         \
 545                                                                         \
 546         ret = sscanf(buf, "%u", &new_policy.object);                    \
 547         if (ret != 1)                                                   \
 548                 return -EINVAL;                                         \
 549                                                                         \
 550         temp = new_policy.object;                                       \
 551         ret = cpufreq_set_policy(policy, &new_policy);          \
 552         if (!ret)                                                       \
 553                 policy->user_policy.object = temp;                      \
 554                                                                         \
 555         return ret ? ret : count;                                       \
 556 }
 557
 558 store_one(scaling_min_freq, min);
 559 store_one(scaling_max_freq, max);
 560
 561 /**
 562  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 563  */
 564 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 565                                         char *buf)
 566 {
 567         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 568         if (!cur_freq)
 569                 return sprintf(buf, "<unknown>");
 570         return sprintf(buf, "%u\n", cur_freq);
 571 }
 572
 573 /**
 574  * show_scaling_governor - show the current policy for the specified CPU
 575  */
 576 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 577 {
 578         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 579                 return sprintf(buf, "powersave\n");
 580         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 581                 return sprintf(buf, "performance\n");
 582         else if (policy->governor)
 583                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 584                                 policy->governor->name);
 585         return -EINVAL;
 586 }
 587
 588 /**
 589  * store_scaling_governor - store policy for the specified CPU
 590  */
 591 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 592                                         const char *buf, size_t count)
 593 {
 594         int ret;
 595         char    str_governor[16];
 596         struct cpufreq_policy new_policy;
 597
 598         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 599         if (ret)
 600                 return ret;
 601
 602         ret = sscanf(buf, "%15s", str_governor);
 603         if (ret != 1)
 604                 return -EINVAL;
 605
 606         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 607                                                 &new_policy.governor))
 608                 return -EINVAL;
 609
 610         ret = cpufreq_set_policy(policy, &new_policy);
 611
 612         policy->user_policy.policy = policy->policy;
 613         policy->user_policy.governor = policy->governor;
 614
 615         if (ret)
 616                 return ret;
 617         else
 618                 return count;
 619 }
 620
 621 /**
 622  * show_scaling_driver - show the cpufreq driver currently loaded
 623  */
 624 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 625 {
 626         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 627 }
 628
 629 /**
 630  * show_scaling_available_governors - show the available CPUfreq governors
 631  */
 632 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 633                                                 char *buf)
 634 {
 635         ssize_t i = 0;
 636         struct cpufreq_governor *t;
 637
 638         if (!has_target()) {
 639                 i += sprintf(buf, "performance powersave");
 640                 goto out;
 641         }
 642
 643         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 644                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 645                     - (CPUFREQ_NAME_LEN + 2)))
 646                         goto out;
 647                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 648         }
 649 out:
 650         i += sprintf(&buf[i], "\n");
 651         return i;
 652 }
 653
 654 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 655 {
 656         ssize_t i = 0;
 657         unsigned int cpu;
 658
 659         for_each_cpu(cpu, mask) {
 660                 if (i)
 661                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 662                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 663                 if (i >= (PAGE_SIZE - 5))
 664                         break;
 665         }
 666         i += sprintf(&buf[i], "\n");
 667         return i;
 668 }
 669 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 670
 671 /**
 672  * show_related_cpus - show the CPUs affected by each transition even if
 673  * hw coordination is in use
 674  */
 675 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 676 {
 677         return cpufreq_show_cpus(policy->related_cpus, buf);
 678 }
 679
 680 /**
 681  * show_affected_cpus - show the CPUs affected by each transition
 682  */
 683 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 684 {
 685         return cpufreq_show_cpus(policy->cpus, buf);
 686 }
 687
 688 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 689                                         const char *buf, size_t count)
 690 {
 691         unsigned int freq = 0;
 692         unsigned int ret;
 693
 694         if (!policy->governor || !policy->governor->store_setspeed)
 695                 return -EINVAL;
 696
 697         ret = sscanf(buf, "%u", &freq);
 698         if (ret != 1)
 699                 return -EINVAL;
 700
 701         policy->governor->store_setspeed(policy, freq);
 702
 703         return count;
 704 }
 705
 706 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 707 {
 708         if (!policy->governor || !policy->governor->show_setspeed)
 709                 return sprintf(buf, "<unsupported>\n");
 710
 711         return policy->governor->show_setspeed(policy, buf);
 712 }
 713
 714 /**
 715  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 716  */
 717 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 718 {
 719         unsigned int limit;
 720         int ret;
 721         if (cpufreq_driver->bios_limit) {
 722                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 723                 if (!ret)
 724                         return sprintf(buf, "%u\n", limit);
 725         }
 726         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 727 }
 728
 729 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 730 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 731 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 732 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 733 cpufreq_freq_attr_ro(scaling_available_governors);
 734 cpufreq_freq_attr_ro(scaling_driver);
 735 cpufreq_freq_attr_ro(scaling_cur_freq);
 736 cpufreq_freq_attr_ro(bios_limit);
 737 cpufreq_freq_attr_ro(related_cpus);
 738 cpufreq_freq_attr_ro(affected_cpus);
 739 cpufreq_freq_attr_rw(scaling_min_freq);
 740 cpufreq_freq_attr_rw(scaling_max_freq);
 741 cpufreq_freq_attr_rw(scaling_governor);
 742 cpufreq_freq_attr_rw(scaling_setspeed);
 743
 744 static struct attribute *default_attrs[] = {
 745         &cpuinfo_min_freq.attr,
 746         &cpuinfo_max_freq.attr,
 747         &cpuinfo_transition_latency.attr,
 748         &scaling_min_freq.attr,
 749         &scaling_max_freq.attr,
 750         &affected_cpus.attr,
 751         &related_cpus.attr,
 752         &scaling_governor.attr,
 753         &scaling_driver.attr,
 754         &scaling_available_governors.attr,
 755         &scaling_setspeed.attr,
 756         NULL
 757 };
 758
 759 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 760 #define to_attr(a) container_of(a, struct freq_attr, attr)
 761
 762 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 763 {
 764         struct cpufreq_policy *policy = to_policy(kobj);
 765         struct freq_attr *fattr = to_attr(attr);
 766         ssize_t ret;
 767
 768         if (!down_read_trylock(&cpufreq_rwsem))
 769                 return -EINVAL;
 770
 771         down_read(&policy->rwsem);
 772
 773         if (fattr->show)
 774                 ret = fattr->show(policy, buf);
 775         else
 776                 ret = -EIO;
 777
 778         up_read(&policy->rwsem);
 779         up_read(&cpufreq_rwsem);
 780
 781         return ret;
 782 }
 783
 784 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 785                      const char *buf, size_t count)
 786 {
 787         struct cpufreq_policy *policy = to_policy(kobj);
 788         struct freq_attr *fattr = to_attr(attr);
 789         ssize_t ret = -EINVAL;
 790
 791         get_online_cpus();
 792
 793         if (!cpu_online(policy->cpu))
 794                 goto unlock;
 795
 796         if (!down_read_trylock(&cpufreq_rwsem))
 797                 goto unlock;
 798
 799         down_write(&policy->rwsem);
 800
 801         if (fattr->store)
 802                 ret = fattr->store(policy, buf, count);
 803         else
 804                 ret = -EIO;
 805
 806         up_write(&policy->rwsem);
 807
 808         up_read(&cpufreq_rwsem);
 809 unlock:
 810         put_online_cpus();
 811
 812         return ret;
 813 }
 814
 815 static void cpufreq_sysfs_release(struct kobject *kobj)
 816 {
 817         struct cpufreq_policy *policy = to_policy(kobj);
 818         pr_debug("last reference is dropped\n");
 819         complete(&policy->kobj_unregister);
 820 }
 821
 822 static const struct sysfs_ops sysfs_ops = {
 823         .show   = show,
 824         .store  = store,
 825 };
 826
 827 static struct kobj_type ktype_cpufreq = {
 828         .sysfs_ops      = &sysfs_ops,
 829         .default_attrs  = default_attrs,
 830         .release        = cpufreq_sysfs_release,
 831 };
 832
 833 struct kobject *cpufreq_global_kobject;
 834 EXPORT_SYMBOL(cpufreq_global_kobject);
 835
 836 static int cpufreq_global_kobject_usage;
 837
 838 int cpufreq_get_global_kobject(void)
 839 {
 840         if (!cpufreq_global_kobject_usage++)
 841                 return kobject_add(cpufreq_global_kobject,
 842                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 843
 844         return 0;
 845 }
 846 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 847
 848 void cpufreq_put_global_kobject(void)
 849 {
 850         if (!--cpufreq_global_kobject_usage)
 851                 kobject_del(cpufreq_global_kobject);
 852 }
 853 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 854
 855 int cpufreq_sysfs_create_file(const struct attribute *attr)
 856 {
 857         int ret = cpufreq_get_global_kobject();
 858
 859         if (!ret) {
 860                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 861                 if (ret)
 862                         cpufreq_put_global_kobject();
 863         }
 864
 865         return ret;
 866 }
 867 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 868
 869 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 870 {
 871         sysfs_remove_file(cpufreq_global_kobject, attr);
 872         cpufreq_put_global_kobject();
 873 }
 874 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 875
 876 /* symlink affected CPUs */
 877 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 878 {
 879         unsigned int j;
 880         int ret = 0;
 881
 882         for_each_cpu(j, policy->cpus) {
 883                 struct device *cpu_dev;
 884
 885                 if (j == policy->cpu)
 886                         continue;
 887
 888                 pr_debug("Adding link for CPU: %u\n", j);
 889                 cpu_dev = get_cpu_device(j);
 890                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 891                                         "cpufreq");
 892                 if (ret)
 893                         break;
 894         }
 895         return ret;
 896 }
 897
 898 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 899                                      struct device *dev)
 900 {
 901         struct freq_attr **drv_attr;
 902         int ret = 0;
 903
 904         /* set up files for this cpu device */
 905         drv_attr = cpufreq_driver->attr;
 906         while (drv_attr && *drv_attr) {
 907                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 908                 if (ret)
 909                         return ret;
 910                 drv_attr++;
 911         }
 912         if (cpufreq_driver->get) {
 913                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 914                 if (ret)
 915                         return ret;
 916         }
 917
 918         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 919         if (ret)
 920                 return ret;
 921
 922         if (cpufreq_driver->bios_limit) {
 923                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 924                 if (ret)
 925                         return ret;
 926         }
 927
 928         return cpufreq_add_dev_symlink(policy);
 929 }
 930
 931 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 932 {
 933         struct cpufreq_governor *gov = NULL;
 934         struct cpufreq_policy new_policy;
 935         int ret = 0;
 936
 937         memcpy(&new_policy, policy, sizeof(*policy));
 938
 939         /* Update governor of new_policy to the governor used before hotplug */
 940         gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 941         if (gov)
 942                 pr_debug("Restoring governor %s for cpu %d\n",
 943                                 policy->governor->name, policy->cpu);
 944         else
 945                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 946
 947         new_policy.governor = gov;
 948
 949         /* Use the default policy if its valid. */
 950         if (cpufreq_driver->setpolicy)
 951                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 952
 953         /* set default policy */
 954         ret = cpufreq_set_policy(policy, &new_policy);
 955         if (ret) {
 956                 pr_debug("setting policy failed\n");
 957                 if (cpufreq_driver->exit)
 958                         cpufreq_driver->exit(policy);
 959         }
 960 }
 961
 962 #ifdef CONFIG_HOTPLUG_CPU
 963 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 964                                   unsigned int cpu, struct device *dev)
 965 {
 966         int ret = 0;
 967         unsigned long flags;
 968
 969         if (has_target()) {
 970                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 971                 if (ret) {
 972                         pr_err("%s: Failed to stop governor\n", __func__);
 973                         return ret;
 974                 }
 975         }
 976
 977         down_write(&policy->rwsem);
 978
 979         write_lock_irqsave(&cpufreq_driver_lock, flags);
 980
 981         cpumask_set_cpu(cpu, policy->cpus);
 982         per_cpu(cpufreq_cpu_data, cpu) = policy;
 983         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 984
 985         up_write(&policy->rwsem);
 986
 987         if (has_target()) {
 988                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 989                 if (!ret)
 990                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 991
 992                 if (ret) {
 993                         pr_err("%s: Failed to start governor\n", __func__);
 994                         return ret;
 995                 }
 996         }
 997
 998         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 999 }
1000 #endif
1001
1002 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1003 {
1004         struct cpufreq_policy *policy;
1005         unsigned long flags;
1006
1007         read_lock_irqsave(&cpufreq_driver_lock, flags);
1008
1009         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1010
1011         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1012
1013         if (policy)
1014                 policy->governor = NULL;
1015
1016         return policy;
1017 }
1018
1019 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1020 {
1021         struct cpufreq_policy *policy;
1022
1023         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1024         if (!policy)
1025                 return NULL;
1026
1027         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1028                 goto err_free_policy;
1029
1030         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1031                 goto err_free_cpumask;
1032
1033         INIT_LIST_HEAD(&policy->policy_list);
1034         init_rwsem(&policy->rwsem);
1035         spin_lock_init(&policy->transition_lock);
1036         init_waitqueue_head(&policy->transition_wait);
1037
1038         return policy;
1039
1040 err_free_cpumask:
1041         free_cpumask_var(policy->cpus);
1042 err_free_policy:
1043         kfree(policy);
1044
1045         return NULL;
1046 }
1047
1048 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1049 {
1050         struct kobject *kobj;
1051         struct completion *cmp;
1052
1053         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1054                         CPUFREQ_REMOVE_POLICY, policy);
1055
1056         down_read(&policy->rwsem);
1057         kobj = &policy->kobj;
1058         cmp = &policy->kobj_unregister;
1059         up_read(&policy->rwsem);
1060         kobject_put(kobj);
1061
1062         /*
1063          * We need to make sure that the underlying kobj is
1064          * actually not referenced anymore by anybody before we
1065          * proceed with unloading.
1066          */
1067         pr_debug("waiting for dropping of refcount\n");
1068         wait_for_completion(cmp);
1069         pr_debug("wait complete\n");
1070 }
1071
1072 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1073 {
1074         free_cpumask_var(policy->related_cpus);
1075         free_cpumask_var(policy->cpus);
1076         kfree(policy);
1077 }
1078
1079 static int update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu,
1080                              struct device *cpu_dev)
1081 {
1082         int ret;
1083
1084         if (WARN_ON(cpu == policy->cpu))
1085                 return 0;
1086
1087         /* Move kobject to the new policy->cpu */
1088         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1089         if (ret) {
1090                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1091                 return ret;
1092         }
1093
1094         down_write(&policy->rwsem);
1095
1096         policy->last_cpu = policy->cpu;
1097         policy->cpu = cpu;
1098
1099         up_write(&policy->rwsem);
1100
1101         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1102                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1103
1104         return 0;
1105 }
1106
1107 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1108 {
1109         unsigned int j, cpu = dev->id;
1110         int ret = -ENOMEM;
1111         struct cpufreq_policy *policy;
1112         unsigned long flags;
1113         bool recover_policy = cpufreq_suspended;
1114 #ifdef CONFIG_HOTPLUG_CPU
1115         struct cpufreq_policy *tpolicy;
1116 #endif
1117
1118         if (cpu_is_offline(cpu))
1119                 return 0;
1120
1121         pr_debug("adding CPU %u\n", cpu);
1122
1123 #ifdef CONFIG_SMP
1124         /* check whether a different CPU already registered this
1125          * CPU because it is in the same boat. */
1126         policy = cpufreq_cpu_get(cpu);
1127         if (unlikely(policy)) {
1128                 cpufreq_cpu_put(policy);
1129                 return 0;
1130         }
1131 #endif
1132
1133         if (!down_read_trylock(&cpufreq_rwsem))
1134                 return 0;
1135
1136 #ifdef CONFIG_HOTPLUG_CPU
1137         /* Check if this cpu was hot-unplugged earlier and has siblings */
1138         read_lock_irqsave(&cpufreq_driver_lock, flags);
1139         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1140                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1141                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1142                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1143                         up_read(&cpufreq_rwsem);
1144                         return ret;
1145                 }
1146         }
1147         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1148 #endif
1149
1150         /*
1151          * Restore the saved policy when doing light-weight init and fall back
1152          * to the full init if that fails.
1153          */
1154         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1155         if (!policy) {
1156                 recover_policy = false;
1157                 policy = cpufreq_policy_alloc();
1158                 if (!policy)
1159                         goto nomem_out;
1160         }
1161
1162         /*
1163          * In the resume path, since we restore a saved policy, the assignment
1164          * to policy->cpu is like an update of the existing policy, rather than
1165          * the creation of a brand new one. So we need to perform this update
1166          * by invoking update_policy_cpu().
1167          */
1168         if (recover_policy && cpu != policy->cpu)
1169                 WARN_ON(update_policy_cpu(policy, cpu, dev));
1170         else
1171                 policy->cpu = cpu;
1172
1173         cpumask_copy(policy->cpus, cpumask_of(cpu));
1174
1175         init_completion(&policy->kobj_unregister);
1176         INIT_WORK(&policy->update, handle_update);
1177
1178         /* call driver. From then on the cpufreq must be able
1179          * to accept all calls to ->verify and ->setpolicy for this CPU
1180          */
1181         ret = cpufreq_driver->init(policy);
1182         if (ret) {
1183                 pr_debug("initialization failed\n");
1184                 goto err_set_policy_cpu;
1185         }
1186
1187         down_write(&policy->rwsem);
1188
1189         /* related cpus should atleast have policy->cpus */
1190         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1191
1192         /*
1193          * affected cpus must always be the one, which are online. We aren't
1194          * managing offline cpus here.
1195          */
1196         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1197
1198         if (!recover_policy) {
1199                 policy->user_policy.min = policy->min;
1200                 policy->user_policy.max = policy->max;
1201
1202                 /* prepare interface data */
1203                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1204                                            &dev->kobj, "cpufreq");
1205                 if (ret) {
1206                         pr_err("%s: failed to init policy->kobj: %d\n",
1207                                __func__, ret);
1208                         goto err_init_policy_kobj;
1209                 }
1210         }
1211
1212         write_lock_irqsave(&cpufreq_driver_lock, flags);
1213         for_each_cpu(j, policy->cpus)
1214                 per_cpu(cpufreq_cpu_data, j) = policy;
1215         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1216
1217         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1218                 policy->cur = cpufreq_driver->get(policy->cpu);
1219                 if (!policy->cur) {
1220                         pr_err("%s: ->get() failed\n", __func__);
1221                         goto err_get_freq;
1222                 }
1223         }
1224
1225         /*
1226          * Sometimes boot loaders set CPU frequency to a value outside of
1227          * frequency table present with cpufreq core. In such cases CPU might be
1228          * unstable if it has to run on that frequency for long duration of time
1229          * and so its better to set it to a frequency which is specified in
1230          * freq-table. This also makes cpufreq stats inconsistent as
1231          * cpufreq-stats would fail to register because current frequency of CPU
1232          * isn't found in freq-table.
1233          *
1234          * Because we don't want this change to effect boot process badly, we go
1235          * for the next freq which is >= policy->cur ('cur' must be set by now,
1236          * otherwise we will end up setting freq to lowest of the table as 'cur'
1237          * is initialized to zero).
1238          *
1239          * We are passing target-freq as "policy->cur - 1" otherwise
1240          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1241          * equal to target-freq.
1242          */
1243         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1244             && has_target()) {
1245                 /* Are we running at unknown frequency ? */
1246                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1247                 if (ret == -EINVAL) {
1248                         /* Warn user and fix it */
1249                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1250                                 __func__, policy->cpu, policy->cur);
1251                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1252                                 CPUFREQ_RELATION_L);
1253
1254                         /*
1255                          * Reaching here after boot in a few seconds may not
1256                          * mean that system will remain stable at "unknown"
1257                          * frequency for longer duration. Hence, a BUG_ON().
1258                          */
1259                         BUG_ON(ret);
1260                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1261                                 __func__, policy->cpu, policy->cur);
1262                 }
1263         }
1264
1265         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1266                                      CPUFREQ_START, policy);
1267
1268         if (!recover_policy) {
1269                 ret = cpufreq_add_dev_interface(policy, dev);
1270                 if (ret)
1271                         goto err_out_unregister;
1272                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1273                                 CPUFREQ_CREATE_POLICY, policy);
1274         }
1275
1276         write_lock_irqsave(&cpufreq_driver_lock, flags);
1277         list_add(&policy->policy_list, &cpufreq_policy_list);
1278         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1279
1280         cpufreq_init_policy(policy);
1281
1282         if (!recover_policy) {
1283                 policy->user_policy.policy = policy->policy;
1284                 policy->user_policy.governor = policy->governor;
1285         }
1286         up_write(&policy->rwsem);
1287
1288         kobject_uevent(&policy->kobj, KOBJ_ADD);
1289
1290         up_read(&cpufreq_rwsem);
1291
1292         /* Callback for handling stuff after policy is ready */
1293         if (cpufreq_driver->ready)
1294                 cpufreq_driver->ready(policy);
1295
1296         pr_debug("initialization complete\n");
1297
1298         return 0;
1299
1300 err_out_unregister:
1301 err_get_freq:
1302         write_lock_irqsave(&cpufreq_driver_lock, flags);
1303         for_each_cpu(j, policy->cpus)
1304                 per_cpu(cpufreq_cpu_data, j) = NULL;
1305         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1306
1307         if (!recover_policy) {
1308                 kobject_put(&policy->kobj);
1309                 wait_for_completion(&policy->kobj_unregister);
1310         }
1311 err_init_policy_kobj:
1312         up_write(&policy->rwsem);
1313
1314         if (cpufreq_driver->exit)
1315                 cpufreq_driver->exit(policy);
1316 err_set_policy_cpu:
1317         if (recover_policy) {
1318                 /* Do not leave stale fallback data behind. */
1319                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1320                 cpufreq_policy_put_kobj(policy);
1321         }
1322         cpufreq_policy_free(policy);
1323
1324 nomem_out:
1325         up_read(&cpufreq_rwsem);
1326
1327         return ret;
1328 }
1329
1330 /**
1331  * cpufreq_add_dev - add a CPU device
1332  *
1333  * Adds the cpufreq interface for a CPU device.
1334  *
1335  * The Oracle says: try running cpufreq registration/unregistration concurrently
1336  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1337  * mess up, but more thorough testing is needed. - Mathieu
1338  */
1339 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1340 {
1341         return __cpufreq_add_dev(dev, sif);
1342 }
1343
1344 static int __cpufreq_remove_dev_prepare(struct device *dev,
1345                                         struct subsys_interface *sif)
1346 {
1347         unsigned int cpu = dev->id, cpus;
1348         int ret;
1349         unsigned long flags;
1350         struct cpufreq_policy *policy;
1351
1352         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1353
1354         write_lock_irqsave(&cpufreq_driver_lock, flags);
1355
1356         policy = per_cpu(cpufreq_cpu_data, cpu);
1357
1358         /* Save the policy somewhere when doing a light-weight tear-down */
1359         if (cpufreq_suspended)
1360                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1361
1362         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1363
1364         if (!policy) {
1365                 pr_debug("%s: No cpu_data found\n", __func__);
1366                 return -EINVAL;
1367         }
1368
1369         if (has_target()) {
1370                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1371                 if (ret) {
1372                         pr_err("%s: Failed to stop governor\n", __func__);
1373                         return ret;
1374                 }
1375         }
1376
1377         if (!cpufreq_driver->setpolicy)
1378                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1379                         policy->governor->name, CPUFREQ_NAME_LEN);
1380
1381         down_read(&policy->rwsem);
1382         cpus = cpumask_weight(policy->cpus);
1383         up_read(&policy->rwsem);
1384
1385         if (cpu != policy->cpu) {
1386                 sysfs_remove_link(&dev->kobj, "cpufreq");
1387         } else if (cpus > 1) {
1388                 /* Nominate new CPU */
1389                 int new_cpu = cpumask_any_but(policy->cpus, cpu);
1390                 struct device *cpu_dev = get_cpu_device(new_cpu);
1391
1392                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1393                 ret = update_policy_cpu(policy, new_cpu, cpu_dev);
1394                 if (ret) {
1395                         if (sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1396                                               "cpufreq"))
1397                                 pr_err("%s: Failed to restore kobj link to cpu:%d\n",
1398                                        __func__, cpu_dev->id);
1399                         return ret;
1400                 }
1401
1402                 if (!cpufreq_suspended)
1403                         pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1404                                  __func__, new_cpu, cpu);
1405         } else if (cpufreq_driver->stop_cpu) {
1406                 cpufreq_driver->stop_cpu(policy);
1407         }
1408
1409         return 0;
1410 }
1411
1412 static int __cpufreq_remove_dev_finish(struct device *dev,
1413                                        struct subsys_interface *sif)
1414 {
1415         unsigned int cpu = dev->id, cpus;
1416         int ret;
1417         unsigned long flags;
1418         struct cpufreq_policy *policy;
1419
1420         read_lock_irqsave(&cpufreq_driver_lock, flags);
1421         policy = per_cpu(cpufreq_cpu_data, cpu);
1422         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1423
1424         if (!policy) {
1425                 pr_debug("%s: No cpu_data found\n", __func__);
1426                 return -EINVAL;
1427         }
1428
1429         down_write(&policy->rwsem);
1430         cpus = cpumask_weight(policy->cpus);
1431
1432         if (cpus > 1)
1433                 cpumask_clear_cpu(cpu, policy->cpus);
1434         up_write(&policy->rwsem);
1435
1436         /* If cpu is last user of policy, free policy */
1437         if (cpus == 1) {
1438                 if (has_target()) {
1439                         ret = __cpufreq_governor(policy,
1440                                         CPUFREQ_GOV_POLICY_EXIT);
1441                         if (ret) {
1442                                 pr_err("%s: Failed to exit governor\n",
1443                                        __func__);
1444                                 return ret;
1445                         }
1446                 }
1447
1448                 if (!cpufreq_suspended)
1449                         cpufreq_policy_put_kobj(policy);
1450
1451                 /*
1452                  * Perform the ->exit() even during light-weight tear-down,
1453                  * since this is a core component, and is essential for the
1454                  * subsequent light-weight ->init() to succeed.
1455                  */
1456                 if (cpufreq_driver->exit)
1457                         cpufreq_driver->exit(policy);
1458
1459                 /* Remove policy from list of active policies */
1460                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1461                 list_del(&policy->policy_list);
1462                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1463
1464                 if (!cpufreq_suspended)
1465                         cpufreq_policy_free(policy);
1466         } else if (has_target()) {
1467                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1468                 if (!ret)
1469                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1470
1471                 if (ret) {
1472                         pr_err("%s: Failed to start governor\n", __func__);
1473                         return ret;
1474                 }
1475         }
1476
1477         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1478         return 0;
1479 }
1480
1481 /**
1482  * cpufreq_remove_dev - remove a CPU device
1483  *
1484  * Removes the cpufreq interface for a CPU device.
1485  */
1486 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1487 {
1488         unsigned int cpu = dev->id;
1489         int ret;
1490
1491         if (cpu_is_offline(cpu))
1492                 return 0;
1493
1494         ret = __cpufreq_remove_dev_prepare(dev, sif);
1495
1496         if (!ret)
1497                 ret = __cpufreq_remove_dev_finish(dev, sif);
1498
1499         return ret;
1500 }
1501
1502 static void handle_update(struct work_struct *work)
1503 {
1504         struct cpufreq_policy *policy =
1505                 container_of(work, struct cpufreq_policy, update);
1506         unsigned int cpu = policy->cpu;
1507         pr_debug("handle_update for cpu %u called\n", cpu);
1508         cpufreq_update_policy(cpu);
1509 }
1510
1511 /**
1512  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1513  *      in deep trouble.
1514  *      @cpu: cpu number
1515  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1516  *      @new_freq: CPU frequency the CPU actually runs at
1517  *
1518  *      We adjust to current frequency first, and need to clean up later.
1519  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1520  */
1521 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1522                                 unsigned int new_freq)
1523 {
1524         struct cpufreq_policy *policy;
1525         struct cpufreq_freqs freqs;
1526         unsigned long flags;
1527
1528         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1529                  old_freq, new_freq);
1530
1531         freqs.old = old_freq;
1532         freqs.new = new_freq;
1533
1534         read_lock_irqsave(&cpufreq_driver_lock, flags);
1535         policy = per_cpu(cpufreq_cpu_data, cpu);
1536         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1537
1538         cpufreq_freq_transition_begin(policy, &freqs);
1539         cpufreq_freq_transition_end(policy, &freqs, 0);
1540 }
1541
1542 /**
1543  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1544  * @cpu: CPU number
1545  *
1546  * This is the last known freq, without actually getting it from the driver.
1547  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1548  */
1549 unsigned int cpufreq_quick_get(unsigned int cpu)
1550 {
1551         struct cpufreq_policy *policy;
1552         unsigned int ret_freq = 0;
1553
1554         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1555                 return cpufreq_driver->get(cpu);
1556
1557         policy = cpufreq_cpu_get(cpu);
1558         if (policy) {
1559                 ret_freq = policy->cur;
1560                 cpufreq_cpu_put(policy);
1561         }
1562
1563         return ret_freq;
1564 }
1565 EXPORT_SYMBOL(cpufreq_quick_get);
1566
1567 /**
1568  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1569  * @cpu: CPU number
1570  *
1571  * Just return the max possible frequency for a given CPU.
1572  */
1573 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1574 {
1575         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1576         unsigned int ret_freq = 0;
1577
1578         if (policy) {
1579                 ret_freq = policy->max;
1580                 cpufreq_cpu_put(policy);
1581         }
1582
1583         return ret_freq;
1584 }
1585 EXPORT_SYMBOL(cpufreq_quick_get_max);
1586
1587 static unsigned int __cpufreq_get(unsigned int cpu)
1588 {
1589         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1590         unsigned int ret_freq = 0;
1591
1592         if (!cpufreq_driver->get)
1593                 return ret_freq;
1594
1595         ret_freq = cpufreq_driver->get(cpu);
1596
1597         if (ret_freq && policy->cur &&
1598                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1599                 /* verify no discrepancy between actual and
1600                                         saved value exists */
1601                 if (unlikely(ret_freq != policy->cur)) {
1602                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1603                         schedule_work(&policy->update);
1604                 }
1605         }
1606
1607         return ret_freq;
1608 }
1609
1610 /**
1611  * cpufreq_get - get the current CPU frequency (in kHz)
1612  * @cpu: CPU number
1613  *
1614  * Get the CPU current (static) CPU frequency
1615  */
1616 unsigned int cpufreq_get(unsigned int cpu)
1617 {
1618         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1619         unsigned int ret_freq = 0;
1620
1621         if (policy) {
1622                 down_read(&policy->rwsem);
1623                 ret_freq = __cpufreq_get(cpu);
1624                 up_read(&policy->rwsem);
1625
1626                 cpufreq_cpu_put(policy);
1627         }
1628
1629         return ret_freq;
1630 }
1631 EXPORT_SYMBOL(cpufreq_get);
1632
1633 static struct subsys_interface cpufreq_interface = {
1634         .name           = "cpufreq",
1635         .subsys         = &cpu_subsys,
1636         .add_dev        = cpufreq_add_dev,
1637         .remove_dev     = cpufreq_remove_dev,
1638 };
1639
1640 /*
1641  * In case platform wants some specific frequency to be configured
1642  * during suspend..
1643  */
1644 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1645 {
1646         int ret;
1647
1648         if (!policy->suspend_freq) {
1649                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1650                 return -EINVAL;
1651         }
1652
1653         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1654                         policy->suspend_freq);
1655
1656         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1657                         CPUFREQ_RELATION_H);
1658         if (ret)
1659                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1660                                 __func__, policy->suspend_freq, ret);
1661
1662         return ret;
1663 }
1664 EXPORT_SYMBOL(cpufreq_generic_suspend);
1665
1666 /**
1667  * cpufreq_suspend() - Suspend CPUFreq governors
1668  *
1669  * Called during system wide Suspend/Hibernate cycles for suspending governors
1670  * as some platforms can't change frequency after this point in suspend cycle.
1671  * Because some of the devices (like: i2c, regulators, etc) they use for
1672  * changing frequency are suspended quickly after this point.
1673  */
1674 void cpufreq_suspend(void)
1675 {
1676         struct cpufreq_policy *policy;
1677
1678         if (!cpufreq_driver)
1679                 return;
1680
1681         if (!has_target())
1682                 goto suspend;
1683
1684         pr_debug("%s: Suspending Governors\n", __func__);
1685
1686         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1687                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1688                         pr_err("%s: Failed to stop governor for policy: %p\n",
1689                                 __func__, policy);
1690                 else if (cpufreq_driver->suspend
1691                     && cpufreq_driver->suspend(policy))
1692                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1693                                 policy);
1694         }
1695
1696 suspend:
1697         cpufreq_suspended = true;
1698 }
1699
1700 /**
1701  * cpufreq_resume() - Resume CPUFreq governors
1702  *
1703  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1704  * are suspended with cpufreq_suspend().
1705  */
1706 void cpufreq_resume(void)
1707 {
1708         struct cpufreq_policy *policy;
1709
1710         if (!cpufreq_driver)
1711                 return;
1712
1713         cpufreq_suspended = false;
1714
1715         if (!has_target())
1716                 return;
1717
1718         pr_debug("%s: Resuming Governors\n", __func__);
1719
1720         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1721                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1722                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1723                                 policy);
1724                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1725                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1726                         pr_err("%s: Failed to start governor for policy: %p\n",
1727                                 __func__, policy);
1728
1729                 /*
1730                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1731                  * policy in list. It will verify that the current freq is in
1732                  * sync with what we believe it to be.
1733                  */
1734                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1735                         schedule_work(&policy->update);
1736         }
1737 }
1738
1739 /**
1740  *      cpufreq_get_current_driver - return current driver's name
1741  *
1742  *      Return the name string of the currently loaded cpufreq driver
1743  *      or NULL, if none.
1744  */
1745 const char *cpufreq_get_current_driver(void)
1746 {
1747         if (cpufreq_driver)
1748                 return cpufreq_driver->name;
1749
1750         return NULL;
1751 }
1752 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1753
1754 /**
1755  *      cpufreq_get_driver_data - return current driver data
1756  *
1757  *      Return the private data of the currently loaded cpufreq
1758  *      driver, or NULL if no cpufreq driver is loaded.
1759  */
1760 void *cpufreq_get_driver_data(void)
1761 {
1762         if (cpufreq_driver)
1763                 return cpufreq_driver->driver_data;
1764
1765         return NULL;
1766 }
1767 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1768
1769 /*********************************************************************
1770  *                     NOTIFIER LISTS INTERFACE                      *
1771  *********************************************************************/
1772
1773 /**
1774  *      cpufreq_register_notifier - register a driver with cpufreq
1775  *      @nb: notifier function to register
1776  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1777  *
1778  *      Add a driver to one of two lists: either a list of drivers that
1779  *      are notified about clock rate changes (once before and once after
1780  *      the transition), or a list of drivers that are notified about
1781  *      changes in cpufreq policy.
1782  *
1783  *      This function may sleep, and has the same return conditions as
1784  *      blocking_notifier_chain_register.
1785  */
1786 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1787 {
1788         int ret;
1789
1790         if (cpufreq_disabled())
1791                 return -EINVAL;
1792
1793         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1794
1795         switch (list) {
1796         case CPUFREQ_TRANSITION_NOTIFIER:
1797                 ret = srcu_notifier_chain_register(
1798                                 &cpufreq_transition_notifier_list, nb);
1799                 break;
1800         case CPUFREQ_POLICY_NOTIFIER:
1801                 ret = blocking_notifier_chain_register(
1802                                 &cpufreq_policy_notifier_list, nb);
1803                 break;
1804         default:
1805                 ret = -EINVAL;
1806         }
1807
1808         return ret;
1809 }
1810 EXPORT_SYMBOL(cpufreq_register_notifier);
1811
1812 /**
1813  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1814  *      @nb: notifier block to be unregistered
1815  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1816  *
1817  *      Remove a driver from the CPU frequency notifier list.
1818  *
1819  *      This function may sleep, and has the same return conditions as
1820  *      blocking_notifier_chain_unregister.
1821  */
1822 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1823 {
1824         int ret;
1825
1826         if (cpufreq_disabled())
1827                 return -EINVAL;
1828
1829         switch (list) {
1830         case CPUFREQ_TRANSITION_NOTIFIER:
1831                 ret = srcu_notifier_chain_unregister(
1832                                 &cpufreq_transition_notifier_list, nb);
1833                 break;
1834         case CPUFREQ_POLICY_NOTIFIER:
1835                 ret = blocking_notifier_chain_unregister(
1836                                 &cpufreq_policy_notifier_list, nb);
1837                 break;
1838         default:
1839                 ret = -EINVAL;
1840         }
1841
1842         return ret;
1843 }
1844 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1845
1846
1847 /*********************************************************************
1848  *                              GOVERNORS                            *
1849  *********************************************************************/
1850
1851 /* Must set freqs->new to intermediate frequency */
1852 static int __target_intermediate(struct cpufreq_policy *policy,
1853                                  struct cpufreq_freqs *freqs, int index)
1854 {
1855         int ret;
1856
1857         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1858
1859         /* We don't need to switch to intermediate freq */
1860         if (!freqs->new)
1861                 return 0;
1862
1863         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1864                  __func__, policy->cpu, freqs->old, freqs->new);
1865
1866         cpufreq_freq_transition_begin(policy, freqs);
1867         ret = cpufreq_driver->target_intermediate(policy, index);
1868         cpufreq_freq_transition_end(policy, freqs, ret);
1869
1870         if (ret)
1871                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1872                        __func__, ret);
1873
1874         return ret;
1875 }
1876
1877 static int __target_index(struct cpufreq_policy *policy,
1878                           struct cpufreq_frequency_table *freq_table, int index)
1879 {
1880         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1881         unsigned int intermediate_freq = 0;
1882         int retval = -EINVAL;
1883         bool notify;
1884
1885         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1886         if (notify) {
1887                 /* Handle switching to intermediate frequency */
1888                 if (cpufreq_driver->get_intermediate) {
1889                         retval = __target_intermediate(policy, &freqs, index);
1890                         if (retval)
1891                                 return retval;
1892
1893                         intermediate_freq = freqs.new;
1894                         /* Set old freq to intermediate */
1895                         if (intermediate_freq)
1896                                 freqs.old = freqs.new;
1897                 }
1898
1899                 freqs.new = freq_table[index].frequency;
1900                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1901                          __func__, policy->cpu, freqs.old, freqs.new);
1902
1903                 cpufreq_freq_transition_begin(policy, &freqs);
1904         }
1905
1906         retval = cpufreq_driver->target_index(policy, index);
1907         if (retval)
1908                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1909                        retval);
1910
1911         if (notify) {
1912                 cpufreq_freq_transition_end(policy, &freqs, retval);
1913
1914                 /*
1915                  * Failed after setting to intermediate freq? Driver should have
1916                  * reverted back to initial frequency and so should we. Check
1917                  * here for intermediate_freq instead of get_intermediate, in
1918                  * case we have't switched to intermediate freq at all.
1919                  */
1920                 if (unlikely(retval && intermediate_freq)) {
1921                         freqs.old = intermediate_freq;
1922                         freqs.new = policy->restore_freq;
1923                         cpufreq_freq_transition_begin(policy, &freqs);
1924                         cpufreq_freq_transition_end(policy, &freqs, 0);
1925                 }
1926         }
1927
1928         return retval;
1929 }
1930
1931 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1932                             unsigned int target_freq,
1933                             unsigned int relation)
1934 {
1935         unsigned int old_target_freq = target_freq;
1936         int retval = -EINVAL;
1937
1938         if (cpufreq_disabled())
1939                 return -ENODEV;
1940
1941         /* Make sure that target_freq is within supported range */
1942         if (target_freq > policy->max)
1943                 target_freq = policy->max;
1944         if (target_freq < policy->min)
1945                 target_freq = policy->min;
1946
1947         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1948                  policy->cpu, target_freq, relation, old_target_freq);
1949
1950         /*
1951          * This might look like a redundant call as we are checking it again
1952          * after finding index. But it is left intentionally for cases where
1953          * exactly same freq is called again and so we can save on few function
1954          * calls.
1955          */
1956         if (target_freq == policy->cur)
1957                 return 0;
1958
1959         /* Save last value to restore later on errors */
1960         policy->restore_freq = policy->cur;
1961
1962         if (cpufreq_driver->target)
1963                 retval = cpufreq_driver->target(policy, target_freq, relation);
1964         else if (cpufreq_driver->target_index) {
1965                 struct cpufreq_frequency_table *freq_table;
1966                 int index;
1967
1968                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1969                 if (unlikely(!freq_table)) {
1970                         pr_err("%s: Unable to find freq_table\n", __func__);
1971                         goto out;
1972                 }
1973
1974                 retval = cpufreq_frequency_table_target(policy, freq_table,
1975                                 target_freq, relation, &index);
1976                 if (unlikely(retval)) {
1977                         pr_err("%s: Unable to find matching freq\n", __func__);
1978                         goto out;
1979                 }
1980
1981                 if (freq_table[index].frequency == policy->cur) {
1982                         retval = 0;
1983                         goto out;
1984                 }
1985
1986                 retval = __target_index(policy, freq_table, index);
1987         }
1988
1989 out:
1990         return retval;
1991 }
1992 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1993
1994 int cpufreq_driver_target(struct cpufreq_policy *policy,
1995                           unsigned int target_freq,
1996                           unsigned int relation)
1997 {
1998         int ret = -EINVAL;
1999
2000         down_write(&policy->rwsem);
2001
2002         ret = __cpufreq_driver_target(policy, target_freq, relation);
2003
2004         up_write(&policy->rwsem);
2005
2006         return ret;
2007 }
2008 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2009
2010 static int __cpufreq_governor(struct cpufreq_policy *policy,
2011                                         unsigned int event)
2012 {
2013         int ret;
2014
2015         /* Only must be defined when default governor is known to have latency
2016            restrictions, like e.g. conservative or ondemand.
2017            That this is the case is already ensured in Kconfig
2018         */
2019 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2020         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2021 #else
2022         struct cpufreq_governor *gov = NULL;
2023 #endif
2024
2025         /* Don't start any governor operations if we are entering suspend */
2026         if (cpufreq_suspended)
2027                 return 0;
2028         /*
2029          * Governor might not be initiated here if ACPI _PPC changed
2030          * notification happened, so check it.
2031          */
2032         if (!policy->governor)
2033                 return -EINVAL;
2034
2035         if (policy->governor->max_transition_latency &&
2036             policy->cpuinfo.transition_latency >
2037             policy->governor->max_transition_latency) {
2038                 if (!gov)
2039                         return -EINVAL;
2040                 else {
2041                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2042                                 policy->governor->name, gov->name);
2043                         policy->governor = gov;
2044                 }
2045         }
2046
2047         if (event == CPUFREQ_GOV_POLICY_INIT)
2048                 if (!try_module_get(policy->governor->owner))
2049                         return -EINVAL;
2050
2051         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2052                  policy->cpu, event);
2053
2054         mutex_lock(&cpufreq_governor_lock);
2055         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2056             || (!policy->governor_enabled
2057             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2058                 mutex_unlock(&cpufreq_governor_lock);
2059                 return -EBUSY;
2060         }
2061
2062         if (event == CPUFREQ_GOV_STOP)
2063                 policy->governor_enabled = false;
2064         else if (event == CPUFREQ_GOV_START)
2065                 policy->governor_enabled = true;
2066
2067         mutex_unlock(&cpufreq_governor_lock);
2068
2069         ret = policy->governor->governor(policy, event);
2070
2071         if (!ret) {
2072                 if (event == CPUFREQ_GOV_POLICY_INIT)
2073                         policy->governor->initialized++;
2074                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2075                         policy->governor->initialized--;
2076         } else {
2077                 /* Restore original values */
2078                 mutex_lock(&cpufreq_governor_lock);
2079                 if (event == CPUFREQ_GOV_STOP)
2080                         policy->governor_enabled = true;
2081                 else if (event == CPUFREQ_GOV_START)
2082                         policy->governor_enabled = false;
2083                 mutex_unlock(&cpufreq_governor_lock);
2084         }
2085
2086         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2087                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2088                 module_put(policy->governor->owner);
2089
2090         return ret;
2091 }
2092
2093 int cpufreq_register_governor(struct cpufreq_governor *governor)
2094 {
2095         int err;
2096
2097         if (!governor)
2098                 return -EINVAL;
2099
2100         if (cpufreq_disabled())
2101                 return -ENODEV;
2102
2103         mutex_lock(&cpufreq_governor_mutex);
2104
2105         governor->initialized = 0;
2106         err = -EBUSY;
2107         if (__find_governor(governor->name) == NULL) {
2108                 err = 0;
2109                 list_add(&governor->governor_list, &cpufreq_governor_list);
2110         }
2111
2112         mutex_unlock(&cpufreq_governor_mutex);
2113         return err;
2114 }
2115 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2116
2117 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2118 {
2119         int cpu;
2120
2121         if (!governor)
2122                 return;
2123
2124         if (cpufreq_disabled())
2125                 return;
2126
2127         for_each_present_cpu(cpu) {
2128                 if (cpu_online(cpu))
2129                         continue;
2130                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2131                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2132         }
2133
2134         mutex_lock(&cpufreq_governor_mutex);
2135         list_del(&governor->governor_list);
2136         mutex_unlock(&cpufreq_governor_mutex);
2137         return;
2138 }
2139 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2140
2141
2142 /*********************************************************************
2143  *                          POLICY INTERFACE                         *
2144  *********************************************************************/
2145
2146 /**
2147  * cpufreq_get_policy - get the current cpufreq_policy
2148  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2149  *      is written
2150  *
2151  * Reads the current cpufreq policy.
2152  */
2153 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2154 {
2155         struct cpufreq_policy *cpu_policy;
2156         if (!policy)
2157                 return -EINVAL;
2158
2159         cpu_policy = cpufreq_cpu_get(cpu);
2160         if (!cpu_policy)
2161                 return -EINVAL;
2162
2163         memcpy(policy, cpu_policy, sizeof(*policy));
2164
2165         cpufreq_cpu_put(cpu_policy);
2166         return 0;
2167 }
2168 EXPORT_SYMBOL(cpufreq_get_policy);
2169
2170 /*
2171  * policy : current policy.
2172  * new_policy: policy to be set.
2173  */
2174 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2175                                 struct cpufreq_policy *new_policy)
2176 {
2177         struct cpufreq_governor *old_gov;
2178         int ret;
2179
2180         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2181                  new_policy->cpu, new_policy->min, new_policy->max);
2182
2183         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2184
2185         if (new_policy->min > policy->max || new_policy->max < policy->min)
2186                 return -EINVAL;
2187
2188         /* verify the cpu speed can be set within this limit */
2189         ret = cpufreq_driver->verify(new_policy);
2190         if (ret)
2191                 return ret;
2192
2193         /* adjust if necessary - all reasons */
2194         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2195                         CPUFREQ_ADJUST, new_policy);
2196
2197         /* adjust if necessary - hardware incompatibility*/
2198         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2199                         CPUFREQ_INCOMPATIBLE, new_policy);
2200
2201         /*
2202          * verify the cpu speed can be set within this limit, which might be
2203          * different to the first one
2204          */
2205         ret = cpufreq_driver->verify(new_policy);
2206         if (ret)
2207                 return ret;
2208
2209         /* notification of the new policy */
2210         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2211                         CPUFREQ_NOTIFY, new_policy);
2212
2213         policy->min = new_policy->min;
2214         policy->max = new_policy->max;
2215
2216         pr_debug("new min and max freqs are %u - %u kHz\n",
2217                  policy->min, policy->max);
2218
2219         if (cpufreq_driver->setpolicy) {
2220                 policy->policy = new_policy->policy;
2221                 pr_debug("setting range\n");
2222                 return cpufreq_driver->setpolicy(new_policy);
2223         }
2224
2225         if (new_policy->governor == policy->governor)
2226                 goto out;
2227
2228         pr_debug("governor switch\n");
2229
2230         /* save old, working values */
2231         old_gov = policy->governor;
2232         /* end old governor */
2233         if (old_gov) {
2234                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2235                 up_write(&policy->rwsem);
2236                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2237                 down_write(&policy->rwsem);
2238         }
2239
2240         /* start new governor */
2241         policy->governor = new_policy->governor;
2242         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2243                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2244                         goto out;
2245
2246                 up_write(&policy->rwsem);
2247                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2248                 down_write(&policy->rwsem);
2249         }
2250
2251         /* new governor failed, so re-start old one */
2252         pr_debug("starting governor %s failed\n", policy->governor->name);
2253         if (old_gov) {
2254                 policy->governor = old_gov;
2255                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2256                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2257         }
2258
2259         return -EINVAL;
2260
2261  out:
2262         pr_debug("governor: change or update limits\n");
2263         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2264 }
2265
2266 /**
2267  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2268  *      @cpu: CPU which shall be re-evaluated
2269  *
2270  *      Useful for policy notifiers which have different necessities
2271  *      at different times.
2272  */
2273 int cpufreq_update_policy(unsigned int cpu)
2274 {
2275         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2276         struct cpufreq_policy new_policy;
2277         int ret;
2278
2279         if (!policy)
2280                 return -ENODEV;
2281
2282         down_write(&policy->rwsem);
2283
2284         pr_debug("updating policy for CPU %u\n", cpu);
2285         memcpy(&new_policy, policy, sizeof(*policy));
2286         new_policy.min = policy->user_policy.min;
2287         new_policy.max = policy->user_policy.max;
2288         new_policy.policy = policy->user_policy.policy;
2289         new_policy.governor = policy->user_policy.governor;
2290
2291         /*
2292          * BIOS might change freq behind our back
2293          * -> ask driver for current freq and notify governors about a change
2294          */
2295         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2296                 new_policy.cur = cpufreq_driver->get(cpu);
2297                 if (WARN_ON(!new_policy.cur)) {
2298                         ret = -EIO;
2299                         goto unlock;
2300                 }
2301
2302                 if (!policy->cur) {
2303                         pr_debug("Driver did not initialize current freq\n");
2304                         policy->cur = new_policy.cur;
2305                 } else {
2306                         if (policy->cur != new_policy.cur && has_target())
2307                                 cpufreq_out_of_sync(cpu, policy->cur,
2308                                                                 new_policy.cur);
2309                 }
2310         }
2311
2312         ret = cpufreq_set_policy(policy, &new_policy);
2313
2314 unlock:
2315         up_write(&policy->rwsem);
2316
2317         cpufreq_cpu_put(policy);
2318         return ret;
2319 }
2320 EXPORT_SYMBOL(cpufreq_update_policy);
2321
2322 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2323                                         unsigned long action, void *hcpu)
2324 {
2325         unsigned int cpu = (unsigned long)hcpu;
2326         struct device *dev;
2327
2328         dev = get_cpu_device(cpu);
2329         if (dev) {
2330                 switch (action & ~CPU_TASKS_FROZEN) {
2331                 case CPU_ONLINE:
2332                         __cpufreq_add_dev(dev, NULL);
2333                         break;
2334
2335                 case CPU_DOWN_PREPARE:
2336                         __cpufreq_remove_dev_prepare(dev, NULL);
2337                         break;
2338
2339                 case CPU_POST_DEAD:
2340                         __cpufreq_remove_dev_finish(dev, NULL);
2341                         break;
2342
2343                 case CPU_DOWN_FAILED:
2344                         __cpufreq_add_dev(dev, NULL);
2345                         break;
2346                 }
2347         }
2348         return NOTIFY_OK;
2349 }
2350
2351 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2352         .notifier_call = cpufreq_cpu_callback,
2353 };
2354
2355 /*********************************************************************
2356  *               BOOST                                               *
2357  *********************************************************************/
2358 static int cpufreq_boost_set_sw(int state)
2359 {
2360         struct cpufreq_frequency_table *freq_table;
2361         struct cpufreq_policy *policy;
2362         int ret = -EINVAL;
2363
2364         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2365                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2366                 if (freq_table) {
2367                         ret = cpufreq_frequency_table_cpuinfo(policy,
2368                                                         freq_table);
2369                         if (ret) {
2370                                 pr_err("%s: Policy frequency update failed\n",
2371                                        __func__);
2372                                 break;
2373                         }
2374                         policy->user_policy.max = policy->max;
2375                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2376                 }
2377         }
2378
2379         return ret;
2380 }
2381
2382 int cpufreq_boost_trigger_state(int state)
2383 {
2384         unsigned long flags;
2385         int ret = 0;
2386
2387         if (cpufreq_driver->boost_enabled == state)
2388                 return 0;
2389
2390         write_lock_irqsave(&cpufreq_driver_lock, flags);
2391         cpufreq_driver->boost_enabled = state;
2392         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2393
2394         ret = cpufreq_driver->set_boost(state);
2395         if (ret) {
2396                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2397                 cpufreq_driver->boost_enabled = !state;
2398                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2399
2400                 pr_err("%s: Cannot %s BOOST\n",
2401                        __func__, state ? "enable" : "disable");
2402         }
2403
2404         return ret;
2405 }
2406
2407 int cpufreq_boost_supported(void)
2408 {
2409         if (likely(cpufreq_driver))
2410                 return cpufreq_driver->boost_supported;
2411
2412         return 0;
2413 }
2414 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2415
2416 int cpufreq_boost_enabled(void)
2417 {
2418         return cpufreq_driver->boost_enabled;
2419 }
2420 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2421
2422 /*********************************************************************
2423  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2424  *********************************************************************/
2425
2426 /**
2427  * cpufreq_register_driver - register a CPU Frequency driver
2428  * @driver_data: A struct cpufreq_driver containing the values#
2429  * submitted by the CPU Frequency driver.
2430  *
2431  * Registers a CPU Frequency driver to this core code. This code
2432  * returns zero on success, -EBUSY when another driver got here first
2433  * (and isn't unregistered in the meantime).
2434  *
2435  */
2436 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2437 {
2438         unsigned long flags;
2439         int ret;
2440
2441         if (cpufreq_disabled())
2442                 return -ENODEV;
2443
2444         if (!driver_data || !driver_data->verify || !driver_data->init ||
2445             !(driver_data->setpolicy || driver_data->target_index ||
2446                     driver_data->target) ||
2447              (driver_data->setpolicy && (driver_data->target_index ||
2448                     driver_data->target)) ||
2449              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2450                 return -EINVAL;
2451
2452         pr_debug("trying to register driver %s\n", driver_data->name);
2453
2454         if (driver_data->setpolicy)
2455                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2456
2457         write_lock_irqsave(&cpufreq_driver_lock, flags);
2458         if (cpufreq_driver) {
2459                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2460                 return -EEXIST;
2461         }
2462         cpufreq_driver = driver_data;
2463         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2464
2465         if (cpufreq_boost_supported()) {
2466                 /*
2467                  * Check if driver provides function to enable boost -
2468                  * if not, use cpufreq_boost_set_sw as default
2469                  */
2470                 if (!cpufreq_driver->set_boost)
2471                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2472
2473                 ret = cpufreq_sysfs_create_file(&boost.attr);
2474                 if (ret) {
2475                         pr_err("%s: cannot register global BOOST sysfs file\n",
2476                                __func__);
2477                         goto err_null_driver;
2478                 }
2479         }
2480
2481         ret = subsys_interface_register(&cpufreq_interface);
2482         if (ret)
2483                 goto err_boost_unreg;
2484
2485         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2486                 int i;
2487                 ret = -ENODEV;
2488
2489                 /* check for at least one working CPU */
2490                 for (i = 0; i < nr_cpu_ids; i++)
2491                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2492                                 ret = 0;
2493                                 break;
2494                         }
2495
2496                 /* if all ->init() calls failed, unregister */
2497                 if (ret) {
2498                         pr_debug("no CPU initialized for driver %s\n",
2499                                  driver_data->name);
2500                         goto err_if_unreg;
2501                 }
2502         }
2503
2504         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2505         pr_debug("driver %s up and running\n", driver_data->name);
2506
2507         return 0;
2508 err_if_unreg:
2509         subsys_interface_unregister(&cpufreq_interface);
2510 err_boost_unreg:
2511         if (cpufreq_boost_supported())
2512                 cpufreq_sysfs_remove_file(&boost.attr);
2513 err_null_driver:
2514         write_lock_irqsave(&cpufreq_driver_lock, flags);
2515         cpufreq_driver = NULL;
2516         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2517         return ret;
2518 }
2519 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2520
2521 /**
2522  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2523  *
2524  * Unregister the current CPUFreq driver. Only call this if you have
2525  * the right to do so, i.e. if you have succeeded in initialising before!
2526  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2527  * currently not initialised.
2528  */
2529 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2530 {
2531         unsigned long flags;
2532
2533         if (!cpufreq_driver || (driver != cpufreq_driver))
2534                 return -EINVAL;
2535
2536         pr_debug("unregistering driver %s\n", driver->name);
2537
2538         subsys_interface_unregister(&cpufreq_interface);
2539         if (cpufreq_boost_supported())
2540                 cpufreq_sysfs_remove_file(&boost.attr);
2541
2542         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2543
2544         down_write(&cpufreq_rwsem);
2545         write_lock_irqsave(&cpufreq_driver_lock, flags);
2546
2547         cpufreq_driver = NULL;
2548
2549         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2550         up_write(&cpufreq_rwsem);
2551
2552         return 0;
2553 }
2554 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2555
2556 /*
2557  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2558  * or mutexes when secondary CPUs are halted.
2559  */
2560 static struct syscore_ops cpufreq_syscore_ops = {
2561         .shutdown = cpufreq_suspend,
2562 };
2563
2564 static int __init cpufreq_core_init(void)
2565 {
2566         if (cpufreq_disabled())
2567                 return -ENODEV;
2568
2569         cpufreq_global_kobject = kobject_create();
2570         BUG_ON(!cpufreq_global_kobject);
2571
2572         register_syscore_ops(&cpufreq_syscore_ops);
2573
2574         return 0;
2575 }
2576 core_initcall(cpufreq_core_init);