drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 /**
  35  * The "cpufreq driver" - the arch- or hardware-dependent low
  36  * level driver of CPUFreq support, and its spinlock. This lock
  37  * also protects the cpufreq_cpu_data array.
  38  */
  39 static struct cpufreq_driver *cpufreq_driver;
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  42 static DEFINE_RWLOCK(cpufreq_driver_lock);
  43 DEFINE_MUTEX(cpufreq_governor_lock);
  44 static LIST_HEAD(cpufreq_policy_list);
  45
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48
  49 /* Flag to suspend/resume CPUFreq governors */
  50 static bool cpufreq_suspended;
  51
  52 static inline bool has_target(void)
  53 {
  54         return cpufreq_driver->target_index || cpufreq_driver->target;
  55 }
  56
  57 /*
  58  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  59  * sections
  60  */
  61 static DECLARE_RWSEM(cpufreq_rwsem);
  62
  63 /* internal prototypes */
  64 static int __cpufreq_governor(struct cpufreq_policy *policy,
  65                 unsigned int event);
  66 static unsigned int __cpufreq_get(unsigned int cpu);
  67 static void handle_update(struct work_struct *work);
  68
  69 /**
  70  * Two notifier lists: the "policy" list is involved in the
  71  * validation process for a new CPU frequency policy; the
  72  * "transition" list for kernel code that needs to handle
  73  * changes to devices when the CPU clock speed changes.
  74  * The mutex locks both lists.
  75  */
  76 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  77 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  78
  79 static bool init_cpufreq_transition_notifier_list_called;
  80 static int __init init_cpufreq_transition_notifier_list(void)
  81 {
  82         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  83         init_cpufreq_transition_notifier_list_called = true;
  84         return 0;
  85 }
  86 pure_initcall(init_cpufreq_transition_notifier_list);
  87
  88 static int off __read_mostly;
  89 static int cpufreq_disabled(void)
  90 {
  91         return off;
  92 }
  93 void disable_cpufreq(void)
  94 {
  95         off = 1;
  96 }
  97 static LIST_HEAD(cpufreq_governor_list);
  98 static DEFINE_MUTEX(cpufreq_governor_mutex);
  99
 100 bool have_governor_per_policy(void)
 101 {
 102         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 103 }
 104 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 105
 106 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 107 {
 108         if (have_governor_per_policy())
 109                 return &policy->kobj;
 110         else
 111                 return cpufreq_global_kobject;
 112 }
 113 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 114
 115 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 116 {
 117         u64 idle_time;
 118         u64 cur_wall_time;
 119         u64 busy_time;
 120
 121         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 122
 123         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 128         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 129
 130         idle_time = cur_wall_time - busy_time;
 131         if (wall)
 132                 *wall = cputime_to_usecs(cur_wall_time);
 133
 134         return cputime_to_usecs(idle_time);
 135 }
 136
 137 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 138 {
 139         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 140
 141         if (idle_time == -1ULL)
 142                 return get_cpu_idle_time_jiffy(cpu, wall);
 143         else if (!io_busy)
 144                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 145
 146         return idle_time;
 147 }
 148 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 149
 150 /*
 151  * This is a generic cpufreq init() routine which can be used by cpufreq
 152  * drivers of SMP systems. It will do following:
 153  * - validate & show freq table passed
 154  * - set policies transition latency
 155  * - policy->cpus with all possible CPUs
 156  */
 157 int cpufreq_generic_init(struct cpufreq_policy *policy,
 158                 struct cpufreq_frequency_table *table,
 159                 unsigned int transition_latency)
 160 {
 161         int ret;
 162
 163         ret = cpufreq_table_validate_and_show(policy, table);
 164         if (ret) {
 165                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 166                 return ret;
 167         }
 168
 169         policy->cpuinfo.transition_latency = transition_latency;
 170
 171         /*
 172          * The driver only supports the SMP configuartion where all processors
 173          * share the clock and voltage and clock.
 174          */
 175         cpumask_setall(policy->cpus);
 176
 177         return 0;
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 180
 181 unsigned int cpufreq_generic_get(unsigned int cpu)
 182 {
 183         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 184
 185         if (!policy || IS_ERR(policy->clk)) {
 186                 pr_err("%s: No %s associated to cpu: %d\n",
 187                        __func__, policy ? "clk" : "policy", cpu);
 188                 return 0;
 189         }
 190
 191         return clk_get_rate(policy->clk) / 1000;
 192 }
 193 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 194
 195 /* Only for cpufreq core internal use */
 196 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 197 {
 198         return per_cpu(cpufreq_cpu_data, cpu);
 199 }
 200
 201 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 202 {
 203         struct cpufreq_policy *policy = NULL;
 204         unsigned long flags;
 205
 206         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 207                 return NULL;
 208
 209         if (!down_read_trylock(&cpufreq_rwsem))
 210                 return NULL;
 211
 212         /* get the cpufreq driver */
 213         read_lock_irqsave(&cpufreq_driver_lock, flags);
 214
 215         if (cpufreq_driver) {
 216                 /* get the CPU */
 217                 policy = per_cpu(cpufreq_cpu_data, cpu);
 218                 if (policy)
 219                         kobject_get(&policy->kobj);
 220         }
 221
 222         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 223
 224         if (!policy)
 225                 up_read(&cpufreq_rwsem);
 226
 227         return policy;
 228 }
 229 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 230
 231 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 232 {
 233         if (cpufreq_disabled())
 234                 return;
 235
 236         kobject_put(&policy->kobj);
 237         up_read(&cpufreq_rwsem);
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 240
 241 /*********************************************************************
 242  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 243  *********************************************************************/
 244
 245 /**
 246  * adjust_jiffies - adjust the system "loops_per_jiffy"
 247  *
 248  * This function alters the system "loops_per_jiffy" for the clock
 249  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 250  * systems as each CPU might be scaled differently. So, use the arch
 251  * per-CPU loops_per_jiffy value wherever possible.
 252  */
 253 #ifndef CONFIG_SMP
 254 static unsigned long l_p_j_ref;
 255 static unsigned int l_p_j_ref_freq;
 256
 257 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 258 {
 259         if (ci->flags & CPUFREQ_CONST_LOOPS)
 260                 return;
 261
 262         if (!l_p_j_ref_freq) {
 263                 l_p_j_ref = loops_per_jiffy;
 264                 l_p_j_ref_freq = ci->old;
 265                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 266                          l_p_j_ref, l_p_j_ref_freq);
 267         }
 268         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 269                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 270                                                                 ci->new);
 271                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 272                          loops_per_jiffy, ci->new);
 273         }
 274 }
 275 #else
 276 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 277 {
 278         return;
 279 }
 280 #endif
 281
 282 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 283                 struct cpufreq_freqs *freqs, unsigned int state)
 284 {
 285         BUG_ON(irqs_disabled());
 286
 287         if (cpufreq_disabled())
 288                 return;
 289
 290         freqs->flags = cpufreq_driver->flags;
 291         pr_debug("notification %u of frequency transition to %u kHz\n",
 292                  state, freqs->new);
 293
 294         switch (state) {
 295
 296         case CPUFREQ_PRECHANGE:
 297                 /* detect if the driver reported a value as "old frequency"
 298                  * which is not equal to what the cpufreq core thinks is
 299                  * "old frequency".
 300                  */
 301                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 302                         if ((policy) && (policy->cpu == freqs->cpu) &&
 303                             (policy->cur) && (policy->cur != freqs->old)) {
 304                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 305                                          freqs->old, policy->cur);
 306                                 freqs->old = policy->cur;
 307                         }
 308                 }
 309                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 310                                 CPUFREQ_PRECHANGE, freqs);
 311                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 312                 break;
 313
 314         case CPUFREQ_POSTCHANGE:
 315                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 316                 pr_debug("FREQ: %lu - CPU: %lu\n",
 317                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 318                 trace_cpu_frequency(freqs->new, freqs->cpu);
 319                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 320                                 CPUFREQ_POSTCHANGE, freqs);
 321                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 322                         policy->cur = freqs->new;
 323                 break;
 324         }
 325 }
 326
 327 /**
 328  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 329  * on frequency transition.
 330  *
 331  * This function calls the transition notifiers and the "adjust_jiffies"
 332  * function. It is called twice on all CPU frequency changes that have
 333  * external effects.
 334  */
 335 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 336                 struct cpufreq_freqs *freqs, unsigned int state)
 337 {
 338         for_each_cpu(freqs->cpu, policy->cpus)
 339                 __cpufreq_notify_transition(policy, freqs, state);
 340 }
 341
 342 /* Do post notifications when there are chances that transition has failed */
 343 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 344                 struct cpufreq_freqs *freqs, int transition_failed)
 345 {
 346         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 347         if (!transition_failed)
 348                 return;
 349
 350         swap(freqs->old, freqs->new);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 352         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 353 }
 354
 355 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 356                 struct cpufreq_freqs *freqs)
 357 {
 358
 359         /*
 360          * Catch double invocations of _begin() which lead to self-deadlock.
 361          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 362          * doesn't invoke _begin() on their behalf, and hence the chances of
 363          * double invocations are very low. Moreover, there are scenarios
 364          * where these checks can emit false-positive warnings in these
 365          * drivers; so we avoid that by skipping them altogether.
 366          */
 367         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 368                                 && current == policy->transition_task);
 369
 370 wait:
 371         wait_event(policy->transition_wait, !policy->transition_ongoing);
 372
 373         spin_lock(&policy->transition_lock);
 374
 375         if (unlikely(policy->transition_ongoing)) {
 376                 spin_unlock(&policy->transition_lock);
 377                 goto wait;
 378         }
 379
 380         policy->transition_ongoing = true;
 381         policy->transition_task = current;
 382
 383         spin_unlock(&policy->transition_lock);
 384
 385         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 386 }
 387 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 388
 389 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 390                 struct cpufreq_freqs *freqs, int transition_failed)
 391 {
 392         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 393                 return;
 394
 395         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 396
 397         policy->transition_ongoing = false;
 398         policy->transition_task = NULL;
 399
 400         wake_up(&policy->transition_wait);
 401 }
 402 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 403
 404
 405 /*********************************************************************
 406  *                          SYSFS INTERFACE                          *
 407  *********************************************************************/
 408 static ssize_t show_boost(struct kobject *kobj,
 409                                  struct attribute *attr, char *buf)
 410 {
 411         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 412 }
 413
 414 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 415                                   const char *buf, size_t count)
 416 {
 417         int ret, enable;
 418
 419         ret = sscanf(buf, "%d", &enable);
 420         if (ret != 1 || enable < 0 || enable > 1)
 421                 return -EINVAL;
 422
 423         if (cpufreq_boost_trigger_state(enable)) {
 424                 pr_err("%s: Cannot %s BOOST!\n",
 425                        __func__, enable ? "enable" : "disable");
 426                 return -EINVAL;
 427         }
 428
 429         pr_debug("%s: cpufreq BOOST %s\n",
 430                  __func__, enable ? "enabled" : "disabled");
 431
 432         return count;
 433 }
 434 define_one_global_rw(boost);
 435
 436 static struct cpufreq_governor *find_governor(const char *str_governor)
 437 {
 438         struct cpufreq_governor *t;
 439
 440         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 441                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 442                         return t;
 443
 444         return NULL;
 445 }
 446
 447 /**
 448  * cpufreq_parse_governor - parse a governor string
 449  */
 450 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 451                                 struct cpufreq_governor **governor)
 452 {
 453         int err = -EINVAL;
 454
 455         if (!cpufreq_driver)
 456                 goto out;
 457
 458         if (cpufreq_driver->setpolicy) {
 459                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 460                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 461                         err = 0;
 462                 } else if (!strncasecmp(str_governor, "powersave",
 463                                                 CPUFREQ_NAME_LEN)) {
 464                         *policy = CPUFREQ_POLICY_POWERSAVE;
 465                         err = 0;
 466                 }
 467         } else if (has_target()) {
 468                 struct cpufreq_governor *t;
 469
 470                 mutex_lock(&cpufreq_governor_mutex);
 471
 472                 t = find_governor(str_governor);
 473
 474                 if (t == NULL) {
 475                         int ret;
 476
 477                         mutex_unlock(&cpufreq_governor_mutex);
 478                         ret = request_module("cpufreq_%s", str_governor);
 479                         mutex_lock(&cpufreq_governor_mutex);
 480
 481                         if (ret == 0)
 482                                 t = find_governor(str_governor);
 483                 }
 484
 485                 if (t != NULL) {
 486                         *governor = t;
 487                         err = 0;
 488                 }
 489
 490                 mutex_unlock(&cpufreq_governor_mutex);
 491         }
 492 out:
 493         return err;
 494 }
 495
 496 /**
 497  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 498  * print out cpufreq information
 499  *
 500  * Write out information from cpufreq_driver->policy[cpu]; object must be
 501  * "unsigned int".
 502  */
 503
 504 #define show_one(file_name, object)                     \
 505 static ssize_t show_##file_name                         \
 506 (struct cpufreq_policy *policy, char *buf)              \
 507 {                                                       \
 508         return sprintf(buf, "%u\n", policy->object);    \
 509 }
 510
 511 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 512 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 513 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 514 show_one(scaling_min_freq, min);
 515 show_one(scaling_max_freq, max);
 516
 517 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 518 {
 519         ssize_t ret;
 520
 521         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 522                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 523         else
 524                 ret = sprintf(buf, "%u\n", policy->cur);
 525         return ret;
 526 }
 527
 528 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 529                                 struct cpufreq_policy *new_policy);
 530
 531 /**
 532  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 533  */
 534 #define store_one(file_name, object)                    \
 535 static ssize_t store_##file_name                                        \
 536 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 537 {                                                                       \
 538         int ret, temp;                                                  \
 539         struct cpufreq_policy new_policy;                               \
 540                                                                         \
 541         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 542         if (ret)                                                        \
 543                 return -EINVAL;                                         \
 544                                                                         \
 545         ret = sscanf(buf, "%u", &new_policy.object);                    \
 546         if (ret != 1)                                                   \
 547                 return -EINVAL;                                         \
 548                                                                         \
 549         temp = new_policy.object;                                       \
 550         ret = cpufreq_set_policy(policy, &new_policy);          \
 551         if (!ret)                                                       \
 552                 policy->user_policy.object = temp;                      \
 553                                                                         \
 554         return ret ? ret : count;                                       \
 555 }
 556
 557 store_one(scaling_min_freq, min);
 558 store_one(scaling_max_freq, max);
 559
 560 /**
 561  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 562  */
 563 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 564                                         char *buf)
 565 {
 566         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 567         if (!cur_freq)
 568                 return sprintf(buf, "<unknown>");
 569         return sprintf(buf, "%u\n", cur_freq);
 570 }
 571
 572 /**
 573  * show_scaling_governor - show the current policy for the specified CPU
 574  */
 575 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 576 {
 577         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 578                 return sprintf(buf, "powersave\n");
 579         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 580                 return sprintf(buf, "performance\n");
 581         else if (policy->governor)
 582                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 583                                 policy->governor->name);
 584         return -EINVAL;
 585 }
 586
 587 /**
 588  * store_scaling_governor - store policy for the specified CPU
 589  */
 590 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 591                                         const char *buf, size_t count)
 592 {
 593         int ret;
 594         char    str_governor[16];
 595         struct cpufreq_policy new_policy;
 596
 597         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 598         if (ret)
 599                 return ret;
 600
 601         ret = sscanf(buf, "%15s", str_governor);
 602         if (ret != 1)
 603                 return -EINVAL;
 604
 605         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 606                                                 &new_policy.governor))
 607                 return -EINVAL;
 608
 609         ret = cpufreq_set_policy(policy, &new_policy);
 610
 611         policy->user_policy.policy = policy->policy;
 612         policy->user_policy.governor = policy->governor;
 613
 614         if (ret)
 615                 return ret;
 616         else
 617                 return count;
 618 }
 619
 620 /**
 621  * show_scaling_driver - show the cpufreq driver currently loaded
 622  */
 623 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 624 {
 625         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 626 }
 627
 628 /**
 629  * show_scaling_available_governors - show the available CPUfreq governors
 630  */
 631 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 632                                                 char *buf)
 633 {
 634         ssize_t i = 0;
 635         struct cpufreq_governor *t;
 636
 637         if (!has_target()) {
 638                 i += sprintf(buf, "performance powersave");
 639                 goto out;
 640         }
 641
 642         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 643                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 644                     - (CPUFREQ_NAME_LEN + 2)))
 645                         goto out;
 646                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 647         }
 648 out:
 649         i += sprintf(&buf[i], "\n");
 650         return i;
 651 }
 652
 653 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 654 {
 655         ssize_t i = 0;
 656         unsigned int cpu;
 657
 658         for_each_cpu(cpu, mask) {
 659                 if (i)
 660                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 661                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 662                 if (i >= (PAGE_SIZE - 5))
 663                         break;
 664         }
 665         i += sprintf(&buf[i], "\n");
 666         return i;
 667 }
 668 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 669
 670 /**
 671  * show_related_cpus - show the CPUs affected by each transition even if
 672  * hw coordination is in use
 673  */
 674 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 675 {
 676         return cpufreq_show_cpus(policy->related_cpus, buf);
 677 }
 678
 679 /**
 680  * show_affected_cpus - show the CPUs affected by each transition
 681  */
 682 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 683 {
 684         return cpufreq_show_cpus(policy->cpus, buf);
 685 }
 686
 687 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 688                                         const char *buf, size_t count)
 689 {
 690         unsigned int freq = 0;
 691         unsigned int ret;
 692
 693         if (!policy->governor || !policy->governor->store_setspeed)
 694                 return -EINVAL;
 695
 696         ret = sscanf(buf, "%u", &freq);
 697         if (ret != 1)
 698                 return -EINVAL;
 699
 700         policy->governor->store_setspeed(policy, freq);
 701
 702         return count;
 703 }
 704
 705 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 706 {
 707         if (!policy->governor || !policy->governor->show_setspeed)
 708                 return sprintf(buf, "<unsupported>\n");
 709
 710         return policy->governor->show_setspeed(policy, buf);
 711 }
 712
 713 /**
 714  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 715  */
 716 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 717 {
 718         unsigned int limit;
 719         int ret;
 720         if (cpufreq_driver->bios_limit) {
 721                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 722                 if (!ret)
 723                         return sprintf(buf, "%u\n", limit);
 724         }
 725         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 726 }
 727
 728 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 729 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 730 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 731 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 732 cpufreq_freq_attr_ro(scaling_available_governors);
 733 cpufreq_freq_attr_ro(scaling_driver);
 734 cpufreq_freq_attr_ro(scaling_cur_freq);
 735 cpufreq_freq_attr_ro(bios_limit);
 736 cpufreq_freq_attr_ro(related_cpus);
 737 cpufreq_freq_attr_ro(affected_cpus);
 738 cpufreq_freq_attr_rw(scaling_min_freq);
 739 cpufreq_freq_attr_rw(scaling_max_freq);
 740 cpufreq_freq_attr_rw(scaling_governor);
 741 cpufreq_freq_attr_rw(scaling_setspeed);
 742
 743 static struct attribute *default_attrs[] = {
 744         &cpuinfo_min_freq.attr,
 745         &cpuinfo_max_freq.attr,
 746         &cpuinfo_transition_latency.attr,
 747         &scaling_min_freq.attr,
 748         &scaling_max_freq.attr,
 749         &affected_cpus.attr,
 750         &related_cpus.attr,
 751         &scaling_governor.attr,
 752         &scaling_driver.attr,
 753         &scaling_available_governors.attr,
 754         &scaling_setspeed.attr,
 755         NULL
 756 };
 757
 758 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 759 #define to_attr(a) container_of(a, struct freq_attr, attr)
 760
 761 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 762 {
 763         struct cpufreq_policy *policy = to_policy(kobj);
 764         struct freq_attr *fattr = to_attr(attr);
 765         ssize_t ret;
 766
 767         if (!down_read_trylock(&cpufreq_rwsem))
 768                 return -EINVAL;
 769
 770         down_read(&policy->rwsem);
 771
 772         if (fattr->show)
 773                 ret = fattr->show(policy, buf);
 774         else
 775                 ret = -EIO;
 776
 777         up_read(&policy->rwsem);
 778         up_read(&cpufreq_rwsem);
 779
 780         return ret;
 781 }
 782
 783 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 784                      const char *buf, size_t count)
 785 {
 786         struct cpufreq_policy *policy = to_policy(kobj);
 787         struct freq_attr *fattr = to_attr(attr);
 788         ssize_t ret = -EINVAL;
 789
 790         get_online_cpus();
 791
 792         if (!cpu_online(policy->cpu))
 793                 goto unlock;
 794
 795         if (!down_read_trylock(&cpufreq_rwsem))
 796                 goto unlock;
 797
 798         down_write(&policy->rwsem);
 799
 800         if (fattr->store)
 801                 ret = fattr->store(policy, buf, count);
 802         else
 803                 ret = -EIO;
 804
 805         up_write(&policy->rwsem);
 806
 807         up_read(&cpufreq_rwsem);
 808 unlock:
 809         put_online_cpus();
 810
 811         return ret;
 812 }
 813
 814 static void cpufreq_sysfs_release(struct kobject *kobj)
 815 {
 816         struct cpufreq_policy *policy = to_policy(kobj);
 817         pr_debug("last reference is dropped\n");
 818         complete(&policy->kobj_unregister);
 819 }
 820
 821 static const struct sysfs_ops sysfs_ops = {
 822         .show   = show,
 823         .store  = store,
 824 };
 825
 826 static struct kobj_type ktype_cpufreq = {
 827         .sysfs_ops      = &sysfs_ops,
 828         .default_attrs  = default_attrs,
 829         .release        = cpufreq_sysfs_release,
 830 };
 831
 832 struct kobject *cpufreq_global_kobject;
 833 EXPORT_SYMBOL(cpufreq_global_kobject);
 834
 835 static int cpufreq_global_kobject_usage;
 836
 837 int cpufreq_get_global_kobject(void)
 838 {
 839         if (!cpufreq_global_kobject_usage++)
 840                 return kobject_add(cpufreq_global_kobject,
 841                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 842
 843         return 0;
 844 }
 845 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 846
 847 void cpufreq_put_global_kobject(void)
 848 {
 849         if (!--cpufreq_global_kobject_usage)
 850                 kobject_del(cpufreq_global_kobject);
 851 }
 852 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 853
 854 int cpufreq_sysfs_create_file(const struct attribute *attr)
 855 {
 856         int ret = cpufreq_get_global_kobject();
 857
 858         if (!ret) {
 859                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 860                 if (ret)
 861                         cpufreq_put_global_kobject();
 862         }
 863
 864         return ret;
 865 }
 866 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 867
 868 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 869 {
 870         sysfs_remove_file(cpufreq_global_kobject, attr);
 871         cpufreq_put_global_kobject();
 872 }
 873 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 874
 875 /* symlink affected CPUs */
 876 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 877 {
 878         unsigned int j;
 879         int ret = 0;
 880
 881         for_each_cpu(j, policy->cpus) {
 882                 struct device *cpu_dev;
 883
 884                 if (j == policy->cpu)
 885                         continue;
 886
 887                 pr_debug("Adding link for CPU: %u\n", j);
 888                 cpu_dev = get_cpu_device(j);
 889                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 890                                         "cpufreq");
 891                 if (ret)
 892                         break;
 893         }
 894         return ret;
 895 }
 896
 897 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 898                                      struct device *dev)
 899 {
 900         struct freq_attr **drv_attr;
 901         int ret = 0;
 902
 903         /* set up files for this cpu device */
 904         drv_attr = cpufreq_driver->attr;
 905         while (drv_attr && *drv_attr) {
 906                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 907                 if (ret)
 908                         return ret;
 909                 drv_attr++;
 910         }
 911         if (cpufreq_driver->get) {
 912                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 913                 if (ret)
 914                         return ret;
 915         }
 916
 917         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 918         if (ret)
 919                 return ret;
 920
 921         if (cpufreq_driver->bios_limit) {
 922                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 923                 if (ret)
 924                         return ret;
 925         }
 926
 927         return cpufreq_add_dev_symlink(policy);
 928 }
 929
 930 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 931 {
 932         struct cpufreq_governor *gov = NULL;
 933         struct cpufreq_policy new_policy;
 934         int ret = 0;
 935
 936         memcpy(&new_policy, policy, sizeof(*policy));
 937
 938         /* Update governor of new_policy to the governor used before hotplug */
 939         gov = find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 940         if (gov)
 941                 pr_debug("Restoring governor %s for cpu %d\n",
 942                                 policy->governor->name, policy->cpu);
 943         else
 944                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 945
 946         new_policy.governor = gov;
 947
 948         /* Use the default policy if its valid. */
 949         if (cpufreq_driver->setpolicy)
 950                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 951
 952         /* set default policy */
 953         ret = cpufreq_set_policy(policy, &new_policy);
 954         if (ret) {
 955                 pr_debug("setting policy failed\n");
 956                 if (cpufreq_driver->exit)
 957                         cpufreq_driver->exit(policy);
 958         }
 959 }
 960
 961 #ifdef CONFIG_HOTPLUG_CPU
 962 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 963                                   unsigned int cpu, struct device *dev)
 964 {
 965         int ret = 0;
 966         unsigned long flags;
 967
 968         if (has_target()) {
 969                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 970                 if (ret) {
 971                         pr_err("%s: Failed to stop governor\n", __func__);
 972                         return ret;
 973                 }
 974         }
 975
 976         down_write(&policy->rwsem);
 977
 978         write_lock_irqsave(&cpufreq_driver_lock, flags);
 979
 980         cpumask_set_cpu(cpu, policy->cpus);
 981         per_cpu(cpufreq_cpu_data, cpu) = policy;
 982         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 983
 984         up_write(&policy->rwsem);
 985
 986         if (has_target()) {
 987                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 988                 if (!ret)
 989                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 990
 991                 if (ret) {
 992                         pr_err("%s: Failed to start governor\n", __func__);
 993                         return ret;
 994                 }
 995         }
 996
 997         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 998 }
 999 #endif
1000
1001 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1002 {
1003         struct cpufreq_policy *policy;
1004         unsigned long flags;
1005
1006         read_lock_irqsave(&cpufreq_driver_lock, flags);
1007
1008         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1009
1010         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1011
1012         if (policy)
1013                 policy->governor = NULL;
1014
1015         return policy;
1016 }
1017
1018 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1019 {
1020         struct cpufreq_policy *policy;
1021
1022         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1023         if (!policy)
1024                 return NULL;
1025
1026         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1027                 goto err_free_policy;
1028
1029         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1030                 goto err_free_cpumask;
1031
1032         INIT_LIST_HEAD(&policy->policy_list);
1033         init_rwsem(&policy->rwsem);
1034         spin_lock_init(&policy->transition_lock);
1035         init_waitqueue_head(&policy->transition_wait);
1036
1037         return policy;
1038
1039 err_free_cpumask:
1040         free_cpumask_var(policy->cpus);
1041 err_free_policy:
1042         kfree(policy);
1043
1044         return NULL;
1045 }
1046
1047 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1048 {
1049         struct kobject *kobj;
1050         struct completion *cmp;
1051
1052         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1053                         CPUFREQ_REMOVE_POLICY, policy);
1054
1055         down_read(&policy->rwsem);
1056         kobj = &policy->kobj;
1057         cmp = &policy->kobj_unregister;
1058         up_read(&policy->rwsem);
1059         kobject_put(kobj);
1060
1061         /*
1062          * We need to make sure that the underlying kobj is
1063          * actually not referenced anymore by anybody before we
1064          * proceed with unloading.
1065          */
1066         pr_debug("waiting for dropping of refcount\n");
1067         wait_for_completion(cmp);
1068         pr_debug("wait complete\n");
1069 }
1070
1071 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1072 {
1073         free_cpumask_var(policy->related_cpus);
1074         free_cpumask_var(policy->cpus);
1075         kfree(policy);
1076 }
1077
1078 static int update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu,
1079                              struct device *cpu_dev)
1080 {
1081         int ret;
1082
1083         if (WARN_ON(cpu == policy->cpu))
1084                 return 0;
1085
1086         /* Move kobject to the new policy->cpu */
1087         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1088         if (ret) {
1089                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1090                 return ret;
1091         }
1092
1093         down_write(&policy->rwsem);
1094
1095         policy->last_cpu = policy->cpu;
1096         policy->cpu = cpu;
1097
1098         up_write(&policy->rwsem);
1099
1100         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1101                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1102
1103         return 0;
1104 }
1105
1106 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1107 {
1108         unsigned int j, cpu = dev->id;
1109         int ret = -ENOMEM;
1110         struct cpufreq_policy *policy;
1111         unsigned long flags;
1112         bool recover_policy = cpufreq_suspended;
1113 #ifdef CONFIG_HOTPLUG_CPU
1114         struct cpufreq_policy *tpolicy;
1115 #endif
1116
1117         if (cpu_is_offline(cpu))
1118                 return 0;
1119
1120         pr_debug("adding CPU %u\n", cpu);
1121
1122 #ifdef CONFIG_SMP
1123         /* check whether a different CPU already registered this
1124          * CPU because it is in the same boat. */
1125         policy = cpufreq_cpu_get(cpu);
1126         if (unlikely(policy)) {
1127                 cpufreq_cpu_put(policy);
1128                 return 0;
1129         }
1130 #endif
1131
1132         if (!down_read_trylock(&cpufreq_rwsem))
1133                 return 0;
1134
1135 #ifdef CONFIG_HOTPLUG_CPU
1136         /* Check if this cpu was hot-unplugged earlier and has siblings */
1137         read_lock_irqsave(&cpufreq_driver_lock, flags);
1138         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1139                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1140                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1141                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1142                         up_read(&cpufreq_rwsem);
1143                         return ret;
1144                 }
1145         }
1146         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1147 #endif
1148
1149         /*
1150          * Restore the saved policy when doing light-weight init and fall back
1151          * to the full init if that fails.
1152          */
1153         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1154         if (!policy) {
1155                 recover_policy = false;
1156                 policy = cpufreq_policy_alloc();
1157                 if (!policy)
1158                         goto nomem_out;
1159         }
1160
1161         /*
1162          * In the resume path, since we restore a saved policy, the assignment
1163          * to policy->cpu is like an update of the existing policy, rather than
1164          * the creation of a brand new one. So we need to perform this update
1165          * by invoking update_policy_cpu().
1166          */
1167         if (recover_policy && cpu != policy->cpu)
1168                 WARN_ON(update_policy_cpu(policy, cpu, dev));
1169         else
1170                 policy->cpu = cpu;
1171
1172         cpumask_copy(policy->cpus, cpumask_of(cpu));
1173
1174         init_completion(&policy->kobj_unregister);
1175         INIT_WORK(&policy->update, handle_update);
1176
1177         /* call driver. From then on the cpufreq must be able
1178          * to accept all calls to ->verify and ->setpolicy for this CPU
1179          */
1180         ret = cpufreq_driver->init(policy);
1181         if (ret) {
1182                 pr_debug("initialization failed\n");
1183                 goto err_set_policy_cpu;
1184         }
1185
1186         down_write(&policy->rwsem);
1187
1188         /* related cpus should atleast have policy->cpus */
1189         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1190
1191         /*
1192          * affected cpus must always be the one, which are online. We aren't
1193          * managing offline cpus here.
1194          */
1195         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1196
1197         if (!recover_policy) {
1198                 policy->user_policy.min = policy->min;
1199                 policy->user_policy.max = policy->max;
1200
1201                 /* prepare interface data */
1202                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1203                                            &dev->kobj, "cpufreq");
1204                 if (ret) {
1205                         pr_err("%s: failed to init policy->kobj: %d\n",
1206                                __func__, ret);
1207                         goto err_init_policy_kobj;
1208                 }
1209         }
1210
1211         write_lock_irqsave(&cpufreq_driver_lock, flags);
1212         for_each_cpu(j, policy->cpus)
1213                 per_cpu(cpufreq_cpu_data, j) = policy;
1214         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1215
1216         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1217                 policy->cur = cpufreq_driver->get(policy->cpu);
1218                 if (!policy->cur) {
1219                         pr_err("%s: ->get() failed\n", __func__);
1220                         goto err_get_freq;
1221                 }
1222         }
1223
1224         /*
1225          * Sometimes boot loaders set CPU frequency to a value outside of
1226          * frequency table present with cpufreq core. In such cases CPU might be
1227          * unstable if it has to run on that frequency for long duration of time
1228          * and so its better to set it to a frequency which is specified in
1229          * freq-table. This also makes cpufreq stats inconsistent as
1230          * cpufreq-stats would fail to register because current frequency of CPU
1231          * isn't found in freq-table.
1232          *
1233          * Because we don't want this change to effect boot process badly, we go
1234          * for the next freq which is >= policy->cur ('cur' must be set by now,
1235          * otherwise we will end up setting freq to lowest of the table as 'cur'
1236          * is initialized to zero).
1237          *
1238          * We are passing target-freq as "policy->cur - 1" otherwise
1239          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1240          * equal to target-freq.
1241          */
1242         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1243             && has_target()) {
1244                 /* Are we running at unknown frequency ? */
1245                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1246                 if (ret == -EINVAL) {
1247                         /* Warn user and fix it */
1248                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1249                                 __func__, policy->cpu, policy->cur);
1250                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1251                                 CPUFREQ_RELATION_L);
1252
1253                         /*
1254                          * Reaching here after boot in a few seconds may not
1255                          * mean that system will remain stable at "unknown"
1256                          * frequency for longer duration. Hence, a BUG_ON().
1257                          */
1258                         BUG_ON(ret);
1259                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1260                                 __func__, policy->cpu, policy->cur);
1261                 }
1262         }
1263
1264         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1265                                      CPUFREQ_START, policy);
1266
1267         if (!recover_policy) {
1268                 ret = cpufreq_add_dev_interface(policy, dev);
1269                 if (ret)
1270                         goto err_out_unregister;
1271                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1272                                 CPUFREQ_CREATE_POLICY, policy);
1273         }
1274
1275         write_lock_irqsave(&cpufreq_driver_lock, flags);
1276         list_add(&policy->policy_list, &cpufreq_policy_list);
1277         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1278
1279         cpufreq_init_policy(policy);
1280
1281         if (!recover_policy) {
1282                 policy->user_policy.policy = policy->policy;
1283                 policy->user_policy.governor = policy->governor;
1284         }
1285         up_write(&policy->rwsem);
1286
1287         kobject_uevent(&policy->kobj, KOBJ_ADD);
1288
1289         up_read(&cpufreq_rwsem);
1290
1291         /* Callback for handling stuff after policy is ready */
1292         if (cpufreq_driver->ready)
1293                 cpufreq_driver->ready(policy);
1294
1295         pr_debug("initialization complete\n");
1296
1297         return 0;
1298
1299 err_out_unregister:
1300 err_get_freq:
1301         write_lock_irqsave(&cpufreq_driver_lock, flags);
1302         for_each_cpu(j, policy->cpus)
1303                 per_cpu(cpufreq_cpu_data, j) = NULL;
1304         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1305
1306         if (!recover_policy) {
1307                 kobject_put(&policy->kobj);
1308                 wait_for_completion(&policy->kobj_unregister);
1309         }
1310 err_init_policy_kobj:
1311         up_write(&policy->rwsem);
1312
1313         if (cpufreq_driver->exit)
1314                 cpufreq_driver->exit(policy);
1315 err_set_policy_cpu:
1316         if (recover_policy) {
1317                 /* Do not leave stale fallback data behind. */
1318                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1319                 cpufreq_policy_put_kobj(policy);
1320         }
1321         cpufreq_policy_free(policy);
1322
1323 nomem_out:
1324         up_read(&cpufreq_rwsem);
1325
1326         return ret;
1327 }
1328
1329 /**
1330  * cpufreq_add_dev - add a CPU device
1331  *
1332  * Adds the cpufreq interface for a CPU device.
1333  *
1334  * The Oracle says: try running cpufreq registration/unregistration concurrently
1335  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1336  * mess up, but more thorough testing is needed. - Mathieu
1337  */
1338 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1339 {
1340         return __cpufreq_add_dev(dev, sif);
1341 }
1342
1343 static int __cpufreq_remove_dev_prepare(struct device *dev,
1344                                         struct subsys_interface *sif)
1345 {
1346         unsigned int cpu = dev->id, cpus;
1347         int ret;
1348         unsigned long flags;
1349         struct cpufreq_policy *policy;
1350
1351         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1352
1353         write_lock_irqsave(&cpufreq_driver_lock, flags);
1354
1355         policy = per_cpu(cpufreq_cpu_data, cpu);
1356
1357         /* Save the policy somewhere when doing a light-weight tear-down */
1358         if (cpufreq_suspended)
1359                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1360
1361         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1362
1363         if (!policy) {
1364                 pr_debug("%s: No cpu_data found\n", __func__);
1365                 return -EINVAL;
1366         }
1367
1368         if (has_target()) {
1369                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1370                 if (ret) {
1371                         pr_err("%s: Failed to stop governor\n", __func__);
1372                         return ret;
1373                 }
1374
1375                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1376                         policy->governor->name, CPUFREQ_NAME_LEN);
1377         }
1378
1379         down_read(&policy->rwsem);
1380         cpus = cpumask_weight(policy->cpus);
1381         up_read(&policy->rwsem);
1382
1383         if (cpu != policy->cpu) {
1384                 sysfs_remove_link(&dev->kobj, "cpufreq");
1385         } else if (cpus > 1) {
1386                 /* Nominate new CPU */
1387                 int new_cpu = cpumask_any_but(policy->cpus, cpu);
1388                 struct device *cpu_dev = get_cpu_device(new_cpu);
1389
1390                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1391                 ret = update_policy_cpu(policy, new_cpu, cpu_dev);
1392                 if (ret) {
1393                         if (sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1394                                               "cpufreq"))
1395                                 pr_err("%s: Failed to restore kobj link to cpu:%d\n",
1396                                        __func__, cpu_dev->id);
1397                         return ret;
1398                 }
1399
1400                 if (!cpufreq_suspended)
1401                         pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1402                                  __func__, new_cpu, cpu);
1403         } else if (cpufreq_driver->stop_cpu) {
1404                 cpufreq_driver->stop_cpu(policy);
1405         }
1406
1407         return 0;
1408 }
1409
1410 static int __cpufreq_remove_dev_finish(struct device *dev,
1411                                        struct subsys_interface *sif)
1412 {
1413         unsigned int cpu = dev->id, cpus;
1414         int ret;
1415         unsigned long flags;
1416         struct cpufreq_policy *policy;
1417
1418         read_lock_irqsave(&cpufreq_driver_lock, flags);
1419         policy = per_cpu(cpufreq_cpu_data, cpu);
1420         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1421
1422         if (!policy) {
1423                 pr_debug("%s: No cpu_data found\n", __func__);
1424                 return -EINVAL;
1425         }
1426
1427         down_write(&policy->rwsem);
1428         cpus = cpumask_weight(policy->cpus);
1429
1430         if (cpus > 1)
1431                 cpumask_clear_cpu(cpu, policy->cpus);
1432         up_write(&policy->rwsem);
1433
1434         /* If cpu is last user of policy, free policy */
1435         if (cpus == 1) {
1436                 if (has_target()) {
1437                         ret = __cpufreq_governor(policy,
1438                                         CPUFREQ_GOV_POLICY_EXIT);
1439                         if (ret) {
1440                                 pr_err("%s: Failed to exit governor\n",
1441                                        __func__);
1442                                 return ret;
1443                         }
1444                 }
1445
1446                 if (!cpufreq_suspended)
1447                         cpufreq_policy_put_kobj(policy);
1448
1449                 /*
1450                  * Perform the ->exit() even during light-weight tear-down,
1451                  * since this is a core component, and is essential for the
1452                  * subsequent light-weight ->init() to succeed.
1453                  */
1454                 if (cpufreq_driver->exit)
1455                         cpufreq_driver->exit(policy);
1456
1457                 /* Remove policy from list of active policies */
1458                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1459                 list_del(&policy->policy_list);
1460                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1461
1462                 if (!cpufreq_suspended)
1463                         cpufreq_policy_free(policy);
1464         } else if (has_target()) {
1465                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1466                 if (!ret)
1467                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1468
1469                 if (ret) {
1470                         pr_err("%s: Failed to start governor\n", __func__);
1471                         return ret;
1472                 }
1473         }
1474
1475         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1476         return 0;
1477 }
1478
1479 /**
1480  * cpufreq_remove_dev - remove a CPU device
1481  *
1482  * Removes the cpufreq interface for a CPU device.
1483  */
1484 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1485 {
1486         unsigned int cpu = dev->id;
1487         int ret;
1488
1489         if (cpu_is_offline(cpu))
1490                 return 0;
1491
1492         ret = __cpufreq_remove_dev_prepare(dev, sif);
1493
1494         if (!ret)
1495                 ret = __cpufreq_remove_dev_finish(dev, sif);
1496
1497         return ret;
1498 }
1499
1500 static void handle_update(struct work_struct *work)
1501 {
1502         struct cpufreq_policy *policy =
1503                 container_of(work, struct cpufreq_policy, update);
1504         unsigned int cpu = policy->cpu;
1505         pr_debug("handle_update for cpu %u called\n", cpu);
1506         cpufreq_update_policy(cpu);
1507 }
1508
1509 /**
1510  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1511  *      in deep trouble.
1512  *      @cpu: cpu number
1513  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1514  *      @new_freq: CPU frequency the CPU actually runs at
1515  *
1516  *      We adjust to current frequency first, and need to clean up later.
1517  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1518  */
1519 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1520                                 unsigned int new_freq)
1521 {
1522         struct cpufreq_policy *policy;
1523         struct cpufreq_freqs freqs;
1524         unsigned long flags;
1525
1526         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1527                  old_freq, new_freq);
1528
1529         freqs.old = old_freq;
1530         freqs.new = new_freq;
1531
1532         read_lock_irqsave(&cpufreq_driver_lock, flags);
1533         policy = per_cpu(cpufreq_cpu_data, cpu);
1534         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1535
1536         cpufreq_freq_transition_begin(policy, &freqs);
1537         cpufreq_freq_transition_end(policy, &freqs, 0);
1538 }
1539
1540 /**
1541  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1542  * @cpu: CPU number
1543  *
1544  * This is the last known freq, without actually getting it from the driver.
1545  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1546  */
1547 unsigned int cpufreq_quick_get(unsigned int cpu)
1548 {
1549         struct cpufreq_policy *policy;
1550         unsigned int ret_freq = 0;
1551
1552         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1553                 return cpufreq_driver->get(cpu);
1554
1555         policy = cpufreq_cpu_get(cpu);
1556         if (policy) {
1557                 ret_freq = policy->cur;
1558                 cpufreq_cpu_put(policy);
1559         }
1560
1561         return ret_freq;
1562 }
1563 EXPORT_SYMBOL(cpufreq_quick_get);
1564
1565 /**
1566  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1567  * @cpu: CPU number
1568  *
1569  * Just return the max possible frequency for a given CPU.
1570  */
1571 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1572 {
1573         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1574         unsigned int ret_freq = 0;
1575
1576         if (policy) {
1577                 ret_freq = policy->max;
1578                 cpufreq_cpu_put(policy);
1579         }
1580
1581         return ret_freq;
1582 }
1583 EXPORT_SYMBOL(cpufreq_quick_get_max);
1584
1585 static unsigned int __cpufreq_get(unsigned int cpu)
1586 {
1587         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1588         unsigned int ret_freq = 0;
1589
1590         if (!cpufreq_driver->get)
1591                 return ret_freq;
1592
1593         ret_freq = cpufreq_driver->get(cpu);
1594
1595         if (ret_freq && policy->cur &&
1596                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1597                 /* verify no discrepancy between actual and
1598                                         saved value exists */
1599                 if (unlikely(ret_freq != policy->cur)) {
1600                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1601                         schedule_work(&policy->update);
1602                 }
1603         }
1604
1605         return ret_freq;
1606 }
1607
1608 /**
1609  * cpufreq_get - get the current CPU frequency (in kHz)
1610  * @cpu: CPU number
1611  *
1612  * Get the CPU current (static) CPU frequency
1613  */
1614 unsigned int cpufreq_get(unsigned int cpu)
1615 {
1616         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1617         unsigned int ret_freq = 0;
1618
1619         if (policy) {
1620                 down_read(&policy->rwsem);
1621                 ret_freq = __cpufreq_get(cpu);
1622                 up_read(&policy->rwsem);
1623
1624                 cpufreq_cpu_put(policy);
1625         }
1626
1627         return ret_freq;
1628 }
1629 EXPORT_SYMBOL(cpufreq_get);
1630
1631 static struct subsys_interface cpufreq_interface = {
1632         .name           = "cpufreq",
1633         .subsys         = &cpu_subsys,
1634         .add_dev        = cpufreq_add_dev,
1635         .remove_dev     = cpufreq_remove_dev,
1636 };
1637
1638 /*
1639  * In case platform wants some specific frequency to be configured
1640  * during suspend..
1641  */
1642 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1643 {
1644         int ret;
1645
1646         if (!policy->suspend_freq) {
1647                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1648                 return -EINVAL;
1649         }
1650
1651         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1652                         policy->suspend_freq);
1653
1654         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1655                         CPUFREQ_RELATION_H);
1656         if (ret)
1657                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1658                                 __func__, policy->suspend_freq, ret);
1659
1660         return ret;
1661 }
1662 EXPORT_SYMBOL(cpufreq_generic_suspend);
1663
1664 /**
1665  * cpufreq_suspend() - Suspend CPUFreq governors
1666  *
1667  * Called during system wide Suspend/Hibernate cycles for suspending governors
1668  * as some platforms can't change frequency after this point in suspend cycle.
1669  * Because some of the devices (like: i2c, regulators, etc) they use for
1670  * changing frequency are suspended quickly after this point.
1671  */
1672 void cpufreq_suspend(void)
1673 {
1674         struct cpufreq_policy *policy;
1675
1676         if (!cpufreq_driver)
1677                 return;
1678
1679         if (!has_target())
1680                 goto suspend;
1681
1682         pr_debug("%s: Suspending Governors\n", __func__);
1683
1684         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1685                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1686                         pr_err("%s: Failed to stop governor for policy: %p\n",
1687                                 __func__, policy);
1688                 else if (cpufreq_driver->suspend
1689                     && cpufreq_driver->suspend(policy))
1690                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1691                                 policy);
1692         }
1693
1694 suspend:
1695         cpufreq_suspended = true;
1696 }
1697
1698 /**
1699  * cpufreq_resume() - Resume CPUFreq governors
1700  *
1701  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1702  * are suspended with cpufreq_suspend().
1703  */
1704 void cpufreq_resume(void)
1705 {
1706         struct cpufreq_policy *policy;
1707
1708         if (!cpufreq_driver)
1709                 return;
1710
1711         cpufreq_suspended = false;
1712
1713         if (!has_target())
1714                 return;
1715
1716         pr_debug("%s: Resuming Governors\n", __func__);
1717
1718         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1719                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1720                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1721                                 policy);
1722                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1723                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1724                         pr_err("%s: Failed to start governor for policy: %p\n",
1725                                 __func__, policy);
1726
1727                 /*
1728                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1729                  * policy in list. It will verify that the current freq is in
1730                  * sync with what we believe it to be.
1731                  */
1732                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1733                         schedule_work(&policy->update);
1734         }
1735 }
1736
1737 /**
1738  *      cpufreq_get_current_driver - return current driver's name
1739  *
1740  *      Return the name string of the currently loaded cpufreq driver
1741  *      or NULL, if none.
1742  */
1743 const char *cpufreq_get_current_driver(void)
1744 {
1745         if (cpufreq_driver)
1746                 return cpufreq_driver->name;
1747
1748         return NULL;
1749 }
1750 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1751
1752 /**
1753  *      cpufreq_get_driver_data - return current driver data
1754  *
1755  *      Return the private data of the currently loaded cpufreq
1756  *      driver, or NULL if no cpufreq driver is loaded.
1757  */
1758 void *cpufreq_get_driver_data(void)
1759 {
1760         if (cpufreq_driver)
1761                 return cpufreq_driver->driver_data;
1762
1763         return NULL;
1764 }
1765 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1766
1767 /*********************************************************************
1768  *                     NOTIFIER LISTS INTERFACE                      *
1769  *********************************************************************/
1770
1771 /**
1772  *      cpufreq_register_notifier - register a driver with cpufreq
1773  *      @nb: notifier function to register
1774  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1775  *
1776  *      Add a driver to one of two lists: either a list of drivers that
1777  *      are notified about clock rate changes (once before and once after
1778  *      the transition), or a list of drivers that are notified about
1779  *      changes in cpufreq policy.
1780  *
1781  *      This function may sleep, and has the same return conditions as
1782  *      blocking_notifier_chain_register.
1783  */
1784 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1785 {
1786         int ret;
1787
1788         if (cpufreq_disabled())
1789                 return -EINVAL;
1790
1791         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1792
1793         switch (list) {
1794         case CPUFREQ_TRANSITION_NOTIFIER:
1795                 ret = srcu_notifier_chain_register(
1796                                 &cpufreq_transition_notifier_list, nb);
1797                 break;
1798         case CPUFREQ_POLICY_NOTIFIER:
1799                 ret = blocking_notifier_chain_register(
1800                                 &cpufreq_policy_notifier_list, nb);
1801                 break;
1802         default:
1803                 ret = -EINVAL;
1804         }
1805
1806         return ret;
1807 }
1808 EXPORT_SYMBOL(cpufreq_register_notifier);
1809
1810 /**
1811  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1812  *      @nb: notifier block to be unregistered
1813  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1814  *
1815  *      Remove a driver from the CPU frequency notifier list.
1816  *
1817  *      This function may sleep, and has the same return conditions as
1818  *      blocking_notifier_chain_unregister.
1819  */
1820 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1821 {
1822         int ret;
1823
1824         if (cpufreq_disabled())
1825                 return -EINVAL;
1826
1827         switch (list) {
1828         case CPUFREQ_TRANSITION_NOTIFIER:
1829                 ret = srcu_notifier_chain_unregister(
1830                                 &cpufreq_transition_notifier_list, nb);
1831                 break;
1832         case CPUFREQ_POLICY_NOTIFIER:
1833                 ret = blocking_notifier_chain_unregister(
1834                                 &cpufreq_policy_notifier_list, nb);
1835                 break;
1836         default:
1837                 ret = -EINVAL;
1838         }
1839
1840         return ret;
1841 }
1842 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1843
1844
1845 /*********************************************************************
1846  *                              GOVERNORS                            *
1847  *********************************************************************/
1848
1849 /* Must set freqs->new to intermediate frequency */
1850 static int __target_intermediate(struct cpufreq_policy *policy,
1851                                  struct cpufreq_freqs *freqs, int index)
1852 {
1853         int ret;
1854
1855         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1856
1857         /* We don't need to switch to intermediate freq */
1858         if (!freqs->new)
1859                 return 0;
1860
1861         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1862                  __func__, policy->cpu, freqs->old, freqs->new);
1863
1864         cpufreq_freq_transition_begin(policy, freqs);
1865         ret = cpufreq_driver->target_intermediate(policy, index);
1866         cpufreq_freq_transition_end(policy, freqs, ret);
1867
1868         if (ret)
1869                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1870                        __func__, ret);
1871
1872         return ret;
1873 }
1874
1875 static int __target_index(struct cpufreq_policy *policy,
1876                           struct cpufreq_frequency_table *freq_table, int index)
1877 {
1878         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1879         unsigned int intermediate_freq = 0;
1880         int retval = -EINVAL;
1881         bool notify;
1882
1883         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1884         if (notify) {
1885                 /* Handle switching to intermediate frequency */
1886                 if (cpufreq_driver->get_intermediate) {
1887                         retval = __target_intermediate(policy, &freqs, index);
1888                         if (retval)
1889                                 return retval;
1890
1891                         intermediate_freq = freqs.new;
1892                         /* Set old freq to intermediate */
1893                         if (intermediate_freq)
1894                                 freqs.old = freqs.new;
1895                 }
1896
1897                 freqs.new = freq_table[index].frequency;
1898                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1899                          __func__, policy->cpu, freqs.old, freqs.new);
1900
1901                 cpufreq_freq_transition_begin(policy, &freqs);
1902         }
1903
1904         retval = cpufreq_driver->target_index(policy, index);
1905         if (retval)
1906                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1907                        retval);
1908
1909         if (notify) {
1910                 cpufreq_freq_transition_end(policy, &freqs, retval);
1911
1912                 /*
1913                  * Failed after setting to intermediate freq? Driver should have
1914                  * reverted back to initial frequency and so should we. Check
1915                  * here for intermediate_freq instead of get_intermediate, in
1916                  * case we have't switched to intermediate freq at all.
1917                  */
1918                 if (unlikely(retval && intermediate_freq)) {
1919                         freqs.old = intermediate_freq;
1920                         freqs.new = policy->restore_freq;
1921                         cpufreq_freq_transition_begin(policy, &freqs);
1922                         cpufreq_freq_transition_end(policy, &freqs, 0);
1923                 }
1924         }
1925
1926         return retval;
1927 }
1928
1929 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1930                             unsigned int target_freq,
1931                             unsigned int relation)
1932 {
1933         unsigned int old_target_freq = target_freq;
1934         int retval = -EINVAL;
1935
1936         if (cpufreq_disabled())
1937                 return -ENODEV;
1938
1939         /* Make sure that target_freq is within supported range */
1940         if (target_freq > policy->max)
1941                 target_freq = policy->max;
1942         if (target_freq < policy->min)
1943                 target_freq = policy->min;
1944
1945         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1946                  policy->cpu, target_freq, relation, old_target_freq);
1947
1948         /*
1949          * This might look like a redundant call as we are checking it again
1950          * after finding index. But it is left intentionally for cases where
1951          * exactly same freq is called again and so we can save on few function
1952          * calls.
1953          */
1954         if (target_freq == policy->cur)
1955                 return 0;
1956
1957         /* Save last value to restore later on errors */
1958         policy->restore_freq = policy->cur;
1959
1960         if (cpufreq_driver->target)
1961                 retval = cpufreq_driver->target(policy, target_freq, relation);
1962         else if (cpufreq_driver->target_index) {
1963                 struct cpufreq_frequency_table *freq_table;
1964                 int index;
1965
1966                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1967                 if (unlikely(!freq_table)) {
1968                         pr_err("%s: Unable to find freq_table\n", __func__);
1969                         goto out;
1970                 }
1971
1972                 retval = cpufreq_frequency_table_target(policy, freq_table,
1973                                 target_freq, relation, &index);
1974                 if (unlikely(retval)) {
1975                         pr_err("%s: Unable to find matching freq\n", __func__);
1976                         goto out;
1977                 }
1978
1979                 if (freq_table[index].frequency == policy->cur) {
1980                         retval = 0;
1981                         goto out;
1982                 }
1983
1984                 retval = __target_index(policy, freq_table, index);
1985         }
1986
1987 out:
1988         return retval;
1989 }
1990 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1991
1992 int cpufreq_driver_target(struct cpufreq_policy *policy,
1993                           unsigned int target_freq,
1994                           unsigned int relation)
1995 {
1996         int ret = -EINVAL;
1997
1998         down_write(&policy->rwsem);
1999
2000         ret = __cpufreq_driver_target(policy, target_freq, relation);
2001
2002         up_write(&policy->rwsem);
2003
2004         return ret;
2005 }
2006 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2007
2008 static int __cpufreq_governor(struct cpufreq_policy *policy,
2009                                         unsigned int event)
2010 {
2011         int ret;
2012
2013         /* Only must be defined when default governor is known to have latency
2014            restrictions, like e.g. conservative or ondemand.
2015            That this is the case is already ensured in Kconfig
2016         */
2017 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2018         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2019 #else
2020         struct cpufreq_governor *gov = NULL;
2021 #endif
2022
2023         /* Don't start any governor operations if we are entering suspend */
2024         if (cpufreq_suspended)
2025                 return 0;
2026         /*
2027          * Governor might not be initiated here if ACPI _PPC changed
2028          * notification happened, so check it.
2029          */
2030         if (!policy->governor)
2031                 return -EINVAL;
2032
2033         if (policy->governor->max_transition_latency &&
2034             policy->cpuinfo.transition_latency >
2035             policy->governor->max_transition_latency) {
2036                 if (!gov)
2037                         return -EINVAL;
2038                 else {
2039                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2040                                 policy->governor->name, gov->name);
2041                         policy->governor = gov;
2042                 }
2043         }
2044
2045         if (event == CPUFREQ_GOV_POLICY_INIT)
2046                 if (!try_module_get(policy->governor->owner))
2047                         return -EINVAL;
2048
2049         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2050                  policy->cpu, event);
2051
2052         mutex_lock(&cpufreq_governor_lock);
2053         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2054             || (!policy->governor_enabled
2055             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2056                 mutex_unlock(&cpufreq_governor_lock);
2057                 return -EBUSY;
2058         }
2059
2060         if (event == CPUFREQ_GOV_STOP)
2061                 policy->governor_enabled = false;
2062         else if (event == CPUFREQ_GOV_START)
2063                 policy->governor_enabled = true;
2064
2065         mutex_unlock(&cpufreq_governor_lock);
2066
2067         ret = policy->governor->governor(policy, event);
2068
2069         if (!ret) {
2070                 if (event == CPUFREQ_GOV_POLICY_INIT)
2071                         policy->governor->initialized++;
2072                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2073                         policy->governor->initialized--;
2074         } else {
2075                 /* Restore original values */
2076                 mutex_lock(&cpufreq_governor_lock);
2077                 if (event == CPUFREQ_GOV_STOP)
2078                         policy->governor_enabled = true;
2079                 else if (event == CPUFREQ_GOV_START)
2080                         policy->governor_enabled = false;
2081                 mutex_unlock(&cpufreq_governor_lock);
2082         }
2083
2084         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2085                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2086                 module_put(policy->governor->owner);
2087
2088         return ret;
2089 }
2090
2091 int cpufreq_register_governor(struct cpufreq_governor *governor)
2092 {
2093         int err;
2094
2095         if (!governor)
2096                 return -EINVAL;
2097
2098         if (cpufreq_disabled())
2099                 return -ENODEV;
2100
2101         mutex_lock(&cpufreq_governor_mutex);
2102
2103         governor->initialized = 0;
2104         err = -EBUSY;
2105         if (!find_governor(governor->name)) {
2106                 err = 0;
2107                 list_add(&governor->governor_list, &cpufreq_governor_list);
2108         }
2109
2110         mutex_unlock(&cpufreq_governor_mutex);
2111         return err;
2112 }
2113 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2114
2115 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2116 {
2117         int cpu;
2118
2119         if (!governor)
2120                 return;
2121
2122         if (cpufreq_disabled())
2123                 return;
2124
2125         for_each_present_cpu(cpu) {
2126                 if (cpu_online(cpu))
2127                         continue;
2128                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2129                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2130         }
2131
2132         mutex_lock(&cpufreq_governor_mutex);
2133         list_del(&governor->governor_list);
2134         mutex_unlock(&cpufreq_governor_mutex);
2135         return;
2136 }
2137 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2138
2139
2140 /*********************************************************************
2141  *                          POLICY INTERFACE                         *
2142  *********************************************************************/
2143
2144 /**
2145  * cpufreq_get_policy - get the current cpufreq_policy
2146  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2147  *      is written
2148  *
2149  * Reads the current cpufreq policy.
2150  */
2151 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2152 {
2153         struct cpufreq_policy *cpu_policy;
2154         if (!policy)
2155                 return -EINVAL;
2156
2157         cpu_policy = cpufreq_cpu_get(cpu);
2158         if (!cpu_policy)
2159                 return -EINVAL;
2160
2161         memcpy(policy, cpu_policy, sizeof(*policy));
2162
2163         cpufreq_cpu_put(cpu_policy);
2164         return 0;
2165 }
2166 EXPORT_SYMBOL(cpufreq_get_policy);
2167
2168 /*
2169  * policy : current policy.
2170  * new_policy: policy to be set.
2171  */
2172 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2173                                 struct cpufreq_policy *new_policy)
2174 {
2175         struct cpufreq_governor *old_gov;
2176         int ret;
2177
2178         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2179                  new_policy->cpu, new_policy->min, new_policy->max);
2180
2181         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2182
2183         if (new_policy->min > policy->max || new_policy->max < policy->min)
2184                 return -EINVAL;
2185
2186         /* verify the cpu speed can be set within this limit */
2187         ret = cpufreq_driver->verify(new_policy);
2188         if (ret)
2189                 return ret;
2190
2191         /* adjust if necessary - all reasons */
2192         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2193                         CPUFREQ_ADJUST, new_policy);
2194
2195         /* adjust if necessary - hardware incompatibility*/
2196         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2197                         CPUFREQ_INCOMPATIBLE, new_policy);
2198
2199         /*
2200          * verify the cpu speed can be set within this limit, which might be
2201          * different to the first one
2202          */
2203         ret = cpufreq_driver->verify(new_policy);
2204         if (ret)
2205                 return ret;
2206
2207         /* notification of the new policy */
2208         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2209                         CPUFREQ_NOTIFY, new_policy);
2210
2211         policy->min = new_policy->min;
2212         policy->max = new_policy->max;
2213
2214         pr_debug("new min and max freqs are %u - %u kHz\n",
2215                  policy->min, policy->max);
2216
2217         if (cpufreq_driver->setpolicy) {
2218                 policy->policy = new_policy->policy;
2219                 pr_debug("setting range\n");
2220                 return cpufreq_driver->setpolicy(new_policy);
2221         }
2222
2223         if (new_policy->governor == policy->governor)
2224                 goto out;
2225
2226         pr_debug("governor switch\n");
2227
2228         /* save old, working values */
2229         old_gov = policy->governor;
2230         /* end old governor */
2231         if (old_gov) {
2232                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2233                 up_write(&policy->rwsem);
2234                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2235                 down_write(&policy->rwsem);
2236         }
2237
2238         /* start new governor */
2239         policy->governor = new_policy->governor;
2240         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2241                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2242                         goto out;
2243
2244                 up_write(&policy->rwsem);
2245                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2246                 down_write(&policy->rwsem);
2247         }
2248
2249         /* new governor failed, so re-start old one */
2250         pr_debug("starting governor %s failed\n", policy->governor->name);
2251         if (old_gov) {
2252                 policy->governor = old_gov;
2253                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2254                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2255         }
2256
2257         return -EINVAL;
2258
2259  out:
2260         pr_debug("governor: change or update limits\n");
2261         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2262 }
2263
2264 /**
2265  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2266  *      @cpu: CPU which shall be re-evaluated
2267  *
2268  *      Useful for policy notifiers which have different necessities
2269  *      at different times.
2270  */
2271 int cpufreq_update_policy(unsigned int cpu)
2272 {
2273         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2274         struct cpufreq_policy new_policy;
2275         int ret;
2276
2277         if (!policy)
2278                 return -ENODEV;
2279
2280         down_write(&policy->rwsem);
2281
2282         pr_debug("updating policy for CPU %u\n", cpu);
2283         memcpy(&new_policy, policy, sizeof(*policy));
2284         new_policy.min = policy->user_policy.min;
2285         new_policy.max = policy->user_policy.max;
2286         new_policy.policy = policy->user_policy.policy;
2287         new_policy.governor = policy->user_policy.governor;
2288
2289         /*
2290          * BIOS might change freq behind our back
2291          * -> ask driver for current freq and notify governors about a change
2292          */
2293         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2294                 new_policy.cur = cpufreq_driver->get(cpu);
2295                 if (WARN_ON(!new_policy.cur)) {
2296                         ret = -EIO;
2297                         goto unlock;
2298                 }
2299
2300                 if (!policy->cur) {
2301                         pr_debug("Driver did not initialize current freq\n");
2302                         policy->cur = new_policy.cur;
2303                 } else {
2304                         if (policy->cur != new_policy.cur && has_target())
2305                                 cpufreq_out_of_sync(cpu, policy->cur,
2306                                                                 new_policy.cur);
2307                 }
2308         }
2309
2310         ret = cpufreq_set_policy(policy, &new_policy);
2311
2312 unlock:
2313         up_write(&policy->rwsem);
2314
2315         cpufreq_cpu_put(policy);
2316         return ret;
2317 }
2318 EXPORT_SYMBOL(cpufreq_update_policy);
2319
2320 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2321                                         unsigned long action, void *hcpu)
2322 {
2323         unsigned int cpu = (unsigned long)hcpu;
2324         struct device *dev;
2325
2326         dev = get_cpu_device(cpu);
2327         if (dev) {
2328                 switch (action & ~CPU_TASKS_FROZEN) {
2329                 case CPU_ONLINE:
2330                         __cpufreq_add_dev(dev, NULL);
2331                         break;
2332
2333                 case CPU_DOWN_PREPARE:
2334                         __cpufreq_remove_dev_prepare(dev, NULL);
2335                         break;
2336
2337                 case CPU_POST_DEAD:
2338                         __cpufreq_remove_dev_finish(dev, NULL);
2339                         break;
2340
2341                 case CPU_DOWN_FAILED:
2342                         __cpufreq_add_dev(dev, NULL);
2343                         break;
2344                 }
2345         }
2346         return NOTIFY_OK;
2347 }
2348
2349 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2350         .notifier_call = cpufreq_cpu_callback,
2351 };
2352
2353 /*********************************************************************
2354  *               BOOST                                               *
2355  *********************************************************************/
2356 static int cpufreq_boost_set_sw(int state)
2357 {
2358         struct cpufreq_frequency_table *freq_table;
2359         struct cpufreq_policy *policy;
2360         int ret = -EINVAL;
2361
2362         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2363                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2364                 if (freq_table) {
2365                         ret = cpufreq_frequency_table_cpuinfo(policy,
2366                                                         freq_table);
2367                         if (ret) {
2368                                 pr_err("%s: Policy frequency update failed\n",
2369                                        __func__);
2370                                 break;
2371                         }
2372                         policy->user_policy.max = policy->max;
2373                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2374                 }
2375         }
2376
2377         return ret;
2378 }
2379
2380 int cpufreq_boost_trigger_state(int state)
2381 {
2382         unsigned long flags;
2383         int ret = 0;
2384
2385         if (cpufreq_driver->boost_enabled == state)
2386                 return 0;
2387
2388         write_lock_irqsave(&cpufreq_driver_lock, flags);
2389         cpufreq_driver->boost_enabled = state;
2390         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2391
2392         ret = cpufreq_driver->set_boost(state);
2393         if (ret) {
2394                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2395                 cpufreq_driver->boost_enabled = !state;
2396                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2397
2398                 pr_err("%s: Cannot %s BOOST\n",
2399                        __func__, state ? "enable" : "disable");
2400         }
2401
2402         return ret;
2403 }
2404
2405 int cpufreq_boost_supported(void)
2406 {
2407         if (likely(cpufreq_driver))
2408                 return cpufreq_driver->boost_supported;
2409
2410         return 0;
2411 }
2412 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2413
2414 int cpufreq_boost_enabled(void)
2415 {
2416         return cpufreq_driver->boost_enabled;
2417 }
2418 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2419
2420 /*********************************************************************
2421  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2422  *********************************************************************/
2423
2424 /**
2425  * cpufreq_register_driver - register a CPU Frequency driver
2426  * @driver_data: A struct cpufreq_driver containing the values#
2427  * submitted by the CPU Frequency driver.
2428  *
2429  * Registers a CPU Frequency driver to this core code. This code
2430  * returns zero on success, -EBUSY when another driver got here first
2431  * (and isn't unregistered in the meantime).
2432  *
2433  */
2434 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2435 {
2436         unsigned long flags;
2437         int ret;
2438
2439         if (cpufreq_disabled())
2440                 return -ENODEV;
2441
2442         if (!driver_data || !driver_data->verify || !driver_data->init ||
2443             !(driver_data->setpolicy || driver_data->target_index ||
2444                     driver_data->target) ||
2445              (driver_data->setpolicy && (driver_data->target_index ||
2446                     driver_data->target)) ||
2447              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2448                 return -EINVAL;
2449
2450         pr_debug("trying to register driver %s\n", driver_data->name);
2451
2452         if (driver_data->setpolicy)
2453                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2454
2455         write_lock_irqsave(&cpufreq_driver_lock, flags);
2456         if (cpufreq_driver) {
2457                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2458                 return -EEXIST;
2459         }
2460         cpufreq_driver = driver_data;
2461         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2462
2463         if (cpufreq_boost_supported()) {
2464                 /*
2465                  * Check if driver provides function to enable boost -
2466                  * if not, use cpufreq_boost_set_sw as default
2467                  */
2468                 if (!cpufreq_driver->set_boost)
2469                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2470
2471                 ret = cpufreq_sysfs_create_file(&boost.attr);
2472                 if (ret) {
2473                         pr_err("%s: cannot register global BOOST sysfs file\n",
2474                                __func__);
2475                         goto err_null_driver;
2476                 }
2477         }
2478
2479         ret = subsys_interface_register(&cpufreq_interface);
2480         if (ret)
2481                 goto err_boost_unreg;
2482
2483         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2484                 int i;
2485                 ret = -ENODEV;
2486
2487                 /* check for at least one working CPU */
2488                 for (i = 0; i < nr_cpu_ids; i++)
2489                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2490                                 ret = 0;
2491                                 break;
2492                         }
2493
2494                 /* if all ->init() calls failed, unregister */
2495                 if (ret) {
2496                         pr_debug("no CPU initialized for driver %s\n",
2497                                  driver_data->name);
2498                         goto err_if_unreg;
2499                 }
2500         }
2501
2502         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2503         pr_debug("driver %s up and running\n", driver_data->name);
2504
2505         return 0;
2506 err_if_unreg:
2507         subsys_interface_unregister(&cpufreq_interface);
2508 err_boost_unreg:
2509         if (cpufreq_boost_supported())
2510                 cpufreq_sysfs_remove_file(&boost.attr);
2511 err_null_driver:
2512         write_lock_irqsave(&cpufreq_driver_lock, flags);
2513         cpufreq_driver = NULL;
2514         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2515         return ret;
2516 }
2517 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2518
2519 /**
2520  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2521  *
2522  * Unregister the current CPUFreq driver. Only call this if you have
2523  * the right to do so, i.e. if you have succeeded in initialising before!
2524  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2525  * currently not initialised.
2526  */
2527 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2528 {
2529         unsigned long flags;
2530
2531         if (!cpufreq_driver || (driver != cpufreq_driver))
2532                 return -EINVAL;
2533
2534         pr_debug("unregistering driver %s\n", driver->name);
2535
2536         subsys_interface_unregister(&cpufreq_interface);
2537         if (cpufreq_boost_supported())
2538                 cpufreq_sysfs_remove_file(&boost.attr);
2539
2540         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2541
2542         down_write(&cpufreq_rwsem);
2543         write_lock_irqsave(&cpufreq_driver_lock, flags);
2544
2545         cpufreq_driver = NULL;
2546
2547         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2548         up_write(&cpufreq_rwsem);
2549
2550         return 0;
2551 }
2552 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2553
2554 /*
2555  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2556  * or mutexes when secondary CPUs are halted.
2557  */
2558 static struct syscore_ops cpufreq_syscore_ops = {
2559         .shutdown = cpufreq_suspend,
2560 };
2561
2562 static int __init cpufreq_core_init(void)
2563 {
2564         if (cpufreq_disabled())
2565                 return -ENODEV;
2566
2567         cpufreq_global_kobject = kobject_create();
2568         BUG_ON(!cpufreq_global_kobject);
2569
2570         register_syscore_ops(&cpufreq_syscore_ops);
2571
2572         return 0;
2573 }
2574 core_initcall(cpufreq_core_init);