drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 /* This one keeps track of the previously set governor of a removed CPU */
  46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  47
  48 /* Flag to suspend/resume CPUFreq governors */
  49 static bool cpufreq_suspended;
  50
  51 static inline bool has_target(void)
  52 {
  53         return cpufreq_driver->target_index || cpufreq_driver->target;
  54 }
  55
  56 /*
  57  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  58  * sections
  59  */
  60 static DECLARE_RWSEM(cpufreq_rwsem);
  61
  62 /* internal prototypes */
  63 static int __cpufreq_governor(struct cpufreq_policy *policy,
  64                 unsigned int event);
  65 static unsigned int __cpufreq_get(unsigned int cpu);
  66 static void handle_update(struct work_struct *work);
  67
  68 /**
  69  * Two notifier lists: the "policy" list is involved in the
  70  * validation process for a new CPU frequency policy; the
  71  * "transition" list for kernel code that needs to handle
  72  * changes to devices when the CPU clock speed changes.
  73  * The mutex locks both lists.
  74  */
  75 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  76 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  77
  78 static bool init_cpufreq_transition_notifier_list_called;
  79 static int __init init_cpufreq_transition_notifier_list(void)
  80 {
  81         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  82         init_cpufreq_transition_notifier_list_called = true;
  83         return 0;
  84 }
  85 pure_initcall(init_cpufreq_transition_notifier_list);
  86
  87 static int off __read_mostly;
  88 static int cpufreq_disabled(void)
  89 {
  90         return off;
  91 }
  92 void disable_cpufreq(void)
  93 {
  94         off = 1;
  95 }
  96 static LIST_HEAD(cpufreq_governor_list);
  97 static DEFINE_MUTEX(cpufreq_governor_mutex);
  98
  99 bool have_governor_per_policy(void)
 100 {
 101         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 102 }
 103 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 104
 105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 106 {
 107         if (have_governor_per_policy())
 108                 return &policy->kobj;
 109         else
 110                 return cpufreq_global_kobject;
 111 }
 112 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 113
 114 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 115 {
 116         u64 idle_time;
 117         u64 cur_wall_time;
 118         u64 busy_time;
 119
 120         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 121
 122         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 128
 129         idle_time = cur_wall_time - busy_time;
 130         if (wall)
 131                 *wall = cputime_to_usecs(cur_wall_time);
 132
 133         return cputime_to_usecs(idle_time);
 134 }
 135
 136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 137 {
 138         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 139
 140         if (idle_time == -1ULL)
 141                 return get_cpu_idle_time_jiffy(cpu, wall);
 142         else if (!io_busy)
 143                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 144
 145         return idle_time;
 146 }
 147 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 148
 149 /*
 150  * This is a generic cpufreq init() routine which can be used by cpufreq
 151  * drivers of SMP systems. It will do following:
 152  * - validate & show freq table passed
 153  * - set policies transition latency
 154  * - policy->cpus with all possible CPUs
 155  */
 156 int cpufreq_generic_init(struct cpufreq_policy *policy,
 157                 struct cpufreq_frequency_table *table,
 158                 unsigned int transition_latency)
 159 {
 160         int ret;
 161
 162         ret = cpufreq_table_validate_and_show(policy, table);
 163         if (ret) {
 164                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 165                 return ret;
 166         }
 167
 168         policy->cpuinfo.transition_latency = transition_latency;
 169
 170         /*
 171          * The driver only supports the SMP configuartion where all processors
 172          * share the clock and voltage and clock.
 173          */
 174         cpumask_setall(policy->cpus);
 175
 176         return 0;
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 179
 180 unsigned int cpufreq_generic_get(unsigned int cpu)
 181 {
 182         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 183
 184         if (!policy || IS_ERR(policy->clk)) {
 185                 pr_err("%s: No %s associated to cpu: %d\n",
 186                        __func__, policy ? "clk" : "policy", cpu);
 187                 return 0;
 188         }
 189
 190         return clk_get_rate(policy->clk) / 1000;
 191 }
 192 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 193
 194 /* Only for cpufreq core internal use */
 195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 196 {
 197         return per_cpu(cpufreq_cpu_data, cpu);
 198 }
 199
 200 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 201 {
 202         struct cpufreq_policy *policy = NULL;
 203         unsigned long flags;
 204
 205         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 206                 return NULL;
 207
 208         if (!down_read_trylock(&cpufreq_rwsem))
 209                 return NULL;
 210
 211         /* get the cpufreq driver */
 212         read_lock_irqsave(&cpufreq_driver_lock, flags);
 213
 214         if (cpufreq_driver) {
 215                 /* get the CPU */
 216                 policy = per_cpu(cpufreq_cpu_data, cpu);
 217                 if (policy)
 218                         kobject_get(&policy->kobj);
 219         }
 220
 221         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 222
 223         if (!policy)
 224                 up_read(&cpufreq_rwsem);
 225
 226         return policy;
 227 }
 228 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 229
 230 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 231 {
 232         if (cpufreq_disabled())
 233                 return;
 234
 235         kobject_put(&policy->kobj);
 236         up_read(&cpufreq_rwsem);
 237 }
 238 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 239
 240 /*********************************************************************
 241  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 242  *********************************************************************/
 243
 244 /**
 245  * adjust_jiffies - adjust the system "loops_per_jiffy"
 246  *
 247  * This function alters the system "loops_per_jiffy" for the clock
 248  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 249  * systems as each CPU might be scaled differently. So, use the arch
 250  * per-CPU loops_per_jiffy value wherever possible.
 251  */
 252 #ifndef CONFIG_SMP
 253 static unsigned long l_p_j_ref;
 254 static unsigned int l_p_j_ref_freq;
 255
 256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 257 {
 258         if (ci->flags & CPUFREQ_CONST_LOOPS)
 259                 return;
 260
 261         if (!l_p_j_ref_freq) {
 262                 l_p_j_ref = loops_per_jiffy;
 263                 l_p_j_ref_freq = ci->old;
 264                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 265                          l_p_j_ref, l_p_j_ref_freq);
 266         }
 267         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 268                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 269                                                                 ci->new);
 270                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 271                          loops_per_jiffy, ci->new);
 272         }
 273 }
 274 #else
 275 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 276 {
 277         return;
 278 }
 279 #endif
 280
 281 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 282                 struct cpufreq_freqs *freqs, unsigned int state)
 283 {
 284         BUG_ON(irqs_disabled());
 285
 286         if (cpufreq_disabled())
 287                 return;
 288
 289         freqs->flags = cpufreq_driver->flags;
 290         pr_debug("notification %u of frequency transition to %u kHz\n",
 291                  state, freqs->new);
 292
 293         switch (state) {
 294
 295         case CPUFREQ_PRECHANGE:
 296                 /* detect if the driver reported a value as "old frequency"
 297                  * which is not equal to what the cpufreq core thinks is
 298                  * "old frequency".
 299                  */
 300                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 301                         if ((policy) && (policy->cpu == freqs->cpu) &&
 302                             (policy->cur) && (policy->cur != freqs->old)) {
 303                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 304                                          freqs->old, policy->cur);
 305                                 freqs->old = policy->cur;
 306                         }
 307                 }
 308                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 309                                 CPUFREQ_PRECHANGE, freqs);
 310                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 311                 break;
 312
 313         case CPUFREQ_POSTCHANGE:
 314                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 315                 pr_debug("FREQ: %lu - CPU: %lu\n",
 316                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 317                 trace_cpu_frequency(freqs->new, freqs->cpu);
 318                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 319                                 CPUFREQ_POSTCHANGE, freqs);
 320                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 321                         policy->cur = freqs->new;
 322                 break;
 323         }
 324 }
 325
 326 /**
 327  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 328  * on frequency transition.
 329  *
 330  * This function calls the transition notifiers and the "adjust_jiffies"
 331  * function. It is called twice on all CPU frequency changes that have
 332  * external effects.
 333  */
 334 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 335                 struct cpufreq_freqs *freqs, unsigned int state)
 336 {
 337         for_each_cpu(freqs->cpu, policy->cpus)
 338                 __cpufreq_notify_transition(policy, freqs, state);
 339 }
 340
 341 /* Do post notifications when there are chances that transition has failed */
 342 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 343                 struct cpufreq_freqs *freqs, int transition_failed)
 344 {
 345         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 346         if (!transition_failed)
 347                 return;
 348
 349         swap(freqs->old, freqs->new);
 350         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 352 }
 353
 354 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 355                 struct cpufreq_freqs *freqs)
 356 {
 357
 358         /*
 359          * Catch double invocations of _begin() which lead to self-deadlock.
 360          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 361          * doesn't invoke _begin() on their behalf, and hence the chances of
 362          * double invocations are very low. Moreover, there are scenarios
 363          * where these checks can emit false-positive warnings in these
 364          * drivers; so we avoid that by skipping them altogether.
 365          */
 366         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 367                                 && current == policy->transition_task);
 368
 369 wait:
 370         wait_event(policy->transition_wait, !policy->transition_ongoing);
 371
 372         spin_lock(&policy->transition_lock);
 373
 374         if (unlikely(policy->transition_ongoing)) {
 375                 spin_unlock(&policy->transition_lock);
 376                 goto wait;
 377         }
 378
 379         policy->transition_ongoing = true;
 380         policy->transition_task = current;
 381
 382         spin_unlock(&policy->transition_lock);
 383
 384         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 385 }
 386 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 387
 388 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 389                 struct cpufreq_freqs *freqs, int transition_failed)
 390 {
 391         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 392                 return;
 393
 394         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 395
 396         policy->transition_ongoing = false;
 397         policy->transition_task = NULL;
 398
 399         wake_up(&policy->transition_wait);
 400 }
 401 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 402
 403
 404 /*********************************************************************
 405  *                          SYSFS INTERFACE                          *
 406  *********************************************************************/
 407 static ssize_t show_boost(struct kobject *kobj,
 408                                  struct attribute *attr, char *buf)
 409 {
 410         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 411 }
 412
 413 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 414                                   const char *buf, size_t count)
 415 {
 416         int ret, enable;
 417
 418         ret = sscanf(buf, "%d", &enable);
 419         if (ret != 1 || enable < 0 || enable > 1)
 420                 return -EINVAL;
 421
 422         if (cpufreq_boost_trigger_state(enable)) {
 423                 pr_err("%s: Cannot %s BOOST!\n",
 424                        __func__, enable ? "enable" : "disable");
 425                 return -EINVAL;
 426         }
 427
 428         pr_debug("%s: cpufreq BOOST %s\n",
 429                  __func__, enable ? "enabled" : "disabled");
 430
 431         return count;
 432 }
 433 define_one_global_rw(boost);
 434
 435 static struct cpufreq_governor *__find_governor(const char *str_governor)
 436 {
 437         struct cpufreq_governor *t;
 438
 439         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 440                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 441                         return t;
 442
 443         return NULL;
 444 }
 445
 446 /**
 447  * cpufreq_parse_governor - parse a governor string
 448  */
 449 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 450                                 struct cpufreq_governor **governor)
 451 {
 452         int err = -EINVAL;
 453
 454         if (!cpufreq_driver)
 455                 goto out;
 456
 457         if (cpufreq_driver->setpolicy) {
 458                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 459                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 460                         err = 0;
 461                 } else if (!strncasecmp(str_governor, "powersave",
 462                                                 CPUFREQ_NAME_LEN)) {
 463                         *policy = CPUFREQ_POLICY_POWERSAVE;
 464                         err = 0;
 465                 }
 466         } else if (has_target()) {
 467                 struct cpufreq_governor *t;
 468
 469                 mutex_lock(&cpufreq_governor_mutex);
 470
 471                 t = __find_governor(str_governor);
 472
 473                 if (t == NULL) {
 474                         int ret;
 475
 476                         mutex_unlock(&cpufreq_governor_mutex);
 477                         ret = request_module("cpufreq_%s", str_governor);
 478                         mutex_lock(&cpufreq_governor_mutex);
 479
 480                         if (ret == 0)
 481                                 t = __find_governor(str_governor);
 482                 }
 483
 484                 if (t != NULL) {
 485                         *governor = t;
 486                         err = 0;
 487                 }
 488
 489                 mutex_unlock(&cpufreq_governor_mutex);
 490         }
 491 out:
 492         return err;
 493 }
 494
 495 /**
 496  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 497  * print out cpufreq information
 498  *
 499  * Write out information from cpufreq_driver->policy[cpu]; object must be
 500  * "unsigned int".
 501  */
 502
 503 #define show_one(file_name, object)                     \
 504 static ssize_t show_##file_name                         \
 505 (struct cpufreq_policy *policy, char *buf)              \
 506 {                                                       \
 507         return sprintf(buf, "%u\n", policy->object);    \
 508 }
 509
 510 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 511 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 512 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 513 show_one(scaling_min_freq, min);
 514 show_one(scaling_max_freq, max);
 515
 516 static ssize_t show_scaling_cur_freq(
 517         struct cpufreq_policy *policy, char *buf)
 518 {
 519         ssize_t ret;
 520
 521         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 522                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 523         else
 524                 ret = sprintf(buf, "%u\n", policy->cur);
 525         return ret;
 526 }
 527
 528 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 529                                 struct cpufreq_policy *new_policy);
 530
 531 /**
 532  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 533  */
 534 #define store_one(file_name, object)                    \
 535 static ssize_t store_##file_name                                        \
 536 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 537 {                                                                       \
 538         int ret;                                                        \
 539         struct cpufreq_policy new_policy;                               \
 540                                                                         \
 541         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 542         if (ret)                                                        \
 543                 return -EINVAL;                                         \
 544                                                                         \
 545         ret = sscanf(buf, "%u", &new_policy.object);                    \
 546         if (ret != 1)                                                   \
 547                 return -EINVAL;                                         \
 548                                                                         \
 549         ret = cpufreq_set_policy(policy, &new_policy);          \
 550         policy->user_policy.object = policy->object;                    \
 551                                                                         \
 552         return ret ? ret : count;                                       \
 553 }
 554
 555 store_one(scaling_min_freq, min);
 556 store_one(scaling_max_freq, max);
 557
 558 /**
 559  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 560  */
 561 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 562                                         char *buf)
 563 {
 564         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 565         if (!cur_freq)
 566                 return sprintf(buf, "<unknown>");
 567         return sprintf(buf, "%u\n", cur_freq);
 568 }
 569
 570 /**
 571  * show_scaling_governor - show the current policy for the specified CPU
 572  */
 573 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 574 {
 575         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 576                 return sprintf(buf, "powersave\n");
 577         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 578                 return sprintf(buf, "performance\n");
 579         else if (policy->governor)
 580                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 581                                 policy->governor->name);
 582         return -EINVAL;
 583 }
 584
 585 /**
 586  * store_scaling_governor - store policy for the specified CPU
 587  */
 588 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 589                                         const char *buf, size_t count)
 590 {
 591         int ret;
 592         char    str_governor[16];
 593         struct cpufreq_policy new_policy;
 594
 595         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 596         if (ret)
 597                 return ret;
 598
 599         ret = sscanf(buf, "%15s", str_governor);
 600         if (ret != 1)
 601                 return -EINVAL;
 602
 603         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 604                                                 &new_policy.governor))
 605                 return -EINVAL;
 606
 607         ret = cpufreq_set_policy(policy, &new_policy);
 608
 609         policy->user_policy.policy = policy->policy;
 610         policy->user_policy.governor = policy->governor;
 611
 612         if (ret)
 613                 return ret;
 614         else
 615                 return count;
 616 }
 617
 618 /**
 619  * show_scaling_driver - show the cpufreq driver currently loaded
 620  */
 621 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 622 {
 623         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 624 }
 625
 626 /**
 627  * show_scaling_available_governors - show the available CPUfreq governors
 628  */
 629 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 630                                                 char *buf)
 631 {
 632         ssize_t i = 0;
 633         struct cpufreq_governor *t;
 634
 635         if (!has_target()) {
 636                 i += sprintf(buf, "performance powersave");
 637                 goto out;
 638         }
 639
 640         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 641                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 642                     - (CPUFREQ_NAME_LEN + 2)))
 643                         goto out;
 644                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 645         }
 646 out:
 647         i += sprintf(&buf[i], "\n");
 648         return i;
 649 }
 650
 651 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 652 {
 653         ssize_t i = 0;
 654         unsigned int cpu;
 655
 656         for_each_cpu(cpu, mask) {
 657                 if (i)
 658                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 659                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 660                 if (i >= (PAGE_SIZE - 5))
 661                         break;
 662         }
 663         i += sprintf(&buf[i], "\n");
 664         return i;
 665 }
 666 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 667
 668 /**
 669  * show_related_cpus - show the CPUs affected by each transition even if
 670  * hw coordination is in use
 671  */
 672 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 673 {
 674         return cpufreq_show_cpus(policy->related_cpus, buf);
 675 }
 676
 677 /**
 678  * show_affected_cpus - show the CPUs affected by each transition
 679  */
 680 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 681 {
 682         return cpufreq_show_cpus(policy->cpus, buf);
 683 }
 684
 685 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 686                                         const char *buf, size_t count)
 687 {
 688         unsigned int freq = 0;
 689         unsigned int ret;
 690
 691         if (!policy->governor || !policy->governor->store_setspeed)
 692                 return -EINVAL;
 693
 694         ret = sscanf(buf, "%u", &freq);
 695         if (ret != 1)
 696                 return -EINVAL;
 697
 698         policy->governor->store_setspeed(policy, freq);
 699
 700         return count;
 701 }
 702
 703 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 704 {
 705         if (!policy->governor || !policy->governor->show_setspeed)
 706                 return sprintf(buf, "<unsupported>\n");
 707
 708         return policy->governor->show_setspeed(policy, buf);
 709 }
 710
 711 /**
 712  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 713  */
 714 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 715 {
 716         unsigned int limit;
 717         int ret;
 718         if (cpufreq_driver->bios_limit) {
 719                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 720                 if (!ret)
 721                         return sprintf(buf, "%u\n", limit);
 722         }
 723         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 724 }
 725
 726 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 727 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 728 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 729 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 730 cpufreq_freq_attr_ro(scaling_available_governors);
 731 cpufreq_freq_attr_ro(scaling_driver);
 732 cpufreq_freq_attr_ro(scaling_cur_freq);
 733 cpufreq_freq_attr_ro(bios_limit);
 734 cpufreq_freq_attr_ro(related_cpus);
 735 cpufreq_freq_attr_ro(affected_cpus);
 736 cpufreq_freq_attr_rw(scaling_min_freq);
 737 cpufreq_freq_attr_rw(scaling_max_freq);
 738 cpufreq_freq_attr_rw(scaling_governor);
 739 cpufreq_freq_attr_rw(scaling_setspeed);
 740
 741 static struct attribute *default_attrs[] = {
 742         &cpuinfo_min_freq.attr,
 743         &cpuinfo_max_freq.attr,
 744         &cpuinfo_transition_latency.attr,
 745         &scaling_min_freq.attr,
 746         &scaling_max_freq.attr,
 747         &affected_cpus.attr,
 748         &related_cpus.attr,
 749         &scaling_governor.attr,
 750         &scaling_driver.attr,
 751         &scaling_available_governors.attr,
 752         &scaling_setspeed.attr,
 753         NULL
 754 };
 755
 756 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 757 #define to_attr(a) container_of(a, struct freq_attr, attr)
 758
 759 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 760 {
 761         struct cpufreq_policy *policy = to_policy(kobj);
 762         struct freq_attr *fattr = to_attr(attr);
 763         ssize_t ret;
 764
 765         if (!down_read_trylock(&cpufreq_rwsem))
 766                 return -EINVAL;
 767
 768         down_read(&policy->rwsem);
 769
 770         if (fattr->show)
 771                 ret = fattr->show(policy, buf);
 772         else
 773                 ret = -EIO;
 774
 775         up_read(&policy->rwsem);
 776         up_read(&cpufreq_rwsem);
 777
 778         return ret;
 779 }
 780
 781 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 782                      const char *buf, size_t count)
 783 {
 784         struct cpufreq_policy *policy = to_policy(kobj);
 785         struct freq_attr *fattr = to_attr(attr);
 786         ssize_t ret = -EINVAL;
 787
 788         get_online_cpus();
 789
 790         if (!cpu_online(policy->cpu))
 791                 goto unlock;
 792
 793         if (!down_read_trylock(&cpufreq_rwsem))
 794                 goto unlock;
 795
 796         down_write(&policy->rwsem);
 797
 798         if (fattr->store)
 799                 ret = fattr->store(policy, buf, count);
 800         else
 801                 ret = -EIO;
 802
 803         up_write(&policy->rwsem);
 804
 805         up_read(&cpufreq_rwsem);
 806 unlock:
 807         put_online_cpus();
 808
 809         return ret;
 810 }
 811
 812 static void cpufreq_sysfs_release(struct kobject *kobj)
 813 {
 814         struct cpufreq_policy *policy = to_policy(kobj);
 815         pr_debug("last reference is dropped\n");
 816         complete(&policy->kobj_unregister);
 817 }
 818
 819 static const struct sysfs_ops sysfs_ops = {
 820         .show   = show,
 821         .store  = store,
 822 };
 823
 824 static struct kobj_type ktype_cpufreq = {
 825         .sysfs_ops      = &sysfs_ops,
 826         .default_attrs  = default_attrs,
 827         .release        = cpufreq_sysfs_release,
 828 };
 829
 830 struct kobject *cpufreq_global_kobject;
 831 EXPORT_SYMBOL(cpufreq_global_kobject);
 832
 833 static int cpufreq_global_kobject_usage;
 834
 835 int cpufreq_get_global_kobject(void)
 836 {
 837         if (!cpufreq_global_kobject_usage++)
 838                 return kobject_add(cpufreq_global_kobject,
 839                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 840
 841         return 0;
 842 }
 843 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 844
 845 void cpufreq_put_global_kobject(void)
 846 {
 847         if (!--cpufreq_global_kobject_usage)
 848                 kobject_del(cpufreq_global_kobject);
 849 }
 850 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 851
 852 int cpufreq_sysfs_create_file(const struct attribute *attr)
 853 {
 854         int ret = cpufreq_get_global_kobject();
 855
 856         if (!ret) {
 857                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 858                 if (ret)
 859                         cpufreq_put_global_kobject();
 860         }
 861
 862         return ret;
 863 }
 864 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 865
 866 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 867 {
 868         sysfs_remove_file(cpufreq_global_kobject, attr);
 869         cpufreq_put_global_kobject();
 870 }
 871 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 872
 873 /* symlink affected CPUs */
 874 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 875 {
 876         unsigned int j;
 877         int ret = 0;
 878
 879         for_each_cpu(j, policy->cpus) {
 880                 struct device *cpu_dev;
 881
 882                 if (j == policy->cpu)
 883                         continue;
 884
 885                 pr_debug("Adding link for CPU: %u\n", j);
 886                 cpu_dev = get_cpu_device(j);
 887                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 888                                         "cpufreq");
 889                 if (ret)
 890                         break;
 891         }
 892         return ret;
 893 }
 894
 895 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 896                                      struct device *dev)
 897 {
 898         struct freq_attr **drv_attr;
 899         int ret = 0;
 900
 901         /* prepare interface data */
 902         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 903                                    &dev->kobj, "cpufreq");
 904         if (ret)
 905                 return ret;
 906
 907         /* set up files for this cpu device */
 908         drv_attr = cpufreq_driver->attr;
 909         while ((drv_attr) && (*drv_attr)) {
 910                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 911                 if (ret)
 912                         goto err_out_kobj_put;
 913                 drv_attr++;
 914         }
 915         if (cpufreq_driver->get) {
 916                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 917                 if (ret)
 918                         goto err_out_kobj_put;
 919         }
 920
 921         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 922         if (ret)
 923                 goto err_out_kobj_put;
 924
 925         if (cpufreq_driver->bios_limit) {
 926                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 927                 if (ret)
 928                         goto err_out_kobj_put;
 929         }
 930
 931         ret = cpufreq_add_dev_symlink(policy);
 932         if (ret)
 933                 goto err_out_kobj_put;
 934
 935         return ret;
 936
 937 err_out_kobj_put:
 938         kobject_put(&policy->kobj);
 939         wait_for_completion(&policy->kobj_unregister);
 940         return ret;
 941 }
 942
 943 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 944 {
 945         struct cpufreq_governor *gov = NULL;
 946         struct cpufreq_policy new_policy;
 947         int ret = 0;
 948
 949         memcpy(&new_policy, policy, sizeof(*policy));
 950
 951         /* Update governor of new_policy to the governor used before hotplug */
 952         gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 953         if (gov)
 954                 pr_debug("Restoring governor %s for cpu %d\n",
 955                                 policy->governor->name, policy->cpu);
 956         else
 957                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 958
 959         new_policy.governor = gov;
 960
 961         /* Use the default policy if its valid. */
 962         if (cpufreq_driver->setpolicy)
 963                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 964
 965         /* set default policy */
 966         ret = cpufreq_set_policy(policy, &new_policy);
 967         if (ret) {
 968                 pr_debug("setting policy failed\n");
 969                 if (cpufreq_driver->exit)
 970                         cpufreq_driver->exit(policy);
 971         }
 972 }
 973
 974 #ifdef CONFIG_HOTPLUG_CPU
 975 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 976                                   unsigned int cpu, struct device *dev)
 977 {
 978         int ret = 0;
 979         unsigned long flags;
 980
 981         if (has_target()) {
 982                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 983                 if (ret) {
 984                         pr_err("%s: Failed to stop governor\n", __func__);
 985                         return ret;
 986                 }
 987         }
 988
 989         down_write(&policy->rwsem);
 990
 991         write_lock_irqsave(&cpufreq_driver_lock, flags);
 992
 993         cpumask_set_cpu(cpu, policy->cpus);
 994         per_cpu(cpufreq_cpu_data, cpu) = policy;
 995         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 996
 997         up_write(&policy->rwsem);
 998
 999         if (has_target()) {
1000                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1001                 if (!ret)
1002                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1003
1004                 if (ret) {
1005                         pr_err("%s: Failed to start governor\n", __func__);
1006                         return ret;
1007                 }
1008         }
1009
1010         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
1011 }
1012 #endif
1013
1014 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1015 {
1016         struct cpufreq_policy *policy;
1017         unsigned long flags;
1018
1019         read_lock_irqsave(&cpufreq_driver_lock, flags);
1020
1021         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1022
1023         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1024
1025         if (policy)
1026                 policy->governor = NULL;
1027
1028         return policy;
1029 }
1030
1031 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1032 {
1033         struct cpufreq_policy *policy;
1034
1035         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1036         if (!policy)
1037                 return NULL;
1038
1039         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1040                 goto err_free_policy;
1041
1042         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1043                 goto err_free_cpumask;
1044
1045         INIT_LIST_HEAD(&policy->policy_list);
1046         init_rwsem(&policy->rwsem);
1047         spin_lock_init(&policy->transition_lock);
1048         init_waitqueue_head(&policy->transition_wait);
1049
1050         return policy;
1051
1052 err_free_cpumask:
1053         free_cpumask_var(policy->cpus);
1054 err_free_policy:
1055         kfree(policy);
1056
1057         return NULL;
1058 }
1059
1060 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1061 {
1062         struct kobject *kobj;
1063         struct completion *cmp;
1064
1065         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1066                         CPUFREQ_REMOVE_POLICY, policy);
1067
1068         down_read(&policy->rwsem);
1069         kobj = &policy->kobj;
1070         cmp = &policy->kobj_unregister;
1071         up_read(&policy->rwsem);
1072         kobject_put(kobj);
1073
1074         /*
1075          * We need to make sure that the underlying kobj is
1076          * actually not referenced anymore by anybody before we
1077          * proceed with unloading.
1078          */
1079         pr_debug("waiting for dropping of refcount\n");
1080         wait_for_completion(cmp);
1081         pr_debug("wait complete\n");
1082 }
1083
1084 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1085 {
1086         free_cpumask_var(policy->related_cpus);
1087         free_cpumask_var(policy->cpus);
1088         kfree(policy);
1089 }
1090
1091 static int update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu,
1092                              struct device *cpu_dev)
1093 {
1094         int ret;
1095
1096         if (WARN_ON(cpu == policy->cpu))
1097                 return 0;
1098
1099         /* Move kobject to the new policy->cpu */
1100         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1101         if (ret) {
1102                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1103                 return ret;
1104         }
1105
1106         down_write(&policy->rwsem);
1107
1108         policy->last_cpu = policy->cpu;
1109         policy->cpu = cpu;
1110
1111         up_write(&policy->rwsem);
1112
1113         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1114                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1115
1116         return 0;
1117 }
1118
1119 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1120 {
1121         unsigned int j, cpu = dev->id;
1122         int ret = -ENOMEM;
1123         struct cpufreq_policy *policy;
1124         unsigned long flags;
1125         bool recover_policy = cpufreq_suspended;
1126 #ifdef CONFIG_HOTPLUG_CPU
1127         struct cpufreq_policy *tpolicy;
1128 #endif
1129
1130         if (cpu_is_offline(cpu))
1131                 return 0;
1132
1133         pr_debug("adding CPU %u\n", cpu);
1134
1135 #ifdef CONFIG_SMP
1136         /* check whether a different CPU already registered this
1137          * CPU because it is in the same boat. */
1138         policy = cpufreq_cpu_get(cpu);
1139         if (unlikely(policy)) {
1140                 cpufreq_cpu_put(policy);
1141                 return 0;
1142         }
1143 #endif
1144
1145         if (!down_read_trylock(&cpufreq_rwsem))
1146                 return 0;
1147
1148 #ifdef CONFIG_HOTPLUG_CPU
1149         /* Check if this cpu was hot-unplugged earlier and has siblings */
1150         read_lock_irqsave(&cpufreq_driver_lock, flags);
1151         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1152                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1153                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1154                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1155                         up_read(&cpufreq_rwsem);
1156                         return ret;
1157                 }
1158         }
1159         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1160 #endif
1161
1162         /*
1163          * Restore the saved policy when doing light-weight init and fall back
1164          * to the full init if that fails.
1165          */
1166         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1167         if (!policy) {
1168                 recover_policy = false;
1169                 policy = cpufreq_policy_alloc();
1170                 if (!policy)
1171                         goto nomem_out;
1172         }
1173
1174         /*
1175          * In the resume path, since we restore a saved policy, the assignment
1176          * to policy->cpu is like an update of the existing policy, rather than
1177          * the creation of a brand new one. So we need to perform this update
1178          * by invoking update_policy_cpu().
1179          */
1180         if (recover_policy && cpu != policy->cpu)
1181                 WARN_ON(update_policy_cpu(policy, cpu, dev));
1182         else
1183                 policy->cpu = cpu;
1184
1185         cpumask_copy(policy->cpus, cpumask_of(cpu));
1186
1187         init_completion(&policy->kobj_unregister);
1188         INIT_WORK(&policy->update, handle_update);
1189
1190         /* call driver. From then on the cpufreq must be able
1191          * to accept all calls to ->verify and ->setpolicy for this CPU
1192          */
1193         ret = cpufreq_driver->init(policy);
1194         if (ret) {
1195                 pr_debug("initialization failed\n");
1196                 goto err_set_policy_cpu;
1197         }
1198
1199         /* related cpus should atleast have policy->cpus */
1200         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1201
1202         /*
1203          * affected cpus must always be the one, which are online. We aren't
1204          * managing offline cpus here.
1205          */
1206         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1207
1208         if (!recover_policy) {
1209                 policy->user_policy.min = policy->min;
1210                 policy->user_policy.max = policy->max;
1211         }
1212
1213         down_write(&policy->rwsem);
1214         write_lock_irqsave(&cpufreq_driver_lock, flags);
1215         for_each_cpu(j, policy->cpus)
1216                 per_cpu(cpufreq_cpu_data, j) = policy;
1217         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1218
1219         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1220                 policy->cur = cpufreq_driver->get(policy->cpu);
1221                 if (!policy->cur) {
1222                         pr_err("%s: ->get() failed\n", __func__);
1223                         goto err_get_freq;
1224                 }
1225         }
1226
1227         /*
1228          * Sometimes boot loaders set CPU frequency to a value outside of
1229          * frequency table present with cpufreq core. In such cases CPU might be
1230          * unstable if it has to run on that frequency for long duration of time
1231          * and so its better to set it to a frequency which is specified in
1232          * freq-table. This also makes cpufreq stats inconsistent as
1233          * cpufreq-stats would fail to register because current frequency of CPU
1234          * isn't found in freq-table.
1235          *
1236          * Because we don't want this change to effect boot process badly, we go
1237          * for the next freq which is >= policy->cur ('cur' must be set by now,
1238          * otherwise we will end up setting freq to lowest of the table as 'cur'
1239          * is initialized to zero).
1240          *
1241          * We are passing target-freq as "policy->cur - 1" otherwise
1242          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1243          * equal to target-freq.
1244          */
1245         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1246             && has_target()) {
1247                 /* Are we running at unknown frequency ? */
1248                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1249                 if (ret == -EINVAL) {
1250                         /* Warn user and fix it */
1251                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1252                                 __func__, policy->cpu, policy->cur);
1253                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1254                                 CPUFREQ_RELATION_L);
1255
1256                         /*
1257                          * Reaching here after boot in a few seconds may not
1258                          * mean that system will remain stable at "unknown"
1259                          * frequency for longer duration. Hence, a BUG_ON().
1260                          */
1261                         BUG_ON(ret);
1262                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1263                                 __func__, policy->cpu, policy->cur);
1264                 }
1265         }
1266
1267         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1268                                      CPUFREQ_START, policy);
1269
1270         if (!recover_policy) {
1271                 ret = cpufreq_add_dev_interface(policy, dev);
1272                 if (ret)
1273                         goto err_out_unregister;
1274                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1275                                 CPUFREQ_CREATE_POLICY, policy);
1276         }
1277
1278         write_lock_irqsave(&cpufreq_driver_lock, flags);
1279         list_add(&policy->policy_list, &cpufreq_policy_list);
1280         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1281
1282         cpufreq_init_policy(policy);
1283
1284         if (!recover_policy) {
1285                 policy->user_policy.policy = policy->policy;
1286                 policy->user_policy.governor = policy->governor;
1287         }
1288         up_write(&policy->rwsem);
1289
1290         kobject_uevent(&policy->kobj, KOBJ_ADD);
1291         up_read(&cpufreq_rwsem);
1292
1293         pr_debug("initialization complete\n");
1294
1295         return 0;
1296
1297 err_out_unregister:
1298 err_get_freq:
1299         write_lock_irqsave(&cpufreq_driver_lock, flags);
1300         for_each_cpu(j, policy->cpus)
1301                 per_cpu(cpufreq_cpu_data, j) = NULL;
1302         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1303
1304         up_write(&policy->rwsem);
1305
1306         if (cpufreq_driver->exit)
1307                 cpufreq_driver->exit(policy);
1308 err_set_policy_cpu:
1309         if (recover_policy) {
1310                 /* Do not leave stale fallback data behind. */
1311                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1312                 cpufreq_policy_put_kobj(policy);
1313         }
1314         cpufreq_policy_free(policy);
1315
1316 nomem_out:
1317         up_read(&cpufreq_rwsem);
1318
1319         return ret;
1320 }
1321
1322 /**
1323  * cpufreq_add_dev - add a CPU device
1324  *
1325  * Adds the cpufreq interface for a CPU device.
1326  *
1327  * The Oracle says: try running cpufreq registration/unregistration concurrently
1328  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1329  * mess up, but more thorough testing is needed. - Mathieu
1330  */
1331 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1332 {
1333         return __cpufreq_add_dev(dev, sif);
1334 }
1335
1336 static int __cpufreq_remove_dev_prepare(struct device *dev,
1337                                         struct subsys_interface *sif)
1338 {
1339         unsigned int cpu = dev->id, cpus;
1340         int ret;
1341         unsigned long flags;
1342         struct cpufreq_policy *policy;
1343
1344         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1345
1346         write_lock_irqsave(&cpufreq_driver_lock, flags);
1347
1348         policy = per_cpu(cpufreq_cpu_data, cpu);
1349
1350         /* Save the policy somewhere when doing a light-weight tear-down */
1351         if (cpufreq_suspended)
1352                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1353
1354         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1355
1356         if (!policy) {
1357                 pr_debug("%s: No cpu_data found\n", __func__);
1358                 return -EINVAL;
1359         }
1360
1361         if (has_target()) {
1362                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1363                 if (ret) {
1364                         pr_err("%s: Failed to stop governor\n", __func__);
1365                         return ret;
1366                 }
1367         }
1368
1369         if (!cpufreq_driver->setpolicy)
1370                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1371                         policy->governor->name, CPUFREQ_NAME_LEN);
1372
1373         down_read(&policy->rwsem);
1374         cpus = cpumask_weight(policy->cpus);
1375         up_read(&policy->rwsem);
1376
1377         if (cpu != policy->cpu) {
1378                 sysfs_remove_link(&dev->kobj, "cpufreq");
1379         } else if (cpus > 1) {
1380                 /* Nominate new CPU */
1381                 int new_cpu = cpumask_any_but(policy->cpus, cpu);
1382                 struct device *cpu_dev = get_cpu_device(new_cpu);
1383
1384                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1385                 ret = update_policy_cpu(policy, new_cpu, cpu_dev);
1386                 if (ret) {
1387                         if (sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1388                                               "cpufreq"))
1389                                 pr_err("%s: Failed to restore kobj link to cpu:%d\n",
1390                                        __func__, cpu_dev->id);
1391                         return ret;
1392                 }
1393
1394                 if (!cpufreq_suspended)
1395                         pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1396                                  __func__, new_cpu, cpu);
1397         } else if (cpufreq_driver->stop_cpu) {
1398                 cpufreq_driver->stop_cpu(policy);
1399         }
1400
1401         return 0;
1402 }
1403
1404 static int __cpufreq_remove_dev_finish(struct device *dev,
1405                                        struct subsys_interface *sif)
1406 {
1407         unsigned int cpu = dev->id, cpus;
1408         int ret;
1409         unsigned long flags;
1410         struct cpufreq_policy *policy;
1411
1412         read_lock_irqsave(&cpufreq_driver_lock, flags);
1413         policy = per_cpu(cpufreq_cpu_data, cpu);
1414         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1415
1416         if (!policy) {
1417                 pr_debug("%s: No cpu_data found\n", __func__);
1418                 return -EINVAL;
1419         }
1420
1421         down_write(&policy->rwsem);
1422         cpus = cpumask_weight(policy->cpus);
1423
1424         if (cpus > 1)
1425                 cpumask_clear_cpu(cpu, policy->cpus);
1426         up_write(&policy->rwsem);
1427
1428         /* If cpu is last user of policy, free policy */
1429         if (cpus == 1) {
1430                 if (has_target()) {
1431                         ret = __cpufreq_governor(policy,
1432                                         CPUFREQ_GOV_POLICY_EXIT);
1433                         if (ret) {
1434                                 pr_err("%s: Failed to exit governor\n",
1435                                        __func__);
1436                                 return ret;
1437                         }
1438                 }
1439
1440                 if (!cpufreq_suspended)
1441                         cpufreq_policy_put_kobj(policy);
1442
1443                 /*
1444                  * Perform the ->exit() even during light-weight tear-down,
1445                  * since this is a core component, and is essential for the
1446                  * subsequent light-weight ->init() to succeed.
1447                  */
1448                 if (cpufreq_driver->exit)
1449                         cpufreq_driver->exit(policy);
1450
1451                 /* Remove policy from list of active policies */
1452                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1453                 list_del(&policy->policy_list);
1454                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1455
1456                 if (!cpufreq_suspended)
1457                         cpufreq_policy_free(policy);
1458         } else if (has_target()) {
1459                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1460                 if (!ret)
1461                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1462
1463                 if (ret) {
1464                         pr_err("%s: Failed to start governor\n", __func__);
1465                         return ret;
1466                 }
1467         }
1468
1469         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1470         return 0;
1471 }
1472
1473 /**
1474  * cpufreq_remove_dev - remove a CPU device
1475  *
1476  * Removes the cpufreq interface for a CPU device.
1477  */
1478 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1479 {
1480         unsigned int cpu = dev->id;
1481         int ret;
1482
1483         if (cpu_is_offline(cpu))
1484                 return 0;
1485
1486         ret = __cpufreq_remove_dev_prepare(dev, sif);
1487
1488         if (!ret)
1489                 ret = __cpufreq_remove_dev_finish(dev, sif);
1490
1491         return ret;
1492 }
1493
1494 static void handle_update(struct work_struct *work)
1495 {
1496         struct cpufreq_policy *policy =
1497                 container_of(work, struct cpufreq_policy, update);
1498         unsigned int cpu = policy->cpu;
1499         pr_debug("handle_update for cpu %u called\n", cpu);
1500         cpufreq_update_policy(cpu);
1501 }
1502
1503 /**
1504  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1505  *      in deep trouble.
1506  *      @cpu: cpu number
1507  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1508  *      @new_freq: CPU frequency the CPU actually runs at
1509  *
1510  *      We adjust to current frequency first, and need to clean up later.
1511  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1512  */
1513 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1514                                 unsigned int new_freq)
1515 {
1516         struct cpufreq_policy *policy;
1517         struct cpufreq_freqs freqs;
1518         unsigned long flags;
1519
1520         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1521                  old_freq, new_freq);
1522
1523         freqs.old = old_freq;
1524         freqs.new = new_freq;
1525
1526         read_lock_irqsave(&cpufreq_driver_lock, flags);
1527         policy = per_cpu(cpufreq_cpu_data, cpu);
1528         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1529
1530         cpufreq_freq_transition_begin(policy, &freqs);
1531         cpufreq_freq_transition_end(policy, &freqs, 0);
1532 }
1533
1534 /**
1535  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1536  * @cpu: CPU number
1537  *
1538  * This is the last known freq, without actually getting it from the driver.
1539  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1540  */
1541 unsigned int cpufreq_quick_get(unsigned int cpu)
1542 {
1543         struct cpufreq_policy *policy;
1544         unsigned int ret_freq = 0;
1545
1546         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1547                 return cpufreq_driver->get(cpu);
1548
1549         policy = cpufreq_cpu_get(cpu);
1550         if (policy) {
1551                 ret_freq = policy->cur;
1552                 cpufreq_cpu_put(policy);
1553         }
1554
1555         return ret_freq;
1556 }
1557 EXPORT_SYMBOL(cpufreq_quick_get);
1558
1559 /**
1560  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1561  * @cpu: CPU number
1562  *
1563  * Just return the max possible frequency for a given CPU.
1564  */
1565 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1566 {
1567         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1568         unsigned int ret_freq = 0;
1569
1570         if (policy) {
1571                 ret_freq = policy->max;
1572                 cpufreq_cpu_put(policy);
1573         }
1574
1575         return ret_freq;
1576 }
1577 EXPORT_SYMBOL(cpufreq_quick_get_max);
1578
1579 static unsigned int __cpufreq_get(unsigned int cpu)
1580 {
1581         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1582         unsigned int ret_freq = 0;
1583
1584         if (!cpufreq_driver->get)
1585                 return ret_freq;
1586
1587         ret_freq = cpufreq_driver->get(cpu);
1588
1589         if (ret_freq && policy->cur &&
1590                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1591                 /* verify no discrepancy between actual and
1592                                         saved value exists */
1593                 if (unlikely(ret_freq != policy->cur)) {
1594                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1595                         schedule_work(&policy->update);
1596                 }
1597         }
1598
1599         return ret_freq;
1600 }
1601
1602 /**
1603  * cpufreq_get - get the current CPU frequency (in kHz)
1604  * @cpu: CPU number
1605  *
1606  * Get the CPU current (static) CPU frequency
1607  */
1608 unsigned int cpufreq_get(unsigned int cpu)
1609 {
1610         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1611         unsigned int ret_freq = 0;
1612
1613         if (policy) {
1614                 down_read(&policy->rwsem);
1615                 ret_freq = __cpufreq_get(cpu);
1616                 up_read(&policy->rwsem);
1617
1618                 cpufreq_cpu_put(policy);
1619         }
1620
1621         return ret_freq;
1622 }
1623 EXPORT_SYMBOL(cpufreq_get);
1624
1625 static struct subsys_interface cpufreq_interface = {
1626         .name           = "cpufreq",
1627         .subsys         = &cpu_subsys,
1628         .add_dev        = cpufreq_add_dev,
1629         .remove_dev     = cpufreq_remove_dev,
1630 };
1631
1632 /*
1633  * In case platform wants some specific frequency to be configured
1634  * during suspend..
1635  */
1636 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1637 {
1638         int ret;
1639
1640         if (!policy->suspend_freq) {
1641                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1642                 return -EINVAL;
1643         }
1644
1645         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1646                         policy->suspend_freq);
1647
1648         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1649                         CPUFREQ_RELATION_H);
1650         if (ret)
1651                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1652                                 __func__, policy->suspend_freq, ret);
1653
1654         return ret;
1655 }
1656 EXPORT_SYMBOL(cpufreq_generic_suspend);
1657
1658 /**
1659  * cpufreq_suspend() - Suspend CPUFreq governors
1660  *
1661  * Called during system wide Suspend/Hibernate cycles for suspending governors
1662  * as some platforms can't change frequency after this point in suspend cycle.
1663  * Because some of the devices (like: i2c, regulators, etc) they use for
1664  * changing frequency are suspended quickly after this point.
1665  */
1666 void cpufreq_suspend(void)
1667 {
1668         struct cpufreq_policy *policy;
1669
1670         if (!cpufreq_driver)
1671                 return;
1672
1673         if (!has_target())
1674                 goto suspend;
1675
1676         pr_debug("%s: Suspending Governors\n", __func__);
1677
1678         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1679                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1680                         pr_err("%s: Failed to stop governor for policy: %p\n",
1681                                 __func__, policy);
1682                 else if (cpufreq_driver->suspend
1683                     && cpufreq_driver->suspend(policy))
1684                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1685                                 policy);
1686         }
1687
1688 suspend:
1689         cpufreq_suspended = true;
1690 }
1691
1692 /**
1693  * cpufreq_resume() - Resume CPUFreq governors
1694  *
1695  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1696  * are suspended with cpufreq_suspend().
1697  */
1698 void cpufreq_resume(void)
1699 {
1700         struct cpufreq_policy *policy;
1701
1702         if (!cpufreq_driver)
1703                 return;
1704
1705         cpufreq_suspended = false;
1706
1707         if (!has_target())
1708                 return;
1709
1710         pr_debug("%s: Resuming Governors\n", __func__);
1711
1712         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1713                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1714                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1715                                 policy);
1716                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1717                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1718                         pr_err("%s: Failed to start governor for policy: %p\n",
1719                                 __func__, policy);
1720
1721                 /*
1722                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1723                  * policy in list. It will verify that the current freq is in
1724                  * sync with what we believe it to be.
1725                  */
1726                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1727                         schedule_work(&policy->update);
1728         }
1729 }
1730
1731 /**
1732  *      cpufreq_get_current_driver - return current driver's name
1733  *
1734  *      Return the name string of the currently loaded cpufreq driver
1735  *      or NULL, if none.
1736  */
1737 const char *cpufreq_get_current_driver(void)
1738 {
1739         if (cpufreq_driver)
1740                 return cpufreq_driver->name;
1741
1742         return NULL;
1743 }
1744 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1745
1746 /**
1747  *      cpufreq_get_driver_data - return current driver data
1748  *
1749  *      Return the private data of the currently loaded cpufreq
1750  *      driver, or NULL if no cpufreq driver is loaded.
1751  */
1752 void *cpufreq_get_driver_data(void)
1753 {
1754         if (cpufreq_driver)
1755                 return cpufreq_driver->driver_data;
1756
1757         return NULL;
1758 }
1759 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1760
1761 /*********************************************************************
1762  *                     NOTIFIER LISTS INTERFACE                      *
1763  *********************************************************************/
1764
1765 /**
1766  *      cpufreq_register_notifier - register a driver with cpufreq
1767  *      @nb: notifier function to register
1768  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1769  *
1770  *      Add a driver to one of two lists: either a list of drivers that
1771  *      are notified about clock rate changes (once before and once after
1772  *      the transition), or a list of drivers that are notified about
1773  *      changes in cpufreq policy.
1774  *
1775  *      This function may sleep, and has the same return conditions as
1776  *      blocking_notifier_chain_register.
1777  */
1778 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1779 {
1780         int ret;
1781
1782         if (cpufreq_disabled())
1783                 return -EINVAL;
1784
1785         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1786
1787         switch (list) {
1788         case CPUFREQ_TRANSITION_NOTIFIER:
1789                 ret = srcu_notifier_chain_register(
1790                                 &cpufreq_transition_notifier_list, nb);
1791                 break;
1792         case CPUFREQ_POLICY_NOTIFIER:
1793                 ret = blocking_notifier_chain_register(
1794                                 &cpufreq_policy_notifier_list, nb);
1795                 break;
1796         default:
1797                 ret = -EINVAL;
1798         }
1799
1800         return ret;
1801 }
1802 EXPORT_SYMBOL(cpufreq_register_notifier);
1803
1804 /**
1805  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1806  *      @nb: notifier block to be unregistered
1807  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1808  *
1809  *      Remove a driver from the CPU frequency notifier list.
1810  *
1811  *      This function may sleep, and has the same return conditions as
1812  *      blocking_notifier_chain_unregister.
1813  */
1814 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1815 {
1816         int ret;
1817
1818         if (cpufreq_disabled())
1819                 return -EINVAL;
1820
1821         switch (list) {
1822         case CPUFREQ_TRANSITION_NOTIFIER:
1823                 ret = srcu_notifier_chain_unregister(
1824                                 &cpufreq_transition_notifier_list, nb);
1825                 break;
1826         case CPUFREQ_POLICY_NOTIFIER:
1827                 ret = blocking_notifier_chain_unregister(
1828                                 &cpufreq_policy_notifier_list, nb);
1829                 break;
1830         default:
1831                 ret = -EINVAL;
1832         }
1833
1834         return ret;
1835 }
1836 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1837
1838
1839 /*********************************************************************
1840  *                              GOVERNORS                            *
1841  *********************************************************************/
1842
1843 /* Must set freqs->new to intermediate frequency */
1844 static int __target_intermediate(struct cpufreq_policy *policy,
1845                                  struct cpufreq_freqs *freqs, int index)
1846 {
1847         int ret;
1848
1849         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1850
1851         /* We don't need to switch to intermediate freq */
1852         if (!freqs->new)
1853                 return 0;
1854
1855         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1856                  __func__, policy->cpu, freqs->old, freqs->new);
1857
1858         cpufreq_freq_transition_begin(policy, freqs);
1859         ret = cpufreq_driver->target_intermediate(policy, index);
1860         cpufreq_freq_transition_end(policy, freqs, ret);
1861
1862         if (ret)
1863                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1864                        __func__, ret);
1865
1866         return ret;
1867 }
1868
1869 static int __target_index(struct cpufreq_policy *policy,
1870                           struct cpufreq_frequency_table *freq_table, int index)
1871 {
1872         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1873         unsigned int intermediate_freq = 0;
1874         int retval = -EINVAL;
1875         bool notify;
1876
1877         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1878         if (notify) {
1879                 /* Handle switching to intermediate frequency */
1880                 if (cpufreq_driver->get_intermediate) {
1881                         retval = __target_intermediate(policy, &freqs, index);
1882                         if (retval)
1883                                 return retval;
1884
1885                         intermediate_freq = freqs.new;
1886                         /* Set old freq to intermediate */
1887                         if (intermediate_freq)
1888                                 freqs.old = freqs.new;
1889                 }
1890
1891                 freqs.new = freq_table[index].frequency;
1892                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1893                          __func__, policy->cpu, freqs.old, freqs.new);
1894
1895                 cpufreq_freq_transition_begin(policy, &freqs);
1896         }
1897
1898         retval = cpufreq_driver->target_index(policy, index);
1899         if (retval)
1900                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1901                        retval);
1902
1903         if (notify) {
1904                 cpufreq_freq_transition_end(policy, &freqs, retval);
1905
1906                 /*
1907                  * Failed after setting to intermediate freq? Driver should have
1908                  * reverted back to initial frequency and so should we. Check
1909                  * here for intermediate_freq instead of get_intermediate, in
1910                  * case we have't switched to intermediate freq at all.
1911                  */
1912                 if (unlikely(retval && intermediate_freq)) {
1913                         freqs.old = intermediate_freq;
1914                         freqs.new = policy->restore_freq;
1915                         cpufreq_freq_transition_begin(policy, &freqs);
1916                         cpufreq_freq_transition_end(policy, &freqs, 0);
1917                 }
1918         }
1919
1920         return retval;
1921 }
1922
1923 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1924                             unsigned int target_freq,
1925                             unsigned int relation)
1926 {
1927         unsigned int old_target_freq = target_freq;
1928         int retval = -EINVAL;
1929
1930         if (cpufreq_disabled())
1931                 return -ENODEV;
1932
1933         /* Make sure that target_freq is within supported range */
1934         if (target_freq > policy->max)
1935                 target_freq = policy->max;
1936         if (target_freq < policy->min)
1937                 target_freq = policy->min;
1938
1939         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1940                  policy->cpu, target_freq, relation, old_target_freq);
1941
1942         /*
1943          * This might look like a redundant call as we are checking it again
1944          * after finding index. But it is left intentionally for cases where
1945          * exactly same freq is called again and so we can save on few function
1946          * calls.
1947          */
1948         if (target_freq == policy->cur)
1949                 return 0;
1950
1951         /* Save last value to restore later on errors */
1952         policy->restore_freq = policy->cur;
1953
1954         if (cpufreq_driver->target)
1955                 retval = cpufreq_driver->target(policy, target_freq, relation);
1956         else if (cpufreq_driver->target_index) {
1957                 struct cpufreq_frequency_table *freq_table;
1958                 int index;
1959
1960                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1961                 if (unlikely(!freq_table)) {
1962                         pr_err("%s: Unable to find freq_table\n", __func__);
1963                         goto out;
1964                 }
1965
1966                 retval = cpufreq_frequency_table_target(policy, freq_table,
1967                                 target_freq, relation, &index);
1968                 if (unlikely(retval)) {
1969                         pr_err("%s: Unable to find matching freq\n", __func__);
1970                         goto out;
1971                 }
1972
1973                 if (freq_table[index].frequency == policy->cur) {
1974                         retval = 0;
1975                         goto out;
1976                 }
1977
1978                 retval = __target_index(policy, freq_table, index);
1979         }
1980
1981 out:
1982         return retval;
1983 }
1984 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1985
1986 int cpufreq_driver_target(struct cpufreq_policy *policy,
1987                           unsigned int target_freq,
1988                           unsigned int relation)
1989 {
1990         int ret = -EINVAL;
1991
1992         down_write(&policy->rwsem);
1993
1994         ret = __cpufreq_driver_target(policy, target_freq, relation);
1995
1996         up_write(&policy->rwsem);
1997
1998         return ret;
1999 }
2000 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2001
2002 /*
2003  * when "event" is CPUFREQ_GOV_LIMITS
2004  */
2005
2006 static int __cpufreq_governor(struct cpufreq_policy *policy,
2007                                         unsigned int event)
2008 {
2009         int ret;
2010
2011         /* Only must be defined when default governor is known to have latency
2012            restrictions, like e.g. conservative or ondemand.
2013            That this is the case is already ensured in Kconfig
2014         */
2015 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2016         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2017 #else
2018         struct cpufreq_governor *gov = NULL;
2019 #endif
2020
2021         /* Don't start any governor operations if we are entering suspend */
2022         if (cpufreq_suspended)
2023                 return 0;
2024
2025         if (policy->governor->max_transition_latency &&
2026             policy->cpuinfo.transition_latency >
2027             policy->governor->max_transition_latency) {
2028                 if (!gov)
2029                         return -EINVAL;
2030                 else {
2031                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2032                                 policy->governor->name, gov->name);
2033                         policy->governor = gov;
2034                 }
2035         }
2036
2037         if (event == CPUFREQ_GOV_POLICY_INIT)
2038                 if (!try_module_get(policy->governor->owner))
2039                         return -EINVAL;
2040
2041         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2042                  policy->cpu, event);
2043
2044         mutex_lock(&cpufreq_governor_lock);
2045         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2046             || (!policy->governor_enabled
2047             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2048                 mutex_unlock(&cpufreq_governor_lock);
2049                 return -EBUSY;
2050         }
2051
2052         if (event == CPUFREQ_GOV_STOP)
2053                 policy->governor_enabled = false;
2054         else if (event == CPUFREQ_GOV_START)
2055                 policy->governor_enabled = true;
2056
2057         mutex_unlock(&cpufreq_governor_lock);
2058
2059         ret = policy->governor->governor(policy, event);
2060
2061         if (!ret) {
2062                 if (event == CPUFREQ_GOV_POLICY_INIT)
2063                         policy->governor->initialized++;
2064                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2065                         policy->governor->initialized--;
2066         } else {
2067                 /* Restore original values */
2068                 mutex_lock(&cpufreq_governor_lock);
2069                 if (event == CPUFREQ_GOV_STOP)
2070                         policy->governor_enabled = true;
2071                 else if (event == CPUFREQ_GOV_START)
2072                         policy->governor_enabled = false;
2073                 mutex_unlock(&cpufreq_governor_lock);
2074         }
2075
2076         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2077                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2078                 module_put(policy->governor->owner);
2079
2080         return ret;
2081 }
2082
2083 int cpufreq_register_governor(struct cpufreq_governor *governor)
2084 {
2085         int err;
2086
2087         if (!governor)
2088                 return -EINVAL;
2089
2090         if (cpufreq_disabled())
2091                 return -ENODEV;
2092
2093         mutex_lock(&cpufreq_governor_mutex);
2094
2095         governor->initialized = 0;
2096         err = -EBUSY;
2097         if (__find_governor(governor->name) == NULL) {
2098                 err = 0;
2099                 list_add(&governor->governor_list, &cpufreq_governor_list);
2100         }
2101
2102         mutex_unlock(&cpufreq_governor_mutex);
2103         return err;
2104 }
2105 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2106
2107 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2108 {
2109         int cpu;
2110
2111         if (!governor)
2112                 return;
2113
2114         if (cpufreq_disabled())
2115                 return;
2116
2117         for_each_present_cpu(cpu) {
2118                 if (cpu_online(cpu))
2119                         continue;
2120                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2121                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2122         }
2123
2124         mutex_lock(&cpufreq_governor_mutex);
2125         list_del(&governor->governor_list);
2126         mutex_unlock(&cpufreq_governor_mutex);
2127         return;
2128 }
2129 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2130
2131
2132 /*********************************************************************
2133  *                          POLICY INTERFACE                         *
2134  *********************************************************************/
2135
2136 /**
2137  * cpufreq_get_policy - get the current cpufreq_policy
2138  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2139  *      is written
2140  *
2141  * Reads the current cpufreq policy.
2142  */
2143 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2144 {
2145         struct cpufreq_policy *cpu_policy;
2146         if (!policy)
2147                 return -EINVAL;
2148
2149         cpu_policy = cpufreq_cpu_get(cpu);
2150         if (!cpu_policy)
2151                 return -EINVAL;
2152
2153         memcpy(policy, cpu_policy, sizeof(*policy));
2154
2155         cpufreq_cpu_put(cpu_policy);
2156         return 0;
2157 }
2158 EXPORT_SYMBOL(cpufreq_get_policy);
2159
2160 /*
2161  * policy : current policy.
2162  * new_policy: policy to be set.
2163  */
2164 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2165                                 struct cpufreq_policy *new_policy)
2166 {
2167         struct cpufreq_governor *old_gov;
2168         int ret;
2169
2170         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2171                  new_policy->cpu, new_policy->min, new_policy->max);
2172
2173         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2174
2175         if (new_policy->min > policy->max || new_policy->max < policy->min)
2176                 return -EINVAL;
2177
2178         /* verify the cpu speed can be set within this limit */
2179         ret = cpufreq_driver->verify(new_policy);
2180         if (ret)
2181                 return ret;
2182
2183         /* adjust if necessary - all reasons */
2184         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2185                         CPUFREQ_ADJUST, new_policy);
2186
2187         /* adjust if necessary - hardware incompatibility*/
2188         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2189                         CPUFREQ_INCOMPATIBLE, new_policy);
2190
2191         /*
2192          * verify the cpu speed can be set within this limit, which might be
2193          * different to the first one
2194          */
2195         ret = cpufreq_driver->verify(new_policy);
2196         if (ret)
2197                 return ret;
2198
2199         /* notification of the new policy */
2200         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2201                         CPUFREQ_NOTIFY, new_policy);
2202
2203         policy->min = new_policy->min;
2204         policy->max = new_policy->max;
2205
2206         pr_debug("new min and max freqs are %u - %u kHz\n",
2207                  policy->min, policy->max);
2208
2209         if (cpufreq_driver->setpolicy) {
2210                 policy->policy = new_policy->policy;
2211                 pr_debug("setting range\n");
2212                 return cpufreq_driver->setpolicy(new_policy);
2213         }
2214
2215         if (new_policy->governor == policy->governor)
2216                 goto out;
2217
2218         pr_debug("governor switch\n");
2219
2220         /* save old, working values */
2221         old_gov = policy->governor;
2222         /* end old governor */
2223         if (old_gov) {
2224                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2225                 up_write(&policy->rwsem);
2226                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2227                 down_write(&policy->rwsem);
2228         }
2229
2230         /* start new governor */
2231         policy->governor = new_policy->governor;
2232         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2233                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2234                         goto out;
2235
2236                 up_write(&policy->rwsem);
2237                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2238                 down_write(&policy->rwsem);
2239         }
2240
2241         /* new governor failed, so re-start old one */
2242         pr_debug("starting governor %s failed\n", policy->governor->name);
2243         if (old_gov) {
2244                 policy->governor = old_gov;
2245                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2246                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2247         }
2248
2249         return -EINVAL;
2250
2251  out:
2252         pr_debug("governor: change or update limits\n");
2253         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2254 }
2255
2256 /**
2257  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2258  *      @cpu: CPU which shall be re-evaluated
2259  *
2260  *      Useful for policy notifiers which have different necessities
2261  *      at different times.
2262  */
2263 int cpufreq_update_policy(unsigned int cpu)
2264 {
2265         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2266         struct cpufreq_policy new_policy;
2267         int ret;
2268
2269         if (!policy)
2270                 return -ENODEV;
2271
2272         down_write(&policy->rwsem);
2273
2274         pr_debug("updating policy for CPU %u\n", cpu);
2275         memcpy(&new_policy, policy, sizeof(*policy));
2276         new_policy.min = policy->user_policy.min;
2277         new_policy.max = policy->user_policy.max;
2278         new_policy.policy = policy->user_policy.policy;
2279         new_policy.governor = policy->user_policy.governor;
2280
2281         /*
2282          * BIOS might change freq behind our back
2283          * -> ask driver for current freq and notify governors about a change
2284          */
2285         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2286                 new_policy.cur = cpufreq_driver->get(cpu);
2287                 if (WARN_ON(!new_policy.cur)) {
2288                         ret = -EIO;
2289                         goto unlock;
2290                 }
2291
2292                 if (!policy->cur) {
2293                         pr_debug("Driver did not initialize current freq\n");
2294                         policy->cur = new_policy.cur;
2295                 } else {
2296                         if (policy->cur != new_policy.cur && has_target())
2297                                 cpufreq_out_of_sync(cpu, policy->cur,
2298                                                                 new_policy.cur);
2299                 }
2300         }
2301
2302         ret = cpufreq_set_policy(policy, &new_policy);
2303
2304 unlock:
2305         up_write(&policy->rwsem);
2306
2307         cpufreq_cpu_put(policy);
2308         return ret;
2309 }
2310 EXPORT_SYMBOL(cpufreq_update_policy);
2311
2312 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2313                                         unsigned long action, void *hcpu)
2314 {
2315         unsigned int cpu = (unsigned long)hcpu;
2316         struct device *dev;
2317
2318         dev = get_cpu_device(cpu);
2319         if (dev) {
2320                 switch (action & ~CPU_TASKS_FROZEN) {
2321                 case CPU_ONLINE:
2322                         __cpufreq_add_dev(dev, NULL);
2323                         break;
2324
2325                 case CPU_DOWN_PREPARE:
2326                         __cpufreq_remove_dev_prepare(dev, NULL);
2327                         break;
2328
2329                 case CPU_POST_DEAD:
2330                         __cpufreq_remove_dev_finish(dev, NULL);
2331                         break;
2332
2333                 case CPU_DOWN_FAILED:
2334                         __cpufreq_add_dev(dev, NULL);
2335                         break;
2336                 }
2337         }
2338         return NOTIFY_OK;
2339 }
2340
2341 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2342         .notifier_call = cpufreq_cpu_callback,
2343 };
2344
2345 /*********************************************************************
2346  *               BOOST                                               *
2347  *********************************************************************/
2348 static int cpufreq_boost_set_sw(int state)
2349 {
2350         struct cpufreq_frequency_table *freq_table;
2351         struct cpufreq_policy *policy;
2352         int ret = -EINVAL;
2353
2354         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2355                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2356                 if (freq_table) {
2357                         ret = cpufreq_frequency_table_cpuinfo(policy,
2358                                                         freq_table);
2359                         if (ret) {
2360                                 pr_err("%s: Policy frequency update failed\n",
2361                                        __func__);
2362                                 break;
2363                         }
2364                         policy->user_policy.max = policy->max;
2365                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2366                 }
2367         }
2368
2369         return ret;
2370 }
2371
2372 int cpufreq_boost_trigger_state(int state)
2373 {
2374         unsigned long flags;
2375         int ret = 0;
2376
2377         if (cpufreq_driver->boost_enabled == state)
2378                 return 0;
2379
2380         write_lock_irqsave(&cpufreq_driver_lock, flags);
2381         cpufreq_driver->boost_enabled = state;
2382         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2383
2384         ret = cpufreq_driver->set_boost(state);
2385         if (ret) {
2386                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2387                 cpufreq_driver->boost_enabled = !state;
2388                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2389
2390                 pr_err("%s: Cannot %s BOOST\n",
2391                        __func__, state ? "enable" : "disable");
2392         }
2393
2394         return ret;
2395 }
2396
2397 int cpufreq_boost_supported(void)
2398 {
2399         if (likely(cpufreq_driver))
2400                 return cpufreq_driver->boost_supported;
2401
2402         return 0;
2403 }
2404 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2405
2406 int cpufreq_boost_enabled(void)
2407 {
2408         return cpufreq_driver->boost_enabled;
2409 }
2410 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2411
2412 /*********************************************************************
2413  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2414  *********************************************************************/
2415
2416 /**
2417  * cpufreq_register_driver - register a CPU Frequency driver
2418  * @driver_data: A struct cpufreq_driver containing the values#
2419  * submitted by the CPU Frequency driver.
2420  *
2421  * Registers a CPU Frequency driver to this core code. This code
2422  * returns zero on success, -EBUSY when another driver got here first
2423  * (and isn't unregistered in the meantime).
2424  *
2425  */
2426 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2427 {
2428         unsigned long flags;
2429         int ret;
2430
2431         if (cpufreq_disabled())
2432                 return -ENODEV;
2433
2434         if (!driver_data || !driver_data->verify || !driver_data->init ||
2435             !(driver_data->setpolicy || driver_data->target_index ||
2436                     driver_data->target) ||
2437              (driver_data->setpolicy && (driver_data->target_index ||
2438                     driver_data->target)) ||
2439              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2440                 return -EINVAL;
2441
2442         pr_debug("trying to register driver %s\n", driver_data->name);
2443
2444         if (driver_data->setpolicy)
2445                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2446
2447         write_lock_irqsave(&cpufreq_driver_lock, flags);
2448         if (cpufreq_driver) {
2449                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2450                 return -EEXIST;
2451         }
2452         cpufreq_driver = driver_data;
2453         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2454
2455         if (cpufreq_boost_supported()) {
2456                 /*
2457                  * Check if driver provides function to enable boost -
2458                  * if not, use cpufreq_boost_set_sw as default
2459                  */
2460                 if (!cpufreq_driver->set_boost)
2461                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2462
2463                 ret = cpufreq_sysfs_create_file(&boost.attr);
2464                 if (ret) {
2465                         pr_err("%s: cannot register global BOOST sysfs file\n",
2466                                __func__);
2467                         goto err_null_driver;
2468                 }
2469         }
2470
2471         ret = subsys_interface_register(&cpufreq_interface);
2472         if (ret)
2473                 goto err_boost_unreg;
2474
2475         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2476                 int i;
2477                 ret = -ENODEV;
2478
2479                 /* check for at least one working CPU */
2480                 for (i = 0; i < nr_cpu_ids; i++)
2481                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2482                                 ret = 0;
2483                                 break;
2484                         }
2485
2486                 /* if all ->init() calls failed, unregister */
2487                 if (ret) {
2488                         pr_debug("no CPU initialized for driver %s\n",
2489                                  driver_data->name);
2490                         goto err_if_unreg;
2491                 }
2492         }
2493
2494         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2495         pr_debug("driver %s up and running\n", driver_data->name);
2496
2497         return 0;
2498 err_if_unreg:
2499         subsys_interface_unregister(&cpufreq_interface);
2500 err_boost_unreg:
2501         if (cpufreq_boost_supported())
2502                 cpufreq_sysfs_remove_file(&boost.attr);
2503 err_null_driver:
2504         write_lock_irqsave(&cpufreq_driver_lock, flags);
2505         cpufreq_driver = NULL;
2506         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2507         return ret;
2508 }
2509 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2510
2511 /**
2512  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2513  *
2514  * Unregister the current CPUFreq driver. Only call this if you have
2515  * the right to do so, i.e. if you have succeeded in initialising before!
2516  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2517  * currently not initialised.
2518  */
2519 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2520 {
2521         unsigned long flags;
2522
2523         if (!cpufreq_driver || (driver != cpufreq_driver))
2524                 return -EINVAL;
2525
2526         pr_debug("unregistering driver %s\n", driver->name);
2527
2528         subsys_interface_unregister(&cpufreq_interface);
2529         if (cpufreq_boost_supported())
2530                 cpufreq_sysfs_remove_file(&boost.attr);
2531
2532         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2533
2534         down_write(&cpufreq_rwsem);
2535         write_lock_irqsave(&cpufreq_driver_lock, flags);
2536
2537         cpufreq_driver = NULL;
2538
2539         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2540         up_write(&cpufreq_rwsem);
2541
2542         return 0;
2543 }
2544 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2545
2546 static int __init cpufreq_core_init(void)
2547 {
2548         if (cpufreq_disabled())
2549                 return -ENODEV;
2550
2551         cpufreq_global_kobject = kobject_create();
2552         BUG_ON(!cpufreq_global_kobject);
2553
2554         return 0;
2555 }
2556 core_initcall(cpufreq_core_init);