drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 static inline bool has_target(void)
  51 {
  52         return cpufreq_driver->target_index || cpufreq_driver->target;
  53 }
  54
  55 /*
  56  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  57  * sections
  58  */
  59 static DECLARE_RWSEM(cpufreq_rwsem);
  60
  61 /* internal prototypes */
  62 static int __cpufreq_governor(struct cpufreq_policy *policy,
  63                 unsigned int event);
  64 static unsigned int __cpufreq_get(unsigned int cpu);
  65 static void handle_update(struct work_struct *work);
  66
  67 /**
  68  * Two notifier lists: the "policy" list is involved in the
  69  * validation process for a new CPU frequency policy; the
  70  * "transition" list for kernel code that needs to handle
  71  * changes to devices when the CPU clock speed changes.
  72  * The mutex locks both lists.
  73  */
  74 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  75 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  76
  77 static bool init_cpufreq_transition_notifier_list_called;
  78 static int __init init_cpufreq_transition_notifier_list(void)
  79 {
  80         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  81         init_cpufreq_transition_notifier_list_called = true;
  82         return 0;
  83 }
  84 pure_initcall(init_cpufreq_transition_notifier_list);
  85
  86 static int off __read_mostly;
  87 static int cpufreq_disabled(void)
  88 {
  89         return off;
  90 }
  91 void disable_cpufreq(void)
  92 {
  93         off = 1;
  94 }
  95 static LIST_HEAD(cpufreq_governor_list);
  96 static DEFINE_MUTEX(cpufreq_governor_mutex);
  97
  98 bool have_governor_per_policy(void)
  99 {
 100         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 101 }
 102 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 103
 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 105 {
 106         if (have_governor_per_policy())
 107                 return &policy->kobj;
 108         else
 109                 return cpufreq_global_kobject;
 110 }
 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 112
 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 114 {
 115         u64 idle_time;
 116         u64 cur_wall_time;
 117         u64 busy_time;
 118
 119         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 120
 121         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 122         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 127
 128         idle_time = cur_wall_time - busy_time;
 129         if (wall)
 130                 *wall = cputime_to_usecs(cur_wall_time);
 131
 132         return cputime_to_usecs(idle_time);
 133 }
 134
 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 136 {
 137         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 138
 139         if (idle_time == -1ULL)
 140                 return get_cpu_idle_time_jiffy(cpu, wall);
 141         else if (!io_busy)
 142                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 143
 144         return idle_time;
 145 }
 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 147
 148 /*
 149  * This is a generic cpufreq init() routine which can be used by cpufreq
 150  * drivers of SMP systems. It will do following:
 151  * - validate & show freq table passed
 152  * - set policies transition latency
 153  * - policy->cpus with all possible CPUs
 154  */
 155 int cpufreq_generic_init(struct cpufreq_policy *policy,
 156                 struct cpufreq_frequency_table *table,
 157                 unsigned int transition_latency)
 158 {
 159         int ret;
 160
 161         ret = cpufreq_table_validate_and_show(policy, table);
 162         if (ret) {
 163                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 164                 return ret;
 165         }
 166
 167         policy->cpuinfo.transition_latency = transition_latency;
 168
 169         /*
 170          * The driver only supports the SMP configuartion where all processors
 171          * share the clock and voltage and clock.
 172          */
 173         cpumask_setall(policy->cpus);
 174
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 178
 179 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 180 {
 181         struct cpufreq_policy *policy = NULL;
 182         unsigned long flags;
 183
 184         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 185                 return NULL;
 186
 187         if (!down_read_trylock(&cpufreq_rwsem))
 188                 return NULL;
 189
 190         /* get the cpufreq driver */
 191         read_lock_irqsave(&cpufreq_driver_lock, flags);
 192
 193         if (cpufreq_driver) {
 194                 /* get the CPU */
 195                 policy = per_cpu(cpufreq_cpu_data, cpu);
 196                 if (policy)
 197                         kobject_get(&policy->kobj);
 198         }
 199
 200         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 201
 202         if (!policy)
 203                 up_read(&cpufreq_rwsem);
 204
 205         return policy;
 206 }
 207 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 208
 209 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 210 {
 211         if (cpufreq_disabled())
 212                 return;
 213
 214         kobject_put(&policy->kobj);
 215         up_read(&cpufreq_rwsem);
 216 }
 217 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 218
 219 /*********************************************************************
 220  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 221  *********************************************************************/
 222
 223 /**
 224  * adjust_jiffies - adjust the system "loops_per_jiffy"
 225  *
 226  * This function alters the system "loops_per_jiffy" for the clock
 227  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 228  * systems as each CPU might be scaled differently. So, use the arch
 229  * per-CPU loops_per_jiffy value wherever possible.
 230  */
 231 #ifndef CONFIG_SMP
 232 static unsigned long l_p_j_ref;
 233 static unsigned int l_p_j_ref_freq;
 234
 235 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 236 {
 237         if (ci->flags & CPUFREQ_CONST_LOOPS)
 238                 return;
 239
 240         if (!l_p_j_ref_freq) {
 241                 l_p_j_ref = loops_per_jiffy;
 242                 l_p_j_ref_freq = ci->old;
 243                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 244                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 245         }
 246         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 247             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 248                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 249                                                                 ci->new);
 250                 pr_debug("scaling loops_per_jiffy to %lu "
 251                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 252         }
 253 }
 254 #else
 255 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 256 {
 257         return;
 258 }
 259 #endif
 260
 261 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 262                 struct cpufreq_freqs *freqs, unsigned int state)
 263 {
 264         BUG_ON(irqs_disabled());
 265
 266         if (cpufreq_disabled())
 267                 return;
 268
 269         freqs->flags = cpufreq_driver->flags;
 270         pr_debug("notification %u of frequency transition to %u kHz\n",
 271                 state, freqs->new);
 272
 273         switch (state) {
 274
 275         case CPUFREQ_PRECHANGE:
 276                 /* detect if the driver reported a value as "old frequency"
 277                  * which is not equal to what the cpufreq core thinks is
 278                  * "old frequency".
 279                  */
 280                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 281                         if ((policy) && (policy->cpu == freqs->cpu) &&
 282                             (policy->cur) && (policy->cur != freqs->old)) {
 283                                 pr_debug("Warning: CPU frequency is"
 284                                         " %u, cpufreq assumed %u kHz.\n",
 285                                         freqs->old, policy->cur);
 286                                 freqs->old = policy->cur;
 287                         }
 288                 }
 289                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 290                                 CPUFREQ_PRECHANGE, freqs);
 291                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 292                 break;
 293
 294         case CPUFREQ_POSTCHANGE:
 295                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 296                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 297                         (unsigned long)freqs->cpu);
 298                 trace_cpu_frequency(freqs->new, freqs->cpu);
 299                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 300                                 CPUFREQ_POSTCHANGE, freqs);
 301                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 302                         policy->cur = freqs->new;
 303                 break;
 304         }
 305 }
 306
 307 /**
 308  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 309  * on frequency transition.
 310  *
 311  * This function calls the transition notifiers and the "adjust_jiffies"
 312  * function. It is called twice on all CPU frequency changes that have
 313  * external effects.
 314  */
 315 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 316                 struct cpufreq_freqs *freqs, unsigned int state)
 317 {
 318         for_each_cpu(freqs->cpu, policy->cpus)
 319                 __cpufreq_notify_transition(policy, freqs, state);
 320 }
 321 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 322
 323 /* Do post notifications when there are chances that transition has failed */
 324 void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 325                 struct cpufreq_freqs *freqs, int transition_failed)
 326 {
 327         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 328         if (!transition_failed)
 329                 return;
 330
 331         swap(freqs->old, freqs->new);
 332         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 333         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 334 }
 335 EXPORT_SYMBOL_GPL(cpufreq_notify_post_transition);
 336
 337
 338 /*********************************************************************
 339  *                          SYSFS INTERFACE                          *
 340  *********************************************************************/
 341
 342 static struct cpufreq_governor *__find_governor(const char *str_governor)
 343 {
 344         struct cpufreq_governor *t;
 345
 346         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 347                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 348                         return t;
 349
 350         return NULL;
 351 }
 352
 353 /**
 354  * cpufreq_parse_governor - parse a governor string
 355  */
 356 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 357                                 struct cpufreq_governor **governor)
 358 {
 359         int err = -EINVAL;
 360
 361         if (!cpufreq_driver)
 362                 goto out;
 363
 364         if (cpufreq_driver->setpolicy) {
 365                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 366                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 367                         err = 0;
 368                 } else if (!strnicmp(str_governor, "powersave",
 369                                                 CPUFREQ_NAME_LEN)) {
 370                         *policy = CPUFREQ_POLICY_POWERSAVE;
 371                         err = 0;
 372                 }
 373         } else if (has_target()) {
 374                 struct cpufreq_governor *t;
 375
 376                 mutex_lock(&cpufreq_governor_mutex);
 377
 378                 t = __find_governor(str_governor);
 379
 380                 if (t == NULL) {
 381                         int ret;
 382
 383                         mutex_unlock(&cpufreq_governor_mutex);
 384                         ret = request_module("cpufreq_%s", str_governor);
 385                         mutex_lock(&cpufreq_governor_mutex);
 386
 387                         if (ret == 0)
 388                                 t = __find_governor(str_governor);
 389                 }
 390
 391                 if (t != NULL) {
 392                         *governor = t;
 393                         err = 0;
 394                 }
 395
 396                 mutex_unlock(&cpufreq_governor_mutex);
 397         }
 398 out:
 399         return err;
 400 }
 401
 402 /**
 403  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 404  * print out cpufreq information
 405  *
 406  * Write out information from cpufreq_driver->policy[cpu]; object must be
 407  * "unsigned int".
 408  */
 409
 410 #define show_one(file_name, object)                     \
 411 static ssize_t show_##file_name                         \
 412 (struct cpufreq_policy *policy, char *buf)              \
 413 {                                                       \
 414         return sprintf(buf, "%u\n", policy->object);    \
 415 }
 416
 417 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 418 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 419 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 420 show_one(scaling_min_freq, min);
 421 show_one(scaling_max_freq, max);
 422 show_one(scaling_cur_freq, cur);
 423
 424 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 425                                 struct cpufreq_policy *new_policy);
 426
 427 /**
 428  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 429  */
 430 #define store_one(file_name, object)                    \
 431 static ssize_t store_##file_name                                        \
 432 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 433 {                                                                       \
 434         int ret;                                                        \
 435         struct cpufreq_policy new_policy;                               \
 436                                                                         \
 437         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 438         if (ret)                                                        \
 439                 return -EINVAL;                                         \
 440                                                                         \
 441         ret = sscanf(buf, "%u", &new_policy.object);                    \
 442         if (ret != 1)                                                   \
 443                 return -EINVAL;                                         \
 444                                                                         \
 445         ret = cpufreq_set_policy(policy, &new_policy);          \
 446         policy->user_policy.object = policy->object;                    \
 447                                                                         \
 448         return ret ? ret : count;                                       \
 449 }
 450
 451 store_one(scaling_min_freq, min);
 452 store_one(scaling_max_freq, max);
 453
 454 /**
 455  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 456  */
 457 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 458                                         char *buf)
 459 {
 460         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 461         if (!cur_freq)
 462                 return sprintf(buf, "<unknown>");
 463         return sprintf(buf, "%u\n", cur_freq);
 464 }
 465
 466 /**
 467  * show_scaling_governor - show the current policy for the specified CPU
 468  */
 469 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 470 {
 471         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 472                 return sprintf(buf, "powersave\n");
 473         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 474                 return sprintf(buf, "performance\n");
 475         else if (policy->governor)
 476                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 477                                 policy->governor->name);
 478         return -EINVAL;
 479 }
 480
 481 /**
 482  * store_scaling_governor - store policy for the specified CPU
 483  */
 484 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 485                                         const char *buf, size_t count)
 486 {
 487         int ret;
 488         char    str_governor[16];
 489         struct cpufreq_policy new_policy;
 490
 491         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 492         if (ret)
 493                 return ret;
 494
 495         ret = sscanf(buf, "%15s", str_governor);
 496         if (ret != 1)
 497                 return -EINVAL;
 498
 499         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 500                                                 &new_policy.governor))
 501                 return -EINVAL;
 502
 503         ret = cpufreq_set_policy(policy, &new_policy);
 504
 505         policy->user_policy.policy = policy->policy;
 506         policy->user_policy.governor = policy->governor;
 507
 508         if (ret)
 509                 return ret;
 510         else
 511                 return count;
 512 }
 513
 514 /**
 515  * show_scaling_driver - show the cpufreq driver currently loaded
 516  */
 517 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 518 {
 519         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 520 }
 521
 522 /**
 523  * show_scaling_available_governors - show the available CPUfreq governors
 524  */
 525 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 526                                                 char *buf)
 527 {
 528         ssize_t i = 0;
 529         struct cpufreq_governor *t;
 530
 531         if (!has_target()) {
 532                 i += sprintf(buf, "performance powersave");
 533                 goto out;
 534         }
 535
 536         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 537                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 538                     - (CPUFREQ_NAME_LEN + 2)))
 539                         goto out;
 540                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 541         }
 542 out:
 543         i += sprintf(&buf[i], "\n");
 544         return i;
 545 }
 546
 547 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 548 {
 549         ssize_t i = 0;
 550         unsigned int cpu;
 551
 552         for_each_cpu(cpu, mask) {
 553                 if (i)
 554                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 555                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 556                 if (i >= (PAGE_SIZE - 5))
 557                         break;
 558         }
 559         i += sprintf(&buf[i], "\n");
 560         return i;
 561 }
 562 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 563
 564 /**
 565  * show_related_cpus - show the CPUs affected by each transition even if
 566  * hw coordination is in use
 567  */
 568 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 569 {
 570         return cpufreq_show_cpus(policy->related_cpus, buf);
 571 }
 572
 573 /**
 574  * show_affected_cpus - show the CPUs affected by each transition
 575  */
 576 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 577 {
 578         return cpufreq_show_cpus(policy->cpus, buf);
 579 }
 580
 581 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 582                                         const char *buf, size_t count)
 583 {
 584         unsigned int freq = 0;
 585         unsigned int ret;
 586
 587         if (!policy->governor || !policy->governor->store_setspeed)
 588                 return -EINVAL;
 589
 590         ret = sscanf(buf, "%u", &freq);
 591         if (ret != 1)
 592                 return -EINVAL;
 593
 594         policy->governor->store_setspeed(policy, freq);
 595
 596         return count;
 597 }
 598
 599 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 600 {
 601         if (!policy->governor || !policy->governor->show_setspeed)
 602                 return sprintf(buf, "<unsupported>\n");
 603
 604         return policy->governor->show_setspeed(policy, buf);
 605 }
 606
 607 /**
 608  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 609  */
 610 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 611 {
 612         unsigned int limit;
 613         int ret;
 614         if (cpufreq_driver->bios_limit) {
 615                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 616                 if (!ret)
 617                         return sprintf(buf, "%u\n", limit);
 618         }
 619         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 620 }
 621
 622 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 623 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 624 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 625 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 626 cpufreq_freq_attr_ro(scaling_available_governors);
 627 cpufreq_freq_attr_ro(scaling_driver);
 628 cpufreq_freq_attr_ro(scaling_cur_freq);
 629 cpufreq_freq_attr_ro(bios_limit);
 630 cpufreq_freq_attr_ro(related_cpus);
 631 cpufreq_freq_attr_ro(affected_cpus);
 632 cpufreq_freq_attr_rw(scaling_min_freq);
 633 cpufreq_freq_attr_rw(scaling_max_freq);
 634 cpufreq_freq_attr_rw(scaling_governor);
 635 cpufreq_freq_attr_rw(scaling_setspeed);
 636
 637 static struct attribute *default_attrs[] = {
 638         &cpuinfo_min_freq.attr,
 639         &cpuinfo_max_freq.attr,
 640         &cpuinfo_transition_latency.attr,
 641         &scaling_min_freq.attr,
 642         &scaling_max_freq.attr,
 643         &affected_cpus.attr,
 644         &related_cpus.attr,
 645         &scaling_governor.attr,
 646         &scaling_driver.attr,
 647         &scaling_available_governors.attr,
 648         &scaling_setspeed.attr,
 649         NULL
 650 };
 651
 652 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 653 #define to_attr(a) container_of(a, struct freq_attr, attr)
 654
 655 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 656 {
 657         struct cpufreq_policy *policy = to_policy(kobj);
 658         struct freq_attr *fattr = to_attr(attr);
 659         ssize_t ret;
 660
 661         if (!down_read_trylock(&cpufreq_rwsem))
 662                 return -EINVAL;
 663
 664         down_read(&policy->rwsem);
 665
 666         if (fattr->show)
 667                 ret = fattr->show(policy, buf);
 668         else
 669                 ret = -EIO;
 670
 671         up_read(&policy->rwsem);
 672         up_read(&cpufreq_rwsem);
 673
 674         return ret;
 675 }
 676
 677 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 678                      const char *buf, size_t count)
 679 {
 680         struct cpufreq_policy *policy = to_policy(kobj);
 681         struct freq_attr *fattr = to_attr(attr);
 682         ssize_t ret = -EINVAL;
 683
 684         get_online_cpus();
 685
 686         if (!cpu_online(policy->cpu))
 687                 goto unlock;
 688
 689         if (!down_read_trylock(&cpufreq_rwsem))
 690                 goto unlock;
 691
 692         down_write(&policy->rwsem);
 693
 694         if (fattr->store)
 695                 ret = fattr->store(policy, buf, count);
 696         else
 697                 ret = -EIO;
 698
 699         up_write(&policy->rwsem);
 700
 701         up_read(&cpufreq_rwsem);
 702 unlock:
 703         put_online_cpus();
 704
 705         return ret;
 706 }
 707
 708 static void cpufreq_sysfs_release(struct kobject *kobj)
 709 {
 710         struct cpufreq_policy *policy = to_policy(kobj);
 711         pr_debug("last reference is dropped\n");
 712         complete(&policy->kobj_unregister);
 713 }
 714
 715 static const struct sysfs_ops sysfs_ops = {
 716         .show   = show,
 717         .store  = store,
 718 };
 719
 720 static struct kobj_type ktype_cpufreq = {
 721         .sysfs_ops      = &sysfs_ops,
 722         .default_attrs  = default_attrs,
 723         .release        = cpufreq_sysfs_release,
 724 };
 725
 726 struct kobject *cpufreq_global_kobject;
 727 EXPORT_SYMBOL(cpufreq_global_kobject);
 728
 729 static int cpufreq_global_kobject_usage;
 730
 731 int cpufreq_get_global_kobject(void)
 732 {
 733         if (!cpufreq_global_kobject_usage++)
 734                 return kobject_add(cpufreq_global_kobject,
 735                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 736
 737         return 0;
 738 }
 739 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 740
 741 void cpufreq_put_global_kobject(void)
 742 {
 743         if (!--cpufreq_global_kobject_usage)
 744                 kobject_del(cpufreq_global_kobject);
 745 }
 746 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 747
 748 int cpufreq_sysfs_create_file(const struct attribute *attr)
 749 {
 750         int ret = cpufreq_get_global_kobject();
 751
 752         if (!ret) {
 753                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 754                 if (ret)
 755                         cpufreq_put_global_kobject();
 756         }
 757
 758         return ret;
 759 }
 760 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 761
 762 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 763 {
 764         sysfs_remove_file(cpufreq_global_kobject, attr);
 765         cpufreq_put_global_kobject();
 766 }
 767 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 768
 769 /* symlink affected CPUs */
 770 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 771 {
 772         unsigned int j;
 773         int ret = 0;
 774
 775         for_each_cpu(j, policy->cpus) {
 776                 struct device *cpu_dev;
 777
 778                 if (j == policy->cpu)
 779                         continue;
 780
 781                 pr_debug("Adding link for CPU: %u\n", j);
 782                 cpu_dev = get_cpu_device(j);
 783                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 784                                         "cpufreq");
 785                 if (ret)
 786                         break;
 787         }
 788         return ret;
 789 }
 790
 791 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 792                                      struct device *dev)
 793 {
 794         struct freq_attr **drv_attr;
 795         int ret = 0;
 796
 797         /* prepare interface data */
 798         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 799                                    &dev->kobj, "cpufreq");
 800         if (ret)
 801                 return ret;
 802
 803         /* set up files for this cpu device */
 804         drv_attr = cpufreq_driver->attr;
 805         while ((drv_attr) && (*drv_attr)) {
 806                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 807                 if (ret)
 808                         goto err_out_kobj_put;
 809                 drv_attr++;
 810         }
 811         if (cpufreq_driver->get) {
 812                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 813                 if (ret)
 814                         goto err_out_kobj_put;
 815         }
 816         if (has_target()) {
 817                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 818                 if (ret)
 819                         goto err_out_kobj_put;
 820         }
 821         if (cpufreq_driver->bios_limit) {
 822                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 823                 if (ret)
 824                         goto err_out_kobj_put;
 825         }
 826
 827         ret = cpufreq_add_dev_symlink(policy);
 828         if (ret)
 829                 goto err_out_kobj_put;
 830
 831         return ret;
 832
 833 err_out_kobj_put:
 834         kobject_put(&policy->kobj);
 835         wait_for_completion(&policy->kobj_unregister);
 836         return ret;
 837 }
 838
 839 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 840 {
 841         struct cpufreq_policy new_policy;
 842         int ret = 0;
 843
 844         memcpy(&new_policy, policy, sizeof(*policy));
 845
 846         /* Use the default policy if its valid. */
 847         if (cpufreq_driver->setpolicy)
 848                 cpufreq_parse_governor(policy->governor->name,
 849                                         &new_policy.policy, NULL);
 850
 851         /* assure that the starting sequence is run in cpufreq_set_policy */
 852         policy->governor = NULL;
 853
 854         /* set default policy */
 855         ret = cpufreq_set_policy(policy, &new_policy);
 856         if (ret) {
 857                 pr_debug("setting policy failed\n");
 858                 if (cpufreq_driver->exit)
 859                         cpufreq_driver->exit(policy);
 860         }
 861 }
 862
 863 #ifdef CONFIG_HOTPLUG_CPU
 864 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 865                                   unsigned int cpu, struct device *dev)
 866 {
 867         int ret = 0;
 868         unsigned long flags;
 869
 870         if (has_target()) {
 871                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 872                 if (ret) {
 873                         pr_err("%s: Failed to stop governor\n", __func__);
 874                         return ret;
 875                 }
 876         }
 877
 878         down_write(&policy->rwsem);
 879
 880         write_lock_irqsave(&cpufreq_driver_lock, flags);
 881
 882         cpumask_set_cpu(cpu, policy->cpus);
 883         per_cpu(cpufreq_cpu_data, cpu) = policy;
 884         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 885
 886         up_write(&policy->rwsem);
 887
 888         if (has_target()) {
 889                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 890                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 891                         pr_err("%s: Failed to start governor\n", __func__);
 892                         return ret;
 893                 }
 894         }
 895
 896         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 897 }
 898 #endif
 899
 900 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 901 {
 902         struct cpufreq_policy *policy;
 903         unsigned long flags;
 904
 905         read_lock_irqsave(&cpufreq_driver_lock, flags);
 906
 907         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 908
 909         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 910
 911         return policy;
 912 }
 913
 914 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 915 {
 916         struct cpufreq_policy *policy;
 917
 918         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 919         if (!policy)
 920                 return NULL;
 921
 922         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 923                 goto err_free_policy;
 924
 925         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 926                 goto err_free_cpumask;
 927
 928         INIT_LIST_HEAD(&policy->policy_list);
 929         init_rwsem(&policy->rwsem);
 930
 931         return policy;
 932
 933 err_free_cpumask:
 934         free_cpumask_var(policy->cpus);
 935 err_free_policy:
 936         kfree(policy);
 937
 938         return NULL;
 939 }
 940
 941 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
 942 {
 943         struct kobject *kobj;
 944         struct completion *cmp;
 945
 946         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 947                         CPUFREQ_REMOVE_POLICY, policy);
 948
 949         down_read(&policy->rwsem);
 950         kobj = &policy->kobj;
 951         cmp = &policy->kobj_unregister;
 952         up_read(&policy->rwsem);
 953         kobject_put(kobj);
 954
 955         /*
 956          * We need to make sure that the underlying kobj is
 957          * actually not referenced anymore by anybody before we
 958          * proceed with unloading.
 959          */
 960         pr_debug("waiting for dropping of refcount\n");
 961         wait_for_completion(cmp);
 962         pr_debug("wait complete\n");
 963 }
 964
 965 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 966 {
 967         free_cpumask_var(policy->related_cpus);
 968         free_cpumask_var(policy->cpus);
 969         kfree(policy);
 970 }
 971
 972 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 973 {
 974         if (WARN_ON(cpu == policy->cpu))
 975                 return;
 976
 977         down_write(&policy->rwsem);
 978
 979         policy->last_cpu = policy->cpu;
 980         policy->cpu = cpu;
 981
 982         up_write(&policy->rwsem);
 983
 984         cpufreq_frequency_table_update_policy_cpu(policy);
 985         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 986                         CPUFREQ_UPDATE_POLICY_CPU, policy);
 987 }
 988
 989 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 990                              bool frozen)
 991 {
 992         unsigned int j, cpu = dev->id;
 993         int ret = -ENOMEM;
 994         struct cpufreq_policy *policy;
 995         unsigned long flags;
 996 #ifdef CONFIG_HOTPLUG_CPU
 997         struct cpufreq_policy *tpolicy;
 998         struct cpufreq_governor *gov;
 999 #endif
1000
1001         if (cpu_is_offline(cpu))
1002                 return 0;
1003
1004         pr_debug("adding CPU %u\n", cpu);
1005
1006 #ifdef CONFIG_SMP
1007         /* check whether a different CPU already registered this
1008          * CPU because it is in the same boat. */
1009         policy = cpufreq_cpu_get(cpu);
1010         if (unlikely(policy)) {
1011                 cpufreq_cpu_put(policy);
1012                 return 0;
1013         }
1014 #endif
1015
1016         if (!down_read_trylock(&cpufreq_rwsem))
1017                 return 0;
1018
1019 #ifdef CONFIG_HOTPLUG_CPU
1020         /* Check if this cpu was hot-unplugged earlier and has siblings */
1021         read_lock_irqsave(&cpufreq_driver_lock, flags);
1022         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1023                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1024                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1025                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1026                         up_read(&cpufreq_rwsem);
1027                         return ret;
1028                 }
1029         }
1030         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1031 #endif
1032
1033         /*
1034          * Restore the saved policy when doing light-weight init and fall back
1035          * to the full init if that fails.
1036          */
1037         policy = frozen ? cpufreq_policy_restore(cpu) : NULL;
1038         if (!policy) {
1039                 frozen = false;
1040                 policy = cpufreq_policy_alloc();
1041                 if (!policy)
1042                         goto nomem_out;
1043         }
1044
1045         /*
1046          * In the resume path, since we restore a saved policy, the assignment
1047          * to policy->cpu is like an update of the existing policy, rather than
1048          * the creation of a brand new one. So we need to perform this update
1049          * by invoking update_policy_cpu().
1050          */
1051         if (frozen && cpu != policy->cpu)
1052                 update_policy_cpu(policy, cpu);
1053         else
1054                 policy->cpu = cpu;
1055
1056         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1057         cpumask_copy(policy->cpus, cpumask_of(cpu));
1058
1059         init_completion(&policy->kobj_unregister);
1060         INIT_WORK(&policy->update, handle_update);
1061
1062         /* call driver. From then on the cpufreq must be able
1063          * to accept all calls to ->verify and ->setpolicy for this CPU
1064          */
1065         ret = cpufreq_driver->init(policy);
1066         if (ret) {
1067                 pr_debug("initialization failed\n");
1068                 goto err_set_policy_cpu;
1069         }
1070
1071         if (cpufreq_driver->get) {
1072                 policy->cur = cpufreq_driver->get(policy->cpu);
1073                 if (!policy->cur) {
1074                         pr_err("%s: ->get() failed\n", __func__);
1075                         goto err_get_freq;
1076                 }
1077         }
1078
1079         /*
1080          * Sometimes boot loaders set CPU frequency to a value outside of
1081          * frequency table present with cpufreq core. In such cases CPU might be
1082          * unstable if it has to run on that frequency for long duration of time
1083          * and so its better to set it to a frequency which is specified in
1084          * freq-table. This also makes cpufreq stats inconsistent as
1085          * cpufreq-stats would fail to register because current frequency of CPU
1086          * isn't found in freq-table.
1087          *
1088          * Because we don't want this change to effect boot process badly, we go
1089          * for the next freq which is >= policy->cur ('cur' must be set by now,
1090          * otherwise we will end up setting freq to lowest of the table as 'cur'
1091          * is initialized to zero).
1092          *
1093          * We are passing target-freq as "policy->cur - 1" otherwise
1094          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1095          * equal to target-freq.
1096          */
1097         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1098             && has_target()) {
1099                 /* Are we running at unknown frequency ? */
1100                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1101                 if (ret == -EINVAL) {
1102                         /* Warn user and fix it */
1103                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1104                                 __func__, policy->cpu, policy->cur);
1105                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1106                                 CPUFREQ_RELATION_L);
1107
1108                         /*
1109                          * Reaching here after boot in a few seconds may not
1110                          * mean that system will remain stable at "unknown"
1111                          * frequency for longer duration. Hence, a BUG_ON().
1112                          */
1113                         BUG_ON(ret);
1114                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1115                                 __func__, policy->cpu, policy->cur);
1116                 }
1117         }
1118
1119         /* related cpus should atleast have policy->cpus */
1120         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1121
1122         /*
1123          * affected cpus must always be the one, which are online. We aren't
1124          * managing offline cpus here.
1125          */
1126         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1127
1128         if (!frozen) {
1129                 policy->user_policy.min = policy->min;
1130                 policy->user_policy.max = policy->max;
1131         }
1132
1133         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1134                                      CPUFREQ_START, policy);
1135
1136 #ifdef CONFIG_HOTPLUG_CPU
1137         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1138         if (gov) {
1139                 policy->governor = gov;
1140                 pr_debug("Restoring governor %s for cpu %d\n",
1141                        policy->governor->name, cpu);
1142         }
1143 #endif
1144
1145         write_lock_irqsave(&cpufreq_driver_lock, flags);
1146         for_each_cpu(j, policy->cpus)
1147                 per_cpu(cpufreq_cpu_data, j) = policy;
1148         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1149
1150         if (!frozen) {
1151                 ret = cpufreq_add_dev_interface(policy, dev);
1152                 if (ret)
1153                         goto err_out_unregister;
1154                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1155                                 CPUFREQ_CREATE_POLICY, policy);
1156         }
1157
1158         write_lock_irqsave(&cpufreq_driver_lock, flags);
1159         list_add(&policy->policy_list, &cpufreq_policy_list);
1160         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1161
1162         cpufreq_init_policy(policy);
1163
1164         if (!frozen) {
1165                 policy->user_policy.policy = policy->policy;
1166                 policy->user_policy.governor = policy->governor;
1167         }
1168
1169         kobject_uevent(&policy->kobj, KOBJ_ADD);
1170         up_read(&cpufreq_rwsem);
1171
1172         pr_debug("initialization complete\n");
1173
1174         return 0;
1175
1176 err_out_unregister:
1177         write_lock_irqsave(&cpufreq_driver_lock, flags);
1178         for_each_cpu(j, policy->cpus)
1179                 per_cpu(cpufreq_cpu_data, j) = NULL;
1180         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1181
1182 err_get_freq:
1183         if (cpufreq_driver->exit)
1184                 cpufreq_driver->exit(policy);
1185 err_set_policy_cpu:
1186         if (frozen) {
1187                 /* Do not leave stale fallback data behind. */
1188                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1189                 cpufreq_policy_put_kobj(policy);
1190         }
1191         cpufreq_policy_free(policy);
1192
1193 nomem_out:
1194         up_read(&cpufreq_rwsem);
1195
1196         return ret;
1197 }
1198
1199 /**
1200  * cpufreq_add_dev - add a CPU device
1201  *
1202  * Adds the cpufreq interface for a CPU device.
1203  *
1204  * The Oracle says: try running cpufreq registration/unregistration concurrently
1205  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1206  * mess up, but more thorough testing is needed. - Mathieu
1207  */
1208 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1209 {
1210         return __cpufreq_add_dev(dev, sif, false);
1211 }
1212
1213 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1214                                            unsigned int old_cpu)
1215 {
1216         struct device *cpu_dev;
1217         int ret;
1218
1219         /* first sibling now owns the new sysfs dir */
1220         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1221
1222         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1223         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1224         if (ret) {
1225                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1226
1227                 down_write(&policy->rwsem);
1228                 cpumask_set_cpu(old_cpu, policy->cpus);
1229                 up_write(&policy->rwsem);
1230
1231                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1232                                         "cpufreq");
1233
1234                 return -EINVAL;
1235         }
1236
1237         return cpu_dev->id;
1238 }
1239
1240 static int __cpufreq_remove_dev_prepare(struct device *dev,
1241                                         struct subsys_interface *sif,
1242                                         bool frozen)
1243 {
1244         unsigned int cpu = dev->id, cpus;
1245         int new_cpu, ret;
1246         unsigned long flags;
1247         struct cpufreq_policy *policy;
1248
1249         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1250
1251         write_lock_irqsave(&cpufreq_driver_lock, flags);
1252
1253         policy = per_cpu(cpufreq_cpu_data, cpu);
1254
1255         /* Save the policy somewhere when doing a light-weight tear-down */
1256         if (frozen)
1257                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1258
1259         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1260
1261         if (!policy) {
1262                 pr_debug("%s: No cpu_data found\n", __func__);
1263                 return -EINVAL;
1264         }
1265
1266         if (has_target()) {
1267                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1268                 if (ret) {
1269                         pr_err("%s: Failed to stop governor\n", __func__);
1270                         return ret;
1271                 }
1272         }
1273
1274 #ifdef CONFIG_HOTPLUG_CPU
1275         if (!cpufreq_driver->setpolicy)
1276                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1277                         policy->governor->name, CPUFREQ_NAME_LEN);
1278 #endif
1279
1280         down_read(&policy->rwsem);
1281         cpus = cpumask_weight(policy->cpus);
1282         up_read(&policy->rwsem);
1283
1284         if (cpu != policy->cpu) {
1285                 if (!frozen)
1286                         sysfs_remove_link(&dev->kobj, "cpufreq");
1287         } else if (cpus > 1) {
1288                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1289                 if (new_cpu >= 0) {
1290                         update_policy_cpu(policy, new_cpu);
1291
1292                         if (!frozen) {
1293                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1294                                                 __func__, new_cpu, cpu);
1295                         }
1296                 }
1297         }
1298
1299         return 0;
1300 }
1301
1302 static int __cpufreq_remove_dev_finish(struct device *dev,
1303                                        struct subsys_interface *sif,
1304                                        bool frozen)
1305 {
1306         unsigned int cpu = dev->id, cpus;
1307         int ret;
1308         unsigned long flags;
1309         struct cpufreq_policy *policy;
1310
1311         read_lock_irqsave(&cpufreq_driver_lock, flags);
1312         policy = per_cpu(cpufreq_cpu_data, cpu);
1313         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1314
1315         if (!policy) {
1316                 pr_debug("%s: No cpu_data found\n", __func__);
1317                 return -EINVAL;
1318         }
1319
1320         down_write(&policy->rwsem);
1321         cpus = cpumask_weight(policy->cpus);
1322
1323         if (cpus > 1)
1324                 cpumask_clear_cpu(cpu, policy->cpus);
1325         up_write(&policy->rwsem);
1326
1327         /* If cpu is last user of policy, free policy */
1328         if (cpus == 1) {
1329                 if (has_target()) {
1330                         ret = __cpufreq_governor(policy,
1331                                         CPUFREQ_GOV_POLICY_EXIT);
1332                         if (ret) {
1333                                 pr_err("%s: Failed to exit governor\n",
1334                                                 __func__);
1335                                 return ret;
1336                         }
1337                 }
1338
1339                 if (!frozen)
1340                         cpufreq_policy_put_kobj(policy);
1341
1342                 /*
1343                  * Perform the ->exit() even during light-weight tear-down,
1344                  * since this is a core component, and is essential for the
1345                  * subsequent light-weight ->init() to succeed.
1346                  */
1347                 if (cpufreq_driver->exit)
1348                         cpufreq_driver->exit(policy);
1349
1350                 /* Remove policy from list of active policies */
1351                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1352                 list_del(&policy->policy_list);
1353                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1354
1355                 if (!frozen)
1356                         cpufreq_policy_free(policy);
1357         } else {
1358                 if (has_target()) {
1359                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1360                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1361                                 pr_err("%s: Failed to start governor\n",
1362                                                 __func__);
1363                                 return ret;
1364                         }
1365                 }
1366         }
1367
1368         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1369         return 0;
1370 }
1371
1372 /**
1373  * cpufreq_remove_dev - remove a CPU device
1374  *
1375  * Removes the cpufreq interface for a CPU device.
1376  */
1377 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1378 {
1379         unsigned int cpu = dev->id;
1380         int ret;
1381
1382         if (cpu_is_offline(cpu))
1383                 return 0;
1384
1385         ret = __cpufreq_remove_dev_prepare(dev, sif, false);
1386
1387         if (!ret)
1388                 ret = __cpufreq_remove_dev_finish(dev, sif, false);
1389
1390         return ret;
1391 }
1392
1393 static void handle_update(struct work_struct *work)
1394 {
1395         struct cpufreq_policy *policy =
1396                 container_of(work, struct cpufreq_policy, update);
1397         unsigned int cpu = policy->cpu;
1398         pr_debug("handle_update for cpu %u called\n", cpu);
1399         cpufreq_update_policy(cpu);
1400 }
1401
1402 /**
1403  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1404  *      in deep trouble.
1405  *      @cpu: cpu number
1406  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1407  *      @new_freq: CPU frequency the CPU actually runs at
1408  *
1409  *      We adjust to current frequency first, and need to clean up later.
1410  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1411  */
1412 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1413                                 unsigned int new_freq)
1414 {
1415         struct cpufreq_policy *policy;
1416         struct cpufreq_freqs freqs;
1417         unsigned long flags;
1418
1419         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1420                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1421
1422         freqs.old = old_freq;
1423         freqs.new = new_freq;
1424
1425         read_lock_irqsave(&cpufreq_driver_lock, flags);
1426         policy = per_cpu(cpufreq_cpu_data, cpu);
1427         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1428
1429         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1430         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1431 }
1432
1433 /**
1434  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1435  * @cpu: CPU number
1436  *
1437  * This is the last known freq, without actually getting it from the driver.
1438  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1439  */
1440 unsigned int cpufreq_quick_get(unsigned int cpu)
1441 {
1442         struct cpufreq_policy *policy;
1443         unsigned int ret_freq = 0;
1444
1445         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1446                 return cpufreq_driver->get(cpu);
1447
1448         policy = cpufreq_cpu_get(cpu);
1449         if (policy) {
1450                 ret_freq = policy->cur;
1451                 cpufreq_cpu_put(policy);
1452         }
1453
1454         return ret_freq;
1455 }
1456 EXPORT_SYMBOL(cpufreq_quick_get);
1457
1458 /**
1459  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1460  * @cpu: CPU number
1461  *
1462  * Just return the max possible frequency for a given CPU.
1463  */
1464 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1465 {
1466         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1467         unsigned int ret_freq = 0;
1468
1469         if (policy) {
1470                 ret_freq = policy->max;
1471                 cpufreq_cpu_put(policy);
1472         }
1473
1474         return ret_freq;
1475 }
1476 EXPORT_SYMBOL(cpufreq_quick_get_max);
1477
1478 static unsigned int __cpufreq_get(unsigned int cpu)
1479 {
1480         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1481         unsigned int ret_freq = 0;
1482
1483         if (!cpufreq_driver->get)
1484                 return ret_freq;
1485
1486         ret_freq = cpufreq_driver->get(cpu);
1487
1488         if (ret_freq && policy->cur &&
1489                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1490                 /* verify no discrepancy between actual and
1491                                         saved value exists */
1492                 if (unlikely(ret_freq != policy->cur)) {
1493                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1494                         schedule_work(&policy->update);
1495                 }
1496         }
1497
1498         return ret_freq;
1499 }
1500
1501 /**
1502  * cpufreq_get - get the current CPU frequency (in kHz)
1503  * @cpu: CPU number
1504  *
1505  * Get the CPU current (static) CPU frequency
1506  */
1507 unsigned int cpufreq_get(unsigned int cpu)
1508 {
1509         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1510         unsigned int ret_freq = 0;
1511
1512         if (cpufreq_disabled() || !cpufreq_driver)
1513                 return -ENOENT;
1514
1515         BUG_ON(!policy);
1516
1517         if (!down_read_trylock(&cpufreq_rwsem))
1518                 return 0;
1519
1520         down_read(&policy->rwsem);
1521
1522         ret_freq = __cpufreq_get(cpu);
1523
1524         up_read(&policy->rwsem);
1525         up_read(&cpufreq_rwsem);
1526
1527         return ret_freq;
1528 }
1529 EXPORT_SYMBOL(cpufreq_get);
1530
1531 static struct subsys_interface cpufreq_interface = {
1532         .name           = "cpufreq",
1533         .subsys         = &cpu_subsys,
1534         .add_dev        = cpufreq_add_dev,
1535         .remove_dev     = cpufreq_remove_dev,
1536 };
1537
1538 /**
1539  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1540  *
1541  * This function is only executed for the boot processor.  The other CPUs
1542  * have been put offline by means of CPU hotplug.
1543  */
1544 static int cpufreq_bp_suspend(void)
1545 {
1546         int ret = 0;
1547
1548         int cpu = smp_processor_id();
1549         struct cpufreq_policy *policy;
1550
1551         pr_debug("suspending cpu %u\n", cpu);
1552
1553         /* If there's no policy for the boot CPU, we have nothing to do. */
1554         policy = cpufreq_cpu_get(cpu);
1555         if (!policy)
1556                 return 0;
1557
1558         if (cpufreq_driver->suspend) {
1559                 ret = cpufreq_driver->suspend(policy);
1560                 if (ret)
1561                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1562                                         "step on CPU %u\n", policy->cpu);
1563         }
1564
1565         cpufreq_cpu_put(policy);
1566         return ret;
1567 }
1568
1569 /**
1570  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1571  *
1572  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1573  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1574  *          restored. It will verify that the current freq is in sync with
1575  *          what we believe it to be. This is a bit later than when it
1576  *          should be, but nonethteless it's better than calling
1577  *          cpufreq_driver->get() here which might re-enable interrupts...
1578  *
1579  * This function is only executed for the boot CPU.  The other CPUs have not
1580  * been turned on yet.
1581  */
1582 static void cpufreq_bp_resume(void)
1583 {
1584         int ret = 0;
1585
1586         int cpu = smp_processor_id();
1587         struct cpufreq_policy *policy;
1588
1589         pr_debug("resuming cpu %u\n", cpu);
1590
1591         /* If there's no policy for the boot CPU, we have nothing to do. */
1592         policy = cpufreq_cpu_get(cpu);
1593         if (!policy)
1594                 return;
1595
1596         if (cpufreq_driver->resume) {
1597                 ret = cpufreq_driver->resume(policy);
1598                 if (ret) {
1599                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1600                                         "step on CPU %u\n", policy->cpu);
1601                         goto fail;
1602                 }
1603         }
1604
1605         schedule_work(&policy->update);
1606
1607 fail:
1608         cpufreq_cpu_put(policy);
1609 }
1610
1611 static struct syscore_ops cpufreq_syscore_ops = {
1612         .suspend        = cpufreq_bp_suspend,
1613         .resume         = cpufreq_bp_resume,
1614 };
1615
1616 /**
1617  *      cpufreq_get_current_driver - return current driver's name
1618  *
1619  *      Return the name string of the currently loaded cpufreq driver
1620  *      or NULL, if none.
1621  */
1622 const char *cpufreq_get_current_driver(void)
1623 {
1624         if (cpufreq_driver)
1625                 return cpufreq_driver->name;
1626
1627         return NULL;
1628 }
1629 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1630
1631 /*********************************************************************
1632  *                     NOTIFIER LISTS INTERFACE                      *
1633  *********************************************************************/
1634
1635 /**
1636  *      cpufreq_register_notifier - register a driver with cpufreq
1637  *      @nb: notifier function to register
1638  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1639  *
1640  *      Add a driver to one of two lists: either a list of drivers that
1641  *      are notified about clock rate changes (once before and once after
1642  *      the transition), or a list of drivers that are notified about
1643  *      changes in cpufreq policy.
1644  *
1645  *      This function may sleep, and has the same return conditions as
1646  *      blocking_notifier_chain_register.
1647  */
1648 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1649 {
1650         int ret;
1651
1652         if (cpufreq_disabled())
1653                 return -EINVAL;
1654
1655         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1656
1657         switch (list) {
1658         case CPUFREQ_TRANSITION_NOTIFIER:
1659                 ret = srcu_notifier_chain_register(
1660                                 &cpufreq_transition_notifier_list, nb);
1661                 break;
1662         case CPUFREQ_POLICY_NOTIFIER:
1663                 ret = blocking_notifier_chain_register(
1664                                 &cpufreq_policy_notifier_list, nb);
1665                 break;
1666         default:
1667                 ret = -EINVAL;
1668         }
1669
1670         return ret;
1671 }
1672 EXPORT_SYMBOL(cpufreq_register_notifier);
1673
1674 /**
1675  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1676  *      @nb: notifier block to be unregistered
1677  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1678  *
1679  *      Remove a driver from the CPU frequency notifier list.
1680  *
1681  *      This function may sleep, and has the same return conditions as
1682  *      blocking_notifier_chain_unregister.
1683  */
1684 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1685 {
1686         int ret;
1687
1688         if (cpufreq_disabled())
1689                 return -EINVAL;
1690
1691         switch (list) {
1692         case CPUFREQ_TRANSITION_NOTIFIER:
1693                 ret = srcu_notifier_chain_unregister(
1694                                 &cpufreq_transition_notifier_list, nb);
1695                 break;
1696         case CPUFREQ_POLICY_NOTIFIER:
1697                 ret = blocking_notifier_chain_unregister(
1698                                 &cpufreq_policy_notifier_list, nb);
1699                 break;
1700         default:
1701                 ret = -EINVAL;
1702         }
1703
1704         return ret;
1705 }
1706 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1707
1708
1709 /*********************************************************************
1710  *                              GOVERNORS                            *
1711  *********************************************************************/
1712
1713 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1714                             unsigned int target_freq,
1715                             unsigned int relation)
1716 {
1717         int retval = -EINVAL;
1718         unsigned int old_target_freq = target_freq;
1719
1720         if (cpufreq_disabled())
1721                 return -ENODEV;
1722
1723         /* Make sure that target_freq is within supported range */
1724         if (target_freq > policy->max)
1725                 target_freq = policy->max;
1726         if (target_freq < policy->min)
1727                 target_freq = policy->min;
1728
1729         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1730                         policy->cpu, target_freq, relation, old_target_freq);
1731
1732         /*
1733          * This might look like a redundant call as we are checking it again
1734          * after finding index. But it is left intentionally for cases where
1735          * exactly same freq is called again and so we can save on few function
1736          * calls.
1737          */
1738         if (target_freq == policy->cur)
1739                 return 0;
1740
1741         if (cpufreq_driver->target)
1742                 retval = cpufreq_driver->target(policy, target_freq, relation);
1743         else if (cpufreq_driver->target_index) {
1744                 struct cpufreq_frequency_table *freq_table;
1745                 struct cpufreq_freqs freqs;
1746                 bool notify;
1747                 int index;
1748
1749                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1750                 if (unlikely(!freq_table)) {
1751                         pr_err("%s: Unable to find freq_table\n", __func__);
1752                         goto out;
1753                 }
1754
1755                 retval = cpufreq_frequency_table_target(policy, freq_table,
1756                                 target_freq, relation, &index);
1757                 if (unlikely(retval)) {
1758                         pr_err("%s: Unable to find matching freq\n", __func__);
1759                         goto out;
1760                 }
1761
1762                 if (freq_table[index].frequency == policy->cur) {
1763                         retval = 0;
1764                         goto out;
1765                 }
1766
1767                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1768
1769                 if (notify) {
1770                         freqs.old = policy->cur;
1771                         freqs.new = freq_table[index].frequency;
1772                         freqs.flags = 0;
1773
1774                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1775                                         __func__, policy->cpu, freqs.old,
1776                                         freqs.new);
1777
1778                         cpufreq_notify_transition(policy, &freqs,
1779                                         CPUFREQ_PRECHANGE);
1780                 }
1781
1782                 retval = cpufreq_driver->target_index(policy, index);
1783                 if (retval)
1784                         pr_err("%s: Failed to change cpu frequency: %d\n",
1785                                         __func__, retval);
1786
1787                 if (notify)
1788                         cpufreq_notify_post_transition(policy, &freqs, retval);
1789         }
1790
1791 out:
1792         return retval;
1793 }
1794 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1795
1796 int cpufreq_driver_target(struct cpufreq_policy *policy,
1797                           unsigned int target_freq,
1798                           unsigned int relation)
1799 {
1800         int ret = -EINVAL;
1801
1802         down_write(&policy->rwsem);
1803
1804         ret = __cpufreq_driver_target(policy, target_freq, relation);
1805
1806         up_write(&policy->rwsem);
1807
1808         return ret;
1809 }
1810 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1811
1812 /*
1813  * when "event" is CPUFREQ_GOV_LIMITS
1814  */
1815
1816 static int __cpufreq_governor(struct cpufreq_policy *policy,
1817                                         unsigned int event)
1818 {
1819         int ret;
1820
1821         /* Only must be defined when default governor is known to have latency
1822            restrictions, like e.g. conservative or ondemand.
1823            That this is the case is already ensured in Kconfig
1824         */
1825 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1826         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1827 #else
1828         struct cpufreq_governor *gov = NULL;
1829 #endif
1830
1831         if (policy->governor->max_transition_latency &&
1832             policy->cpuinfo.transition_latency >
1833             policy->governor->max_transition_latency) {
1834                 if (!gov)
1835                         return -EINVAL;
1836                 else {
1837                         printk(KERN_WARNING "%s governor failed, too long"
1838                                " transition latency of HW, fallback"
1839                                " to %s governor\n",
1840                                policy->governor->name,
1841                                gov->name);
1842                         policy->governor = gov;
1843                 }
1844         }
1845
1846         if (event == CPUFREQ_GOV_POLICY_INIT)
1847                 if (!try_module_get(policy->governor->owner))
1848                         return -EINVAL;
1849
1850         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1851                                                 policy->cpu, event);
1852
1853         mutex_lock(&cpufreq_governor_lock);
1854         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1855             || (!policy->governor_enabled
1856             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1857                 mutex_unlock(&cpufreq_governor_lock);
1858                 return -EBUSY;
1859         }
1860
1861         if (event == CPUFREQ_GOV_STOP)
1862                 policy->governor_enabled = false;
1863         else if (event == CPUFREQ_GOV_START)
1864                 policy->governor_enabled = true;
1865
1866         mutex_unlock(&cpufreq_governor_lock);
1867
1868         ret = policy->governor->governor(policy, event);
1869
1870         if (!ret) {
1871                 if (event == CPUFREQ_GOV_POLICY_INIT)
1872                         policy->governor->initialized++;
1873                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1874                         policy->governor->initialized--;
1875         } else {
1876                 /* Restore original values */
1877                 mutex_lock(&cpufreq_governor_lock);
1878                 if (event == CPUFREQ_GOV_STOP)
1879                         policy->governor_enabled = true;
1880                 else if (event == CPUFREQ_GOV_START)
1881                         policy->governor_enabled = false;
1882                 mutex_unlock(&cpufreq_governor_lock);
1883         }
1884
1885         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1886                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1887                 module_put(policy->governor->owner);
1888
1889         return ret;
1890 }
1891
1892 int cpufreq_register_governor(struct cpufreq_governor *governor)
1893 {
1894         int err;
1895
1896         if (!governor)
1897                 return -EINVAL;
1898
1899         if (cpufreq_disabled())
1900                 return -ENODEV;
1901
1902         mutex_lock(&cpufreq_governor_mutex);
1903
1904         governor->initialized = 0;
1905         err = -EBUSY;
1906         if (__find_governor(governor->name) == NULL) {
1907                 err = 0;
1908                 list_add(&governor->governor_list, &cpufreq_governor_list);
1909         }
1910
1911         mutex_unlock(&cpufreq_governor_mutex);
1912         return err;
1913 }
1914 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1915
1916 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1917 {
1918 #ifdef CONFIG_HOTPLUG_CPU
1919         int cpu;
1920 #endif
1921
1922         if (!governor)
1923                 return;
1924
1925         if (cpufreq_disabled())
1926                 return;
1927
1928 #ifdef CONFIG_HOTPLUG_CPU
1929         for_each_present_cpu(cpu) {
1930                 if (cpu_online(cpu))
1931                         continue;
1932                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1933                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1934         }
1935 #endif
1936
1937         mutex_lock(&cpufreq_governor_mutex);
1938         list_del(&governor->governor_list);
1939         mutex_unlock(&cpufreq_governor_mutex);
1940         return;
1941 }
1942 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1943
1944
1945 /*********************************************************************
1946  *                          POLICY INTERFACE                         *
1947  *********************************************************************/
1948
1949 /**
1950  * cpufreq_get_policy - get the current cpufreq_policy
1951  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1952  *      is written
1953  *
1954  * Reads the current cpufreq policy.
1955  */
1956 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1957 {
1958         struct cpufreq_policy *cpu_policy;
1959         if (!policy)
1960                 return -EINVAL;
1961
1962         cpu_policy = cpufreq_cpu_get(cpu);
1963         if (!cpu_policy)
1964                 return -EINVAL;
1965
1966         memcpy(policy, cpu_policy, sizeof(*policy));
1967
1968         cpufreq_cpu_put(cpu_policy);
1969         return 0;
1970 }
1971 EXPORT_SYMBOL(cpufreq_get_policy);
1972
1973 /*
1974  * policy : current policy.
1975  * new_policy: policy to be set.
1976  */
1977 static int cpufreq_set_policy(struct cpufreq_policy *policy,
1978                                 struct cpufreq_policy *new_policy)
1979 {
1980         int ret = 0, failed = 1;
1981
1982         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
1983                 new_policy->min, new_policy->max);
1984
1985         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
1986
1987         if (new_policy->min > policy->max || new_policy->max < policy->min) {
1988                 ret = -EINVAL;
1989                 goto error_out;
1990         }
1991
1992         /* verify the cpu speed can be set within this limit */
1993         ret = cpufreq_driver->verify(new_policy);
1994         if (ret)
1995                 goto error_out;
1996
1997         /* adjust if necessary - all reasons */
1998         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1999                         CPUFREQ_ADJUST, new_policy);
2000
2001         /* adjust if necessary - hardware incompatibility*/
2002         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2003                         CPUFREQ_INCOMPATIBLE, new_policy);
2004
2005         /*
2006          * verify the cpu speed can be set within this limit, which might be
2007          * different to the first one
2008          */
2009         ret = cpufreq_driver->verify(new_policy);
2010         if (ret)
2011                 goto error_out;
2012
2013         /* notification of the new policy */
2014         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2015                         CPUFREQ_NOTIFY, new_policy);
2016
2017         policy->min = new_policy->min;
2018         policy->max = new_policy->max;
2019
2020         pr_debug("new min and max freqs are %u - %u kHz\n",
2021                                         policy->min, policy->max);
2022
2023         if (cpufreq_driver->setpolicy) {
2024                 policy->policy = new_policy->policy;
2025                 pr_debug("setting range\n");
2026                 ret = cpufreq_driver->setpolicy(new_policy);
2027         } else {
2028                 if (new_policy->governor != policy->governor) {
2029                         /* save old, working values */
2030                         struct cpufreq_governor *old_gov = policy->governor;
2031
2032                         pr_debug("governor switch\n");
2033
2034                         /* end old governor */
2035                         if (policy->governor) {
2036                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2037                                 up_write(&policy->rwsem);
2038                                 __cpufreq_governor(policy,
2039                                                 CPUFREQ_GOV_POLICY_EXIT);
2040                                 down_write(&policy->rwsem);
2041                         }
2042
2043                         /* start new governor */
2044                         policy->governor = new_policy->governor;
2045                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2046                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
2047                                         failed = 0;
2048                                 } else {
2049                                         up_write(&policy->rwsem);
2050                                         __cpufreq_governor(policy,
2051                                                         CPUFREQ_GOV_POLICY_EXIT);
2052                                         down_write(&policy->rwsem);
2053                                 }
2054                         }
2055
2056                         if (failed) {
2057                                 /* new governor failed, so re-start old one */
2058                                 pr_debug("starting governor %s failed\n",
2059                                                         policy->governor->name);
2060                                 if (old_gov) {
2061                                         policy->governor = old_gov;
2062                                         __cpufreq_governor(policy,
2063                                                         CPUFREQ_GOV_POLICY_INIT);
2064                                         __cpufreq_governor(policy,
2065                                                            CPUFREQ_GOV_START);
2066                                 }
2067                                 ret = -EINVAL;
2068                                 goto error_out;
2069                         }
2070                         /* might be a policy change, too, so fall through */
2071                 }
2072                 pr_debug("governor: change or update limits\n");
2073                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2074         }
2075
2076 error_out:
2077         return ret;
2078 }
2079
2080 /**
2081  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2082  *      @cpu: CPU which shall be re-evaluated
2083  *
2084  *      Useful for policy notifiers which have different necessities
2085  *      at different times.
2086  */
2087 int cpufreq_update_policy(unsigned int cpu)
2088 {
2089         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2090         struct cpufreq_policy new_policy;
2091         int ret;
2092
2093         if (!policy) {
2094                 ret = -ENODEV;
2095                 goto no_policy;
2096         }
2097
2098         down_write(&policy->rwsem);
2099
2100         pr_debug("updating policy for CPU %u\n", cpu);
2101         memcpy(&new_policy, policy, sizeof(*policy));
2102         new_policy.min = policy->user_policy.min;
2103         new_policy.max = policy->user_policy.max;
2104         new_policy.policy = policy->user_policy.policy;
2105         new_policy.governor = policy->user_policy.governor;
2106
2107         /*
2108          * BIOS might change freq behind our back
2109          * -> ask driver for current freq and notify governors about a change
2110          */
2111         if (cpufreq_driver->get) {
2112                 new_policy.cur = cpufreq_driver->get(cpu);
2113                 if (!policy->cur) {
2114                         pr_debug("Driver did not initialize current freq");
2115                         policy->cur = new_policy.cur;
2116                 } else {
2117                         if (policy->cur != new_policy.cur && has_target())
2118                                 cpufreq_out_of_sync(cpu, policy->cur,
2119                                                                 new_policy.cur);
2120                 }
2121         }
2122
2123         ret = cpufreq_set_policy(policy, &new_policy);
2124
2125         up_write(&policy->rwsem);
2126
2127         cpufreq_cpu_put(policy);
2128 no_policy:
2129         return ret;
2130 }
2131 EXPORT_SYMBOL(cpufreq_update_policy);
2132
2133 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2134                                         unsigned long action, void *hcpu)
2135 {
2136         unsigned int cpu = (unsigned long)hcpu;
2137         struct device *dev;
2138         bool frozen = false;
2139
2140         dev = get_cpu_device(cpu);
2141         if (dev) {
2142
2143                 if (action & CPU_TASKS_FROZEN)
2144                         frozen = true;
2145
2146                 switch (action & ~CPU_TASKS_FROZEN) {
2147                 case CPU_ONLINE:
2148                         __cpufreq_add_dev(dev, NULL, frozen);
2149                         cpufreq_update_policy(cpu);
2150                         break;
2151
2152                 case CPU_DOWN_PREPARE:
2153                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2154                         break;
2155
2156                 case CPU_POST_DEAD:
2157                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2158                         break;
2159
2160                 case CPU_DOWN_FAILED:
2161                         __cpufreq_add_dev(dev, NULL, frozen);
2162                         break;
2163                 }
2164         }
2165         return NOTIFY_OK;
2166 }
2167
2168 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2169         .notifier_call = cpufreq_cpu_callback,
2170 };
2171
2172 /*********************************************************************
2173  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2174  *********************************************************************/
2175
2176 /**
2177  * cpufreq_register_driver - register a CPU Frequency driver
2178  * @driver_data: A struct cpufreq_driver containing the values#
2179  * submitted by the CPU Frequency driver.
2180  *
2181  * Registers a CPU Frequency driver to this core code. This code
2182  * returns zero on success, -EBUSY when another driver got here first
2183  * (and isn't unregistered in the meantime).
2184  *
2185  */
2186 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2187 {
2188         unsigned long flags;
2189         int ret;
2190
2191         if (cpufreq_disabled())
2192                 return -ENODEV;
2193
2194         if (!driver_data || !driver_data->verify || !driver_data->init ||
2195             !(driver_data->setpolicy || driver_data->target_index ||
2196                     driver_data->target))
2197                 return -EINVAL;
2198
2199         pr_debug("trying to register driver %s\n", driver_data->name);
2200
2201         if (driver_data->setpolicy)
2202                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2203
2204         write_lock_irqsave(&cpufreq_driver_lock, flags);
2205         if (cpufreq_driver) {
2206                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2207                 return -EEXIST;
2208         }
2209         cpufreq_driver = driver_data;
2210         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2211
2212         ret = subsys_interface_register(&cpufreq_interface);
2213         if (ret)
2214                 goto err_null_driver;
2215
2216         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2217                 int i;
2218                 ret = -ENODEV;
2219
2220                 /* check for at least one working CPU */
2221                 for (i = 0; i < nr_cpu_ids; i++)
2222                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2223                                 ret = 0;
2224                                 break;
2225                         }
2226
2227                 /* if all ->init() calls failed, unregister */
2228                 if (ret) {
2229                         pr_debug("no CPU initialized for driver %s\n",
2230                                                         driver_data->name);
2231                         goto err_if_unreg;
2232                 }
2233         }
2234
2235         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2236         pr_debug("driver %s up and running\n", driver_data->name);
2237
2238         return 0;
2239 err_if_unreg:
2240         subsys_interface_unregister(&cpufreq_interface);
2241 err_null_driver:
2242         write_lock_irqsave(&cpufreq_driver_lock, flags);
2243         cpufreq_driver = NULL;
2244         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2245         return ret;
2246 }
2247 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2248
2249 /**
2250  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2251  *
2252  * Unregister the current CPUFreq driver. Only call this if you have
2253  * the right to do so, i.e. if you have succeeded in initialising before!
2254  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2255  * currently not initialised.
2256  */
2257 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2258 {
2259         unsigned long flags;
2260
2261         if (!cpufreq_driver || (driver != cpufreq_driver))
2262                 return -EINVAL;
2263
2264         pr_debug("unregistering driver %s\n", driver->name);
2265
2266         subsys_interface_unregister(&cpufreq_interface);
2267         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2268
2269         down_write(&cpufreq_rwsem);
2270         write_lock_irqsave(&cpufreq_driver_lock, flags);
2271
2272         cpufreq_driver = NULL;
2273
2274         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2275         up_write(&cpufreq_rwsem);
2276
2277         return 0;
2278 }
2279 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2280
2281 static int __init cpufreq_core_init(void)
2282 {
2283         if (cpufreq_disabled())
2284                 return -ENODEV;
2285
2286         cpufreq_global_kobject = kobject_create();
2287         BUG_ON(!cpufreq_global_kobject);
2288         register_syscore_ops(&cpufreq_syscore_ops);
2289
2290         return 0;
2291 }
2292 core_initcall(cpufreq_core_init);