drivers/base/power/main.c

   1 /*
   2  * drivers/base/power/main.c - Where the driver meets power management.
   3  *
   4  * Copyright (c) 2003 Patrick Mochel
   5  * Copyright (c) 2003 Open Source Development Lab
   6  *
   7  * This file is released under the GPLv2
   8  *
   9  *
  10  * The driver model core calls device_pm_add() when a device is registered.
  11  * This will initialize the embedded device_pm_info object in the device
  12  * and add it to the list of power-controlled devices. sysfs entries for
  13  * controlling device power management will also be added.
  14  *
  15  * A separate list is used for keeping track of power info, because the power
  16  * domain dependencies may differ from the ancestral dependencies that the
  17  * subsystem list maintains.
  18  */
  19
  20 #include <linux/device.h>
  21 #include <linux/kallsyms.h>
  22 #include <linux/export.h>
  23 #include <linux/mutex.h>
  24 #include <linux/pm.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/resume-trace.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/sched.h>
  29 #include <linux/async.h>
  30 #include <linux/suspend.h>
  31 #include <trace/events/power.h>
  32 #include <linux/cpufreq.h>
  33 #include <linux/cpuidle.h>
  34 #include <linux/timer.h>
  35
  36 #include "../base.h"
  37 #include "power.h"
  38
  39 typedef int (*pm_callback_t)(struct device *);
  40
  41 /*
  42  * The entries in the dpm_list list are in a depth first order, simply
  43  * because children are guaranteed to be discovered after parents, and
  44  * are inserted at the back of the list on discovery.
  45  *
  46  * Since device_pm_add() may be called with a device lock held,
  47  * we must never try to acquire a device lock while holding
  48  * dpm_list_mutex.
  49  */
  50
  51 LIST_HEAD(dpm_list);
  52 static LIST_HEAD(dpm_prepared_list);
  53 static LIST_HEAD(dpm_suspended_list);
  54 static LIST_HEAD(dpm_late_early_list);
  55 static LIST_HEAD(dpm_noirq_list);
  56
  57 struct suspend_stats suspend_stats;
  58 static DEFINE_MUTEX(dpm_list_mtx);
  59 static pm_message_t pm_transition;
  60
  61 static int async_error;
  62
  63 static char *pm_verb(int event)
  64 {
  65         switch (event) {
  66         case PM_EVENT_SUSPEND:
  67                 return "suspend";
  68         case PM_EVENT_RESUME:
  69                 return "resume";
  70         case PM_EVENT_FREEZE:
  71                 return "freeze";
  72         case PM_EVENT_QUIESCE:
  73                 return "quiesce";
  74         case PM_EVENT_HIBERNATE:
  75                 return "hibernate";
  76         case PM_EVENT_THAW:
  77                 return "thaw";
  78         case PM_EVENT_RESTORE:
  79                 return "restore";
  80         case PM_EVENT_RECOVER:
  81                 return "recover";
  82         default:
  83                 return "(unknown PM event)";
  84         }
  85 }
  86
  87 /**
  88  * device_pm_sleep_init - Initialize system suspend-related device fields.
  89  * @dev: Device object being initialized.
  90  */
  91 void device_pm_sleep_init(struct device *dev)
  92 {
  93         dev->power.is_prepared = false;
  94         dev->power.is_suspended = false;
  95         dev->power.is_noirq_suspended = false;
  96         dev->power.is_late_suspended = false;
  97         init_completion(&dev->power.completion);
  98         complete_all(&dev->power.completion);
  99         dev->power.wakeup = NULL;
 100         INIT_LIST_HEAD(&dev->power.entry);
 101 }
 102
 103 /**
 104  * device_pm_lock - Lock the list of active devices used by the PM core.
 105  */
 106 void device_pm_lock(void)
 107 {
 108         mutex_lock(&dpm_list_mtx);
 109 }
 110
 111 /**
 112  * device_pm_unlock - Unlock the list of active devices used by the PM core.
 113  */
 114 void device_pm_unlock(void)
 115 {
 116         mutex_unlock(&dpm_list_mtx);
 117 }
 118
 119 /**
 120  * device_pm_add - Add a device to the PM core's list of active devices.
 121  * @dev: Device to add to the list.
 122  */
 123 void device_pm_add(struct device *dev)
 124 {
 125         pr_debug("PM: Adding info for %s:%s\n",
 126                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 127         mutex_lock(&dpm_list_mtx);
 128         if (dev->parent && dev->parent->power.is_prepared)
 129                 dev_warn(dev, "parent %s should not be sleeping\n",
 130                         dev_name(dev->parent));
 131         list_add_tail(&dev->power.entry, &dpm_list);
 132         mutex_unlock(&dpm_list_mtx);
 133 }
 134
 135 /**
 136  * device_pm_remove - Remove a device from the PM core's list of active devices.
 137  * @dev: Device to be removed from the list.
 138  */
 139 void device_pm_remove(struct device *dev)
 140 {
 141         pr_debug("PM: Removing info for %s:%s\n",
 142                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 143         complete_all(&dev->power.completion);
 144         mutex_lock(&dpm_list_mtx);
 145         list_del_init(&dev->power.entry);
 146         mutex_unlock(&dpm_list_mtx);
 147         device_wakeup_disable(dev);
 148         pm_runtime_remove(dev);
 149 }
 150
 151 /**
 152  * device_pm_move_before - Move device in the PM core's list of active devices.
 153  * @deva: Device to move in dpm_list.
 154  * @devb: Device @deva should come before.
 155  */
 156 void device_pm_move_before(struct device *deva, struct device *devb)
 157 {
 158         pr_debug("PM: Moving %s:%s before %s:%s\n",
 159                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 160                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 161         /* Delete deva from dpm_list and reinsert before devb. */
 162         list_move_tail(&deva->power.entry, &devb->power.entry);
 163 }
 164
 165 /**
 166  * device_pm_move_after - Move device in the PM core's list of active devices.
 167  * @deva: Device to move in dpm_list.
 168  * @devb: Device @deva should come after.
 169  */
 170 void device_pm_move_after(struct device *deva, struct device *devb)
 171 {
 172         pr_debug("PM: Moving %s:%s after %s:%s\n",
 173                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 174                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 175         /* Delete deva from dpm_list and reinsert after devb. */
 176         list_move(&deva->power.entry, &devb->power.entry);
 177 }
 178
 179 /**
 180  * device_pm_move_last - Move device to end of the PM core's list of devices.
 181  * @dev: Device to move in dpm_list.
 182  */
 183 void device_pm_move_last(struct device *dev)
 184 {
 185         pr_debug("PM: Moving %s:%s to end of list\n",
 186                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 187         list_move_tail(&dev->power.entry, &dpm_list);
 188 }
 189
 190 static ktime_t initcall_debug_start(struct device *dev)
 191 {
 192         ktime_t calltime = ktime_set(0, 0);
 193
 194         if (pm_print_times_enabled) {
 195                 pr_info("calling  %s+ @ %i, parent: %s\n",
 196                         dev_name(dev), task_pid_nr(current),
 197                         dev->parent ? dev_name(dev->parent) : "none");
 198                 calltime = ktime_get();
 199         }
 200
 201         return calltime;
 202 }
 203
 204 static void initcall_debug_report(struct device *dev, ktime_t calltime,
 205                                   int error, pm_message_t state, char *info)
 206 {
 207         ktime_t rettime;
 208         s64 nsecs;
 209
 210         rettime = ktime_get();
 211         nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
 212
 213         if (pm_print_times_enabled) {
 214                 pr_info("call %s+ returned %d after %Ld usecs\n", dev_name(dev),
 215                         error, (unsigned long long)nsecs >> 10);
 216         }
 217 }
 218
 219 /**
 220  * dpm_wait - Wait for a PM operation to complete.
 221  * @dev: Device to wait for.
 222  * @async: If unset, wait only if the device's power.async_suspend flag is set.
 223  */
 224 static void dpm_wait(struct device *dev, bool async)
 225 {
 226         if (!dev)
 227                 return;
 228
 229         if (async || (pm_async_enabled && dev->power.async_suspend))
 230                 wait_for_completion(&dev->power.completion);
 231 }
 232
 233 static int dpm_wait_fn(struct device *dev, void *async_ptr)
 234 {
 235         dpm_wait(dev, *((bool *)async_ptr));
 236         return 0;
 237 }
 238
 239 static void dpm_wait_for_children(struct device *dev, bool async)
 240 {
 241        device_for_each_child(dev, &async, dpm_wait_fn);
 242 }
 243
 244 /**
 245  * pm_op - Return the PM operation appropriate for given PM event.
 246  * @ops: PM operations to choose from.
 247  * @state: PM transition of the system being carried out.
 248  */
 249 static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
 250 {
 251         switch (state.event) {
 252 #ifdef CONFIG_SUSPEND
 253         case PM_EVENT_SUSPEND:
 254                 return ops->suspend;
 255         case PM_EVENT_RESUME:
 256                 return ops->resume;
 257 #endif /* CONFIG_SUSPEND */
 258 #ifdef CONFIG_HIBERNATE_CALLBACKS
 259         case PM_EVENT_FREEZE:
 260         case PM_EVENT_QUIESCE:
 261                 return ops->freeze;
 262         case PM_EVENT_HIBERNATE:
 263                 return ops->poweroff;
 264         case PM_EVENT_THAW:
 265         case PM_EVENT_RECOVER:
 266                 return ops->thaw;
 267                 break;
 268         case PM_EVENT_RESTORE:
 269                 return ops->restore;
 270 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 271         }
 272
 273         return NULL;
 274 }
 275
 276 /**
 277  * pm_late_early_op - Return the PM operation appropriate for given PM event.
 278  * @ops: PM operations to choose from.
 279  * @state: PM transition of the system being carried out.
 280  *
 281  * Runtime PM is disabled for @dev while this function is being executed.
 282  */
 283 static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
 284                                       pm_message_t state)
 285 {
 286         switch (state.event) {
 287 #ifdef CONFIG_SUSPEND
 288         case PM_EVENT_SUSPEND:
 289                 return ops->suspend_late;
 290         case PM_EVENT_RESUME:
 291                 return ops->resume_early;
 292 #endif /* CONFIG_SUSPEND */
 293 #ifdef CONFIG_HIBERNATE_CALLBACKS
 294         case PM_EVENT_FREEZE:
 295         case PM_EVENT_QUIESCE:
 296                 return ops->freeze_late;
 297         case PM_EVENT_HIBERNATE:
 298                 return ops->poweroff_late;
 299         case PM_EVENT_THAW:
 300         case PM_EVENT_RECOVER:
 301                 return ops->thaw_early;
 302         case PM_EVENT_RESTORE:
 303                 return ops->restore_early;
 304 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 305         }
 306
 307         return NULL;
 308 }
 309
 310 /**
 311  * pm_noirq_op - Return the PM operation appropriate for given PM event.
 312  * @ops: PM operations to choose from.
 313  * @state: PM transition of the system being carried out.
 314  *
 315  * The driver of @dev will not receive interrupts while this function is being
 316  * executed.
 317  */
 318 static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
 319 {
 320         switch (state.event) {
 321 #ifdef CONFIG_SUSPEND
 322         case PM_EVENT_SUSPEND:
 323                 return ops->suspend_noirq;
 324         case PM_EVENT_RESUME:
 325                 return ops->resume_noirq;
 326 #endif /* CONFIG_SUSPEND */
 327 #ifdef CONFIG_HIBERNATE_CALLBACKS
 328         case PM_EVENT_FREEZE:
 329         case PM_EVENT_QUIESCE:
 330                 return ops->freeze_noirq;
 331         case PM_EVENT_HIBERNATE:
 332                 return ops->poweroff_noirq;
 333         case PM_EVENT_THAW:
 334         case PM_EVENT_RECOVER:
 335                 return ops->thaw_noirq;
 336         case PM_EVENT_RESTORE:
 337                 return ops->restore_noirq;
 338 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 339         }
 340
 341         return NULL;
 342 }
 343
 344 static void pm_dev_dbg(struct device *dev, pm_message_t state, char *info)
 345 {
 346         dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event),
 347                 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
 348                 ", may wakeup" : "");
 349 }
 350
 351 static void pm_dev_err(struct device *dev, pm_message_t state, char *info,
 352                         int error)
 353 {
 354         printk(KERN_ERR "PM: Device %s failed to %s%s: error %d\n",
 355                 dev_name(dev), pm_verb(state.event), info, error);
 356 }
 357
 358 static void dpm_show_time(ktime_t starttime, pm_message_t state, char *info)
 359 {
 360         ktime_t calltime;
 361         u64 usecs64;
 362         int usecs;
 363
 364         calltime = ktime_get();
 365         usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
 366         do_div(usecs64, NSEC_PER_USEC);
 367         usecs = usecs64;
 368         if (usecs == 0)
 369                 usecs = 1;
 370         pr_info("PM: %s%s%s of devices complete after %ld.%03ld msecs\n",
 371                 info ?: "", info ? " " : "", pm_verb(state.event),
 372                 usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 373 }
 374
 375 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 376                             pm_message_t state, char *info)
 377 {
 378         ktime_t calltime;
 379         int error;
 380
 381         if (!cb)
 382                 return 0;
 383
 384         calltime = initcall_debug_start(dev);
 385
 386         pm_dev_dbg(dev, state, info);
 387         trace_device_pm_callback_start(dev, info, state.event);
 388         error = cb(dev);
 389         trace_device_pm_callback_end(dev, error);
 390         suspend_report_result(cb, error);
 391
 392         initcall_debug_report(dev, calltime, error, state, info);
 393
 394         return error;
 395 }
 396
 397 #ifdef CONFIG_DPM_WATCHDOG
 398 struct dpm_watchdog {
 399         struct device           *dev;
 400         struct task_struct      *tsk;
 401         struct timer_list       timer;
 402 };
 403
 404 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
 405         struct dpm_watchdog wd
 406
 407 /**
 408  * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 409  * @data: Watchdog object address.
 410  *
 411  * Called when a driver has timed out suspending or resuming.
 412  * There's not much we can do here to recover so panic() to
 413  * capture a crash-dump in pstore.
 414  */
 415 static void dpm_watchdog_handler(unsigned long data)
 416 {
 417         struct dpm_watchdog *wd = (void *)data;
 418
 419         dev_emerg(wd->dev, "**** DPM device timeout ****\n");
 420         show_stack(wd->tsk, NULL);
 421         panic("%s %s: unrecoverable failure\n",
 422                 dev_driver_string(wd->dev), dev_name(wd->dev));
 423 }
 424
 425 /**
 426  * dpm_watchdog_set - Enable pm watchdog for given device.
 427  * @wd: Watchdog. Must be allocated on the stack.
 428  * @dev: Device to handle.
 429  */
 430 static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
 431 {
 432         struct timer_list *timer = &wd->timer;
 433
 434         wd->dev = dev;
 435         wd->tsk = current;
 436
 437         init_timer_on_stack(timer);
 438         /* use same timeout value for both suspend and resume */
 439         timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
 440         timer->function = dpm_watchdog_handler;
 441         timer->data = (unsigned long)wd;
 442         add_timer(timer);
 443 }
 444
 445 /**
 446  * dpm_watchdog_clear - Disable suspend/resume watchdog.
 447  * @wd: Watchdog to disable.
 448  */
 449 static void dpm_watchdog_clear(struct dpm_watchdog *wd)
 450 {
 451         struct timer_list *timer = &wd->timer;
 452
 453         del_timer_sync(timer);
 454         destroy_timer_on_stack(timer);
 455 }
 456 #else
 457 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
 458 #define dpm_watchdog_set(x, y)
 459 #define dpm_watchdog_clear(x)
 460 #endif
 461
 462 /*------------------------- Resume routines -------------------------*/
 463
 464 /**
 465  * device_resume_noirq - Execute an "early resume" callback for given device.
 466  * @dev: Device to handle.
 467  * @state: PM transition of the system being carried out.
 468  * @async: If true, the device is being resumed asynchronously.
 469  *
 470  * The driver of @dev will not receive interrupts while this function is being
 471  * executed.
 472  */
 473 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 474 {
 475         pm_callback_t callback = NULL;
 476         char *info = NULL;
 477         int error = 0;
 478
 479         TRACE_DEVICE(dev);
 480         TRACE_RESUME(0);
 481
 482         if (dev->power.syscore || dev->power.direct_complete)
 483                 goto Out;
 484
 485         if (!dev->power.is_noirq_suspended)
 486                 goto Out;
 487
 488         dpm_wait(dev->parent, async);
 489
 490         if (dev->pm_domain) {
 491                 info = "noirq power domain ";
 492                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
 493         } else if (dev->type && dev->type->pm) {
 494                 info = "noirq type ";
 495                 callback = pm_noirq_op(dev->type->pm, state);
 496         } else if (dev->class && dev->class->pm) {
 497                 info = "noirq class ";
 498                 callback = pm_noirq_op(dev->class->pm, state);
 499         } else if (dev->bus && dev->bus->pm) {
 500                 info = "noirq bus ";
 501                 callback = pm_noirq_op(dev->bus->pm, state);
 502         }
 503
 504         if (!callback && dev->driver && dev->driver->pm) {
 505                 info = "noirq driver ";
 506                 callback = pm_noirq_op(dev->driver->pm, state);
 507         }
 508
 509         error = dpm_run_callback(callback, dev, state, info);
 510         dev->power.is_noirq_suspended = false;
 511
 512  Out:
 513         complete_all(&dev->power.completion);
 514         TRACE_RESUME(error);
 515         return error;
 516 }
 517
 518 static bool is_async(struct device *dev)
 519 {
 520         return dev->power.async_suspend && pm_async_enabled
 521                 && !pm_trace_is_enabled();
 522 }
 523
 524 static void async_resume_noirq(void *data, async_cookie_t cookie)
 525 {
 526         struct device *dev = (struct device *)data;
 527         int error;
 528
 529         error = device_resume_noirq(dev, pm_transition, true);
 530         if (error)
 531                 pm_dev_err(dev, pm_transition, " async", error);
 532
 533         put_device(dev);
 534 }
 535
 536 /**
 537  * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 538  * @state: PM transition of the system being carried out.
 539  *
 540  * Call the "noirq" resume handlers for all devices in dpm_noirq_list and
 541  * enable device drivers to receive interrupts.
 542  */
 543 static void dpm_resume_noirq(pm_message_t state)
 544 {
 545         struct device *dev;
 546         ktime_t starttime = ktime_get();
 547
 548         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
 549         mutex_lock(&dpm_list_mtx);
 550         pm_transition = state;
 551
 552         /*
 553          * Advanced the async threads upfront,
 554          * in case the starting of async threads is
 555          * delayed by non-async resuming devices.
 556          */
 557         list_for_each_entry(dev, &dpm_noirq_list, power.entry) {
 558                 reinit_completion(&dev->power.completion);
 559                 if (is_async(dev)) {
 560                         get_device(dev);
 561                         async_schedule(async_resume_noirq, dev);
 562                 }
 563         }
 564
 565         while (!list_empty(&dpm_noirq_list)) {
 566                 dev = to_device(dpm_noirq_list.next);
 567                 get_device(dev);
 568                 list_move_tail(&dev->power.entry, &dpm_late_early_list);
 569                 mutex_unlock(&dpm_list_mtx);
 570
 571                 if (!is_async(dev)) {
 572                         int error;
 573
 574                         error = device_resume_noirq(dev, state, false);
 575                         if (error) {
 576                                 suspend_stats.failed_resume_noirq++;
 577                                 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
 578                                 dpm_save_failed_dev(dev_name(dev));
 579                                 pm_dev_err(dev, state, " noirq", error);
 580                         }
 581                 }
 582
 583                 mutex_lock(&dpm_list_mtx);
 584                 put_device(dev);
 585         }
 586         mutex_unlock(&dpm_list_mtx);
 587         async_synchronize_full();
 588         dpm_show_time(starttime, state, "noirq");
 589         resume_device_irqs();
 590         cpuidle_resume();
 591         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
 592 }
 593
 594 /**
 595  * device_resume_early - Execute an "early resume" callback for given device.
 596  * @dev: Device to handle.
 597  * @state: PM transition of the system being carried out.
 598  * @async: If true, the device is being resumed asynchronously.
 599  *
 600  * Runtime PM is disabled for @dev while this function is being executed.
 601  */
 602 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 603 {
 604         pm_callback_t callback = NULL;
 605         char *info = NULL;
 606         int error = 0;
 607
 608         TRACE_DEVICE(dev);
 609         TRACE_RESUME(0);
 610
 611         if (dev->power.syscore || dev->power.direct_complete)
 612                 goto Out;
 613
 614         if (!dev->power.is_late_suspended)
 615                 goto Out;
 616
 617         dpm_wait(dev->parent, async);
 618
 619         if (dev->pm_domain) {
 620                 info = "early power domain ";
 621                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
 622         } else if (dev->type && dev->type->pm) {
 623                 info = "early type ";
 624                 callback = pm_late_early_op(dev->type->pm, state);
 625         } else if (dev->class && dev->class->pm) {
 626                 info = "early class ";
 627                 callback = pm_late_early_op(dev->class->pm, state);
 628         } else if (dev->bus && dev->bus->pm) {
 629                 info = "early bus ";
 630                 callback = pm_late_early_op(dev->bus->pm, state);
 631         }
 632
 633         if (!callback && dev->driver && dev->driver->pm) {
 634                 info = "early driver ";
 635                 callback = pm_late_early_op(dev->driver->pm, state);
 636         }
 637
 638         error = dpm_run_callback(callback, dev, state, info);
 639         dev->power.is_late_suspended = false;
 640
 641  Out:
 642         TRACE_RESUME(error);
 643
 644         pm_runtime_enable(dev);
 645         complete_all(&dev->power.completion);
 646         return error;
 647 }
 648
 649 static void async_resume_early(void *data, async_cookie_t cookie)
 650 {
 651         struct device *dev = (struct device *)data;
 652         int error;
 653
 654         error = device_resume_early(dev, pm_transition, true);
 655         if (error)
 656                 pm_dev_err(dev, pm_transition, " async", error);
 657
 658         put_device(dev);
 659 }
 660
 661 /**
 662  * dpm_resume_early - Execute "early resume" callbacks for all devices.
 663  * @state: PM transition of the system being carried out.
 664  */
 665 static void dpm_resume_early(pm_message_t state)
 666 {
 667         struct device *dev;
 668         ktime_t starttime = ktime_get();
 669
 670         trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
 671         mutex_lock(&dpm_list_mtx);
 672         pm_transition = state;
 673
 674         /*
 675          * Advanced the async threads upfront,
 676          * in case the starting of async threads is
 677          * delayed by non-async resuming devices.
 678          */
 679         list_for_each_entry(dev, &dpm_late_early_list, power.entry) {
 680                 reinit_completion(&dev->power.completion);
 681                 if (is_async(dev)) {
 682                         get_device(dev);
 683                         async_schedule(async_resume_early, dev);
 684                 }
 685         }
 686
 687         while (!list_empty(&dpm_late_early_list)) {
 688                 dev = to_device(dpm_late_early_list.next);
 689                 get_device(dev);
 690                 list_move_tail(&dev->power.entry, &dpm_suspended_list);
 691                 mutex_unlock(&dpm_list_mtx);
 692
 693                 if (!is_async(dev)) {
 694                         int error;
 695
 696                         error = device_resume_early(dev, state, false);
 697                         if (error) {
 698                                 suspend_stats.failed_resume_early++;
 699                                 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
 700                                 dpm_save_failed_dev(dev_name(dev));
 701                                 pm_dev_err(dev, state, " early", error);
 702                         }
 703                 }
 704                 mutex_lock(&dpm_list_mtx);
 705                 put_device(dev);
 706         }
 707         mutex_unlock(&dpm_list_mtx);
 708         async_synchronize_full();
 709         dpm_show_time(starttime, state, "early");
 710         trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 711 }
 712
 713 /**
 714  * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 715  * @state: PM transition of the system being carried out.
 716  */
 717 void dpm_resume_start(pm_message_t state)
 718 {
 719         dpm_resume_noirq(state);
 720         dpm_resume_early(state);
 721 }
 722 EXPORT_SYMBOL_GPL(dpm_resume_start);
 723
 724 /**
 725  * device_resume - Execute "resume" callbacks for given device.
 726  * @dev: Device to handle.
 727  * @state: PM transition of the system being carried out.
 728  * @async: If true, the device is being resumed asynchronously.
 729  */
 730 static int device_resume(struct device *dev, pm_message_t state, bool async)
 731 {
 732         pm_callback_t callback = NULL;
 733         char *info = NULL;
 734         int error = 0;
 735         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 736
 737         TRACE_DEVICE(dev);
 738         TRACE_RESUME(0);
 739
 740         if (dev->power.syscore)
 741                 goto Complete;
 742
 743         if (dev->power.direct_complete) {
 744                 /* Match the pm_runtime_disable() in __device_suspend(). */
 745                 pm_runtime_enable(dev);
 746                 goto Complete;
 747         }
 748
 749         dpm_wait(dev->parent, async);
 750         dpm_watchdog_set(&wd, dev);
 751         device_lock(dev);
 752
 753         /*
 754          * This is a fib.  But we'll allow new children to be added below
 755          * a resumed device, even if the device hasn't been completed yet.
 756          */
 757         dev->power.is_prepared = false;
 758
 759         if (!dev->power.is_suspended)
 760                 goto Unlock;
 761
 762         if (dev->pm_domain) {
 763                 info = "power domain ";
 764                 callback = pm_op(&dev->pm_domain->ops, state);
 765                 goto Driver;
 766         }
 767
 768         if (dev->type && dev->type->pm) {
 769                 info = "type ";
 770                 callback = pm_op(dev->type->pm, state);
 771                 goto Driver;
 772         }
 773
 774         if (dev->class) {
 775                 if (dev->class->pm) {
 776                         info = "class ";
 777                         callback = pm_op(dev->class->pm, state);
 778                         goto Driver;
 779                 } else if (dev->class->resume) {
 780                         info = "legacy class ";
 781                         callback = dev->class->resume;
 782                         goto End;
 783                 }
 784         }
 785
 786         if (dev->bus) {
 787                 if (dev->bus->pm) {
 788                         info = "bus ";
 789                         callback = pm_op(dev->bus->pm, state);
 790                 } else if (dev->bus->resume) {
 791                         info = "legacy bus ";
 792                         callback = dev->bus->resume;
 793                         goto End;
 794                 }
 795         }
 796
 797  Driver:
 798         if (!callback && dev->driver && dev->driver->pm) {
 799                 info = "driver ";
 800                 callback = pm_op(dev->driver->pm, state);
 801         }
 802
 803  End:
 804         error = dpm_run_callback(callback, dev, state, info);
 805         dev->power.is_suspended = false;
 806
 807  Unlock:
 808         device_unlock(dev);
 809         dpm_watchdog_clear(&wd);
 810
 811  Complete:
 812         complete_all(&dev->power.completion);
 813
 814         TRACE_RESUME(error);
 815
 816         return error;
 817 }
 818
 819 static void async_resume(void *data, async_cookie_t cookie)
 820 {
 821         struct device *dev = (struct device *)data;
 822         int error;
 823
 824         error = device_resume(dev, pm_transition, true);
 825         if (error)
 826                 pm_dev_err(dev, pm_transition, " async", error);
 827         put_device(dev);
 828 }
 829
 830 /**
 831  * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 832  * @state: PM transition of the system being carried out.
 833  *
 834  * Execute the appropriate "resume" callback for all devices whose status
 835  * indicates that they are suspended.
 836  */
 837 void dpm_resume(pm_message_t state)
 838 {
 839         struct device *dev;
 840         ktime_t starttime = ktime_get();
 841
 842         trace_suspend_resume(TPS("dpm_resume"), state.event, true);
 843         might_sleep();
 844
 845         mutex_lock(&dpm_list_mtx);
 846         pm_transition = state;
 847         async_error = 0;
 848
 849         list_for_each_entry(dev, &dpm_suspended_list, power.entry) {
 850                 reinit_completion(&dev->power.completion);
 851                 if (is_async(dev)) {
 852                         get_device(dev);
 853                         async_schedule(async_resume, dev);
 854                 }
 855         }
 856
 857         while (!list_empty(&dpm_suspended_list)) {
 858                 dev = to_device(dpm_suspended_list.next);
 859                 get_device(dev);
 860                 if (!is_async(dev)) {
 861                         int error;
 862
 863                         mutex_unlock(&dpm_list_mtx);
 864
 865                         error = device_resume(dev, state, false);
 866                         if (error) {
 867                                 suspend_stats.failed_resume++;
 868                                 dpm_save_failed_step(SUSPEND_RESUME);
 869                                 dpm_save_failed_dev(dev_name(dev));
 870                                 pm_dev_err(dev, state, "", error);
 871                         }
 872
 873                         mutex_lock(&dpm_list_mtx);
 874                 }
 875                 if (!list_empty(&dev->power.entry))
 876                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
 877                 put_device(dev);
 878         }
 879         mutex_unlock(&dpm_list_mtx);
 880         async_synchronize_full();
 881         dpm_show_time(starttime, state, NULL);
 882
 883         cpufreq_resume();
 884         trace_suspend_resume(TPS("dpm_resume"), state.event, false);
 885 }
 886
 887 /**
 888  * device_complete - Complete a PM transition for given device.
 889  * @dev: Device to handle.
 890  * @state: PM transition of the system being carried out.
 891  */
 892 static void device_complete(struct device *dev, pm_message_t state)
 893 {
 894         void (*callback)(struct device *) = NULL;
 895         char *info = NULL;
 896
 897         if (dev->power.syscore)
 898                 return;
 899
 900         device_lock(dev);
 901
 902         if (dev->pm_domain) {
 903                 info = "completing power domain ";
 904                 callback = dev->pm_domain->ops.complete;
 905         } else if (dev->type && dev->type->pm) {
 906                 info = "completing type ";
 907                 callback = dev->type->pm->complete;
 908         } else if (dev->class && dev->class->pm) {
 909                 info = "completing class ";
 910                 callback = dev->class->pm->complete;
 911         } else if (dev->bus && dev->bus->pm) {
 912                 info = "completing bus ";
 913                 callback = dev->bus->pm->complete;
 914         }
 915
 916         if (!callback && dev->driver && dev->driver->pm) {
 917                 info = "completing driver ";
 918                 callback = dev->driver->pm->complete;
 919         }
 920
 921         if (callback) {
 922                 pm_dev_dbg(dev, state, info);
 923                 trace_device_pm_callback_start(dev, info, state.event);
 924                 callback(dev);
 925                 trace_device_pm_callback_end(dev, 0);
 926         }
 927
 928         device_unlock(dev);
 929
 930         pm_runtime_put(dev);
 931 }
 932
 933 /**
 934  * dpm_complete - Complete a PM transition for all non-sysdev devices.
 935  * @state: PM transition of the system being carried out.
 936  *
 937  * Execute the ->complete() callbacks for all devices whose PM status is not
 938  * DPM_ON (this allows new devices to be registered).
 939  */
 940 void dpm_complete(pm_message_t state)
 941 {
 942         struct list_head list;
 943
 944         trace_suspend_resume(TPS("dpm_complete"), state.event, true);
 945         might_sleep();
 946
 947         INIT_LIST_HEAD(&list);
 948         mutex_lock(&dpm_list_mtx);
 949         while (!list_empty(&dpm_prepared_list)) {
 950                 struct device *dev = to_device(dpm_prepared_list.prev);
 951
 952                 get_device(dev);
 953                 dev->power.is_prepared = false;
 954                 list_move(&dev->power.entry, &list);
 955                 mutex_unlock(&dpm_list_mtx);
 956
 957                 device_complete(dev, state);
 958
 959                 mutex_lock(&dpm_list_mtx);
 960                 put_device(dev);
 961         }
 962         list_splice(&list, &dpm_list);
 963         mutex_unlock(&dpm_list_mtx);
 964         trace_suspend_resume(TPS("dpm_complete"), state.event, false);
 965 }
 966
 967 /**
 968  * dpm_resume_end - Execute "resume" callbacks and complete system transition.
 969  * @state: PM transition of the system being carried out.
 970  *
 971  * Execute "resume" callbacks for all devices and complete the PM transition of
 972  * the system.
 973  */
 974 void dpm_resume_end(pm_message_t state)
 975 {
 976         dpm_resume(state);
 977         dpm_complete(state);
 978 }
 979 EXPORT_SYMBOL_GPL(dpm_resume_end);
 980
 981
 982 /*------------------------- Suspend routines -------------------------*/
 983
 984 /**
 985  * resume_event - Return a "resume" message for given "suspend" sleep state.
 986  * @sleep_state: PM message representing a sleep state.
 987  *
 988  * Return a PM message representing the resume event corresponding to given
 989  * sleep state.
 990  */
 991 static pm_message_t resume_event(pm_message_t sleep_state)
 992 {
 993         switch (sleep_state.event) {
 994         case PM_EVENT_SUSPEND:
 995                 return PMSG_RESUME;
 996         case PM_EVENT_FREEZE:
 997         case PM_EVENT_QUIESCE:
 998                 return PMSG_RECOVER;
 999         case PM_EVENT_HIBERNATE:
1000                 return PMSG_RESTORE;
1001         }
1002         return PMSG_ON;
1003 }
1004
1005 /**
1006  * device_suspend_noirq - Execute a "late suspend" callback for given device.
1007  * @dev: Device to handle.
1008  * @state: PM transition of the system being carried out.
1009  * @async: If true, the device is being suspended asynchronously.
1010  *
1011  * The driver of @dev will not receive interrupts while this function is being
1012  * executed.
1013  */
1014 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1015 {
1016         pm_callback_t callback = NULL;
1017         char *info = NULL;
1018         int error = 0;
1019
1020         if (async_error)
1021                 goto Complete;
1022
1023         if (pm_wakeup_pending()) {
1024                 async_error = -EBUSY;
1025                 goto Complete;
1026         }
1027
1028         if (dev->power.syscore || dev->power.direct_complete)
1029                 goto Complete;
1030
1031         dpm_wait_for_children(dev, async);
1032
1033         if (dev->pm_domain) {
1034                 info = "noirq power domain ";
1035                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1036         } else if (dev->type && dev->type->pm) {
1037                 info = "noirq type ";
1038                 callback = pm_noirq_op(dev->type->pm, state);
1039         } else if (dev->class && dev->class->pm) {
1040                 info = "noirq class ";
1041                 callback = pm_noirq_op(dev->class->pm, state);
1042         } else if (dev->bus && dev->bus->pm) {
1043                 info = "noirq bus ";
1044                 callback = pm_noirq_op(dev->bus->pm, state);
1045         }
1046
1047         if (!callback && dev->driver && dev->driver->pm) {
1048                 info = "noirq driver ";
1049                 callback = pm_noirq_op(dev->driver->pm, state);
1050         }
1051
1052         error = dpm_run_callback(callback, dev, state, info);
1053         if (!error)
1054                 dev->power.is_noirq_suspended = true;
1055         else
1056                 async_error = error;
1057
1058 Complete:
1059         complete_all(&dev->power.completion);
1060         return error;
1061 }
1062
1063 static void async_suspend_noirq(void *data, async_cookie_t cookie)
1064 {
1065         struct device *dev = (struct device *)data;
1066         int error;
1067
1068         error = __device_suspend_noirq(dev, pm_transition, true);
1069         if (error) {
1070                 dpm_save_failed_dev(dev_name(dev));
1071                 pm_dev_err(dev, pm_transition, " async", error);
1072         }
1073
1074         put_device(dev);
1075 }
1076
1077 static int device_suspend_noirq(struct device *dev)
1078 {
1079         reinit_completion(&dev->power.completion);
1080
1081         if (pm_async_enabled && dev->power.async_suspend) {
1082                 get_device(dev);
1083                 async_schedule(async_suspend_noirq, dev);
1084                 return 0;
1085         }
1086         return __device_suspend_noirq(dev, pm_transition, false);
1087 }
1088
1089 /**
1090  * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1091  * @state: PM transition of the system being carried out.
1092  *
1093  * Prevent device drivers from receiving interrupts and call the "noirq" suspend
1094  * handlers for all non-sysdev devices.
1095  */
1096 static int dpm_suspend_noirq(pm_message_t state)
1097 {
1098         ktime_t starttime = ktime_get();
1099         int error = 0;
1100
1101         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1102         cpuidle_pause();
1103         suspend_device_irqs();
1104         mutex_lock(&dpm_list_mtx);
1105         pm_transition = state;
1106         async_error = 0;
1107
1108         while (!list_empty(&dpm_late_early_list)) {
1109                 struct device *dev = to_device(dpm_late_early_list.prev);
1110
1111                 get_device(dev);
1112                 mutex_unlock(&dpm_list_mtx);
1113
1114                 error = device_suspend_noirq(dev);
1115
1116                 mutex_lock(&dpm_list_mtx);
1117                 if (error) {
1118                         pm_dev_err(dev, state, " noirq", error);
1119                         dpm_save_failed_dev(dev_name(dev));
1120                         put_device(dev);
1121                         break;
1122                 }
1123                 if (!list_empty(&dev->power.entry))
1124                         list_move(&dev->power.entry, &dpm_noirq_list);
1125                 put_device(dev);
1126
1127                 if (async_error)
1128                         break;
1129         }
1130         mutex_unlock(&dpm_list_mtx);
1131         async_synchronize_full();
1132         if (!error)
1133                 error = async_error;
1134
1135         if (error) {
1136                 suspend_stats.failed_suspend_noirq++;
1137                 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1138                 dpm_resume_noirq(resume_event(state));
1139         } else {
1140                 dpm_show_time(starttime, state, "noirq");
1141         }
1142         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1143         return error;
1144 }
1145
1146 /**
1147  * device_suspend_late - Execute a "late suspend" callback for given device.
1148  * @dev: Device to handle.
1149  * @state: PM transition of the system being carried out.
1150  * @async: If true, the device is being suspended asynchronously.
1151  *
1152  * Runtime PM is disabled for @dev while this function is being executed.
1153  */
1154 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1155 {
1156         pm_callback_t callback = NULL;
1157         char *info = NULL;
1158         int error = 0;
1159
1160         __pm_runtime_disable(dev, false);
1161
1162         if (async_error)
1163                 goto Complete;
1164
1165         if (pm_wakeup_pending()) {
1166                 async_error = -EBUSY;
1167                 goto Complete;
1168         }
1169
1170         if (dev->power.syscore || dev->power.direct_complete)
1171                 goto Complete;
1172
1173         dpm_wait_for_children(dev, async);
1174
1175         if (dev->pm_domain) {
1176                 info = "late power domain ";
1177                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1178         } else if (dev->type && dev->type->pm) {
1179                 info = "late type ";
1180                 callback = pm_late_early_op(dev->type->pm, state);
1181         } else if (dev->class && dev->class->pm) {
1182                 info = "late class ";
1183                 callback = pm_late_early_op(dev->class->pm, state);
1184         } else if (dev->bus && dev->bus->pm) {
1185                 info = "late bus ";
1186                 callback = pm_late_early_op(dev->bus->pm, state);
1187         }
1188
1189         if (!callback && dev->driver && dev->driver->pm) {
1190                 info = "late driver ";
1191                 callback = pm_late_early_op(dev->driver->pm, state);
1192         }
1193
1194         error = dpm_run_callback(callback, dev, state, info);
1195         if (!error)
1196                 dev->power.is_late_suspended = true;
1197         else
1198                 async_error = error;
1199
1200 Complete:
1201         complete_all(&dev->power.completion);
1202         return error;
1203 }
1204
1205 static void async_suspend_late(void *data, async_cookie_t cookie)
1206 {
1207         struct device *dev = (struct device *)data;
1208         int error;
1209
1210         error = __device_suspend_late(dev, pm_transition, true);
1211         if (error) {
1212                 dpm_save_failed_dev(dev_name(dev));
1213                 pm_dev_err(dev, pm_transition, " async", error);
1214         }
1215         put_device(dev);
1216 }
1217
1218 static int device_suspend_late(struct device *dev)
1219 {
1220         reinit_completion(&dev->power.completion);
1221
1222         if (pm_async_enabled && dev->power.async_suspend) {
1223                 get_device(dev);
1224                 async_schedule(async_suspend_late, dev);
1225                 return 0;
1226         }
1227
1228         return __device_suspend_late(dev, pm_transition, false);
1229 }
1230
1231 /**
1232  * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1233  * @state: PM transition of the system being carried out.
1234  */
1235 static int dpm_suspend_late(pm_message_t state)
1236 {
1237         ktime_t starttime = ktime_get();
1238         int error = 0;
1239
1240         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1241         mutex_lock(&dpm_list_mtx);
1242         pm_transition = state;
1243         async_error = 0;
1244
1245         while (!list_empty(&dpm_suspended_list)) {
1246                 struct device *dev = to_device(dpm_suspended_list.prev);
1247
1248                 get_device(dev);
1249                 mutex_unlock(&dpm_list_mtx);
1250
1251                 error = device_suspend_late(dev);
1252
1253                 mutex_lock(&dpm_list_mtx);
1254                 if (error) {
1255                         pm_dev_err(dev, state, " late", error);
1256                         dpm_save_failed_dev(dev_name(dev));
1257                         put_device(dev);
1258                         break;
1259                 }
1260                 if (!list_empty(&dev->power.entry))
1261                         list_move(&dev->power.entry, &dpm_late_early_list);
1262                 put_device(dev);
1263
1264                 if (async_error)
1265                         break;
1266         }
1267         mutex_unlock(&dpm_list_mtx);
1268         async_synchronize_full();
1269         if (error) {
1270                 suspend_stats.failed_suspend_late++;
1271                 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1272                 dpm_resume_early(resume_event(state));
1273         } else {
1274                 dpm_show_time(starttime, state, "late");
1275         }
1276         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1277         return error;
1278 }
1279
1280 /**
1281  * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1282  * @state: PM transition of the system being carried out.
1283  */
1284 int dpm_suspend_end(pm_message_t state)
1285 {
1286         int error = dpm_suspend_late(state);
1287         if (error)
1288                 return error;
1289
1290         error = dpm_suspend_noirq(state);
1291         if (error) {
1292                 dpm_resume_early(resume_event(state));
1293                 return error;
1294         }
1295
1296         return 0;
1297 }
1298 EXPORT_SYMBOL_GPL(dpm_suspend_end);
1299
1300 /**
1301  * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1302  * @dev: Device to suspend.
1303  * @state: PM transition of the system being carried out.
1304  * @cb: Suspend callback to execute.
1305  * @info: string description of caller.
1306  */
1307 static int legacy_suspend(struct device *dev, pm_message_t state,
1308                           int (*cb)(struct device *dev, pm_message_t state),
1309                           char *info)
1310 {
1311         int error;
1312         ktime_t calltime;
1313
1314         calltime = initcall_debug_start(dev);
1315
1316         trace_device_pm_callback_start(dev, info, state.event);
1317         error = cb(dev, state);
1318         trace_device_pm_callback_end(dev, error);
1319         suspend_report_result(cb, error);
1320
1321         initcall_debug_report(dev, calltime, error, state, info);
1322
1323         return error;
1324 }
1325
1326 /**
1327  * device_suspend - Execute "suspend" callbacks for given device.
1328  * @dev: Device to handle.
1329  * @state: PM transition of the system being carried out.
1330  * @async: If true, the device is being suspended asynchronously.
1331  */
1332 static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1333 {
1334         pm_callback_t callback = NULL;
1335         char *info = NULL;
1336         int error = 0;
1337         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1338
1339         dpm_wait_for_children(dev, async);
1340
1341         if (async_error)
1342                 goto Complete;
1343
1344         /*
1345          * If a device configured to wake up the system from sleep states
1346          * has been suspended at run time and there's a resume request pending
1347          * for it, this is equivalent to the device signaling wakeup, so the
1348          * system suspend operation should be aborted.
1349          */
1350         if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
1351                 pm_wakeup_event(dev, 0);
1352
1353         if (pm_wakeup_pending()) {
1354                 async_error = -EBUSY;
1355                 goto Complete;
1356         }
1357
1358         if (dev->power.syscore)
1359                 goto Complete;
1360
1361         if (dev->power.direct_complete) {
1362                 if (pm_runtime_status_suspended(dev)) {
1363                         pm_runtime_disable(dev);
1364                         if (pm_runtime_suspended_if_enabled(dev))
1365                                 goto Complete;
1366
1367                         pm_runtime_enable(dev);
1368                 }
1369                 dev->power.direct_complete = false;
1370         }
1371
1372         dpm_watchdog_set(&wd, dev);
1373         device_lock(dev);
1374
1375         if (dev->pm_domain) {
1376                 info = "power domain ";
1377                 callback = pm_op(&dev->pm_domain->ops, state);
1378                 goto Run;
1379         }
1380
1381         if (dev->type && dev->type->pm) {
1382                 info = "type ";
1383                 callback = pm_op(dev->type->pm, state);
1384                 goto Run;
1385         }
1386
1387         if (dev->class) {
1388                 if (dev->class->pm) {
1389                         info = "class ";
1390                         callback = pm_op(dev->class->pm, state);
1391                         goto Run;
1392                 } else if (dev->class->suspend) {
1393                         pm_dev_dbg(dev, state, "legacy class ");
1394                         error = legacy_suspend(dev, state, dev->class->suspend,
1395                                                 "legacy class ");
1396                         goto End;
1397                 }
1398         }
1399
1400         if (dev->bus) {
1401                 if (dev->bus->pm) {
1402                         info = "bus ";
1403                         callback = pm_op(dev->bus->pm, state);
1404                 } else if (dev->bus->suspend) {
1405                         pm_dev_dbg(dev, state, "legacy bus ");
1406                         error = legacy_suspend(dev, state, dev->bus->suspend,
1407                                                 "legacy bus ");
1408                         goto End;
1409                 }
1410         }
1411
1412  Run:
1413         if (!callback && dev->driver && dev->driver->pm) {
1414                 info = "driver ";
1415                 callback = pm_op(dev->driver->pm, state);
1416         }
1417
1418         error = dpm_run_callback(callback, dev, state, info);
1419
1420  End:
1421         if (!error) {
1422                 struct device *parent = dev->parent;
1423
1424                 dev->power.is_suspended = true;
1425                 if (parent) {
1426                         spin_lock_irq(&parent->power.lock);
1427
1428                         dev->parent->power.direct_complete = false;
1429                         if (dev->power.wakeup_path
1430                             && !dev->parent->power.ignore_children)
1431                                 dev->parent->power.wakeup_path = true;
1432
1433                         spin_unlock_irq(&parent->power.lock);
1434                 }
1435         }
1436
1437         device_unlock(dev);
1438         dpm_watchdog_clear(&wd);
1439
1440  Complete:
1441         complete_all(&dev->power.completion);
1442         if (error)
1443                 async_error = error;
1444
1445         return error;
1446 }
1447
1448 static void async_suspend(void *data, async_cookie_t cookie)
1449 {
1450         struct device *dev = (struct device *)data;
1451         int error;
1452
1453         error = __device_suspend(dev, pm_transition, true);
1454         if (error) {
1455                 dpm_save_failed_dev(dev_name(dev));
1456                 pm_dev_err(dev, pm_transition, " async", error);
1457         }
1458
1459         put_device(dev);
1460 }
1461
1462 static int device_suspend(struct device *dev)
1463 {
1464         reinit_completion(&dev->power.completion);
1465
1466         if (pm_async_enabled && dev->power.async_suspend) {
1467                 get_device(dev);
1468                 async_schedule(async_suspend, dev);
1469                 return 0;
1470         }
1471
1472         return __device_suspend(dev, pm_transition, false);
1473 }
1474
1475 /**
1476  * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1477  * @state: PM transition of the system being carried out.
1478  */
1479 int dpm_suspend(pm_message_t state)
1480 {
1481         ktime_t starttime = ktime_get();
1482         int error = 0;
1483
1484         trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1485         might_sleep();
1486
1487         cpufreq_suspend();
1488
1489         mutex_lock(&dpm_list_mtx);
1490         pm_transition = state;
1491         async_error = 0;
1492         while (!list_empty(&dpm_prepared_list)) {
1493                 struct device *dev = to_device(dpm_prepared_list.prev);
1494
1495                 get_device(dev);
1496                 mutex_unlock(&dpm_list_mtx);
1497
1498                 error = device_suspend(dev);
1499
1500                 mutex_lock(&dpm_list_mtx);
1501                 if (error) {
1502                         pm_dev_err(dev, state, "", error);
1503                         dpm_save_failed_dev(dev_name(dev));
1504                         put_device(dev);
1505                         break;
1506                 }
1507                 if (!list_empty(&dev->power.entry))
1508                         list_move(&dev->power.entry, &dpm_suspended_list);
1509                 put_device(dev);
1510                 if (async_error)
1511                         break;
1512         }
1513         mutex_unlock(&dpm_list_mtx);
1514         async_synchronize_full();
1515         if (!error)
1516                 error = async_error;
1517         if (error) {
1518                 suspend_stats.failed_suspend++;
1519                 dpm_save_failed_step(SUSPEND_SUSPEND);
1520         } else
1521                 dpm_show_time(starttime, state, NULL);
1522         trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1523         return error;
1524 }
1525
1526 /**
1527  * device_prepare - Prepare a device for system power transition.
1528  * @dev: Device to handle.
1529  * @state: PM transition of the system being carried out.
1530  *
1531  * Execute the ->prepare() callback(s) for given device.  No new children of the
1532  * device may be registered after this function has returned.
1533  */
1534 static int device_prepare(struct device *dev, pm_message_t state)
1535 {
1536         int (*callback)(struct device *) = NULL;
1537         char *info = NULL;
1538         int ret = 0;
1539
1540         if (dev->power.syscore)
1541                 return 0;
1542
1543         /*
1544          * If a device's parent goes into runtime suspend at the wrong time,
1545          * it won't be possible to resume the device.  To prevent this we
1546          * block runtime suspend here, during the prepare phase, and allow
1547          * it again during the complete phase.
1548          */
1549         pm_runtime_get_noresume(dev);
1550
1551         device_lock(dev);
1552
1553         dev->power.wakeup_path = device_may_wakeup(dev);
1554
1555         if (dev->pm_domain) {
1556                 info = "preparing power domain ";
1557                 callback = dev->pm_domain->ops.prepare;
1558         } else if (dev->type && dev->type->pm) {
1559                 info = "preparing type ";
1560                 callback = dev->type->pm->prepare;
1561         } else if (dev->class && dev->class->pm) {
1562                 info = "preparing class ";
1563                 callback = dev->class->pm->prepare;
1564         } else if (dev->bus && dev->bus->pm) {
1565                 info = "preparing bus ";
1566                 callback = dev->bus->pm->prepare;
1567         }
1568
1569         if (!callback && dev->driver && dev->driver->pm) {
1570                 info = "preparing driver ";
1571                 callback = dev->driver->pm->prepare;
1572         }
1573
1574         if (callback) {
1575                 trace_device_pm_callback_start(dev, info, state.event);
1576                 ret = callback(dev);
1577                 trace_device_pm_callback_end(dev, ret);
1578         }
1579
1580         device_unlock(dev);
1581
1582         if (ret < 0) {
1583                 suspend_report_result(callback, ret);
1584                 pm_runtime_put(dev);
1585                 return ret;
1586         }
1587         /*
1588          * A positive return value from ->prepare() means "this device appears
1589          * to be runtime-suspended and its state is fine, so if it really is
1590          * runtime-suspended, you can leave it in that state provided that you
1591          * will do the same thing with all of its descendants".  This only
1592          * applies to suspend transitions, however.
1593          */
1594         spin_lock_irq(&dev->power.lock);
1595         dev->power.direct_complete = ret > 0 && state.event == PM_EVENT_SUSPEND;
1596         spin_unlock_irq(&dev->power.lock);
1597         return 0;
1598 }
1599
1600 /**
1601  * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1602  * @state: PM transition of the system being carried out.
1603  *
1604  * Execute the ->prepare() callback(s) for all devices.
1605  */
1606 int dpm_prepare(pm_message_t state)
1607 {
1608         int error = 0;
1609
1610         trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1611         might_sleep();
1612
1613         mutex_lock(&dpm_list_mtx);
1614         while (!list_empty(&dpm_list)) {
1615                 struct device *dev = to_device(dpm_list.next);
1616
1617                 get_device(dev);
1618                 mutex_unlock(&dpm_list_mtx);
1619
1620                 error = device_prepare(dev, state);
1621
1622                 mutex_lock(&dpm_list_mtx);
1623                 if (error) {
1624                         if (error == -EAGAIN) {
1625                                 put_device(dev);
1626                                 error = 0;
1627                                 continue;
1628                         }
1629                         printk(KERN_INFO "PM: Device %s not prepared "
1630                                 "for power transition: code %d\n",
1631                                 dev_name(dev), error);
1632                         put_device(dev);
1633                         break;
1634                 }
1635                 dev->power.is_prepared = true;
1636                 if (!list_empty(&dev->power.entry))
1637                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1638                 put_device(dev);
1639         }
1640         mutex_unlock(&dpm_list_mtx);
1641         trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
1642         return error;
1643 }
1644
1645 /**
1646  * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1647  * @state: PM transition of the system being carried out.
1648  *
1649  * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1650  * callbacks for them.
1651  */
1652 int dpm_suspend_start(pm_message_t state)
1653 {
1654         int error;
1655
1656         error = dpm_prepare(state);
1657         if (error) {
1658                 suspend_stats.failed_prepare++;
1659                 dpm_save_failed_step(SUSPEND_PREPARE);
1660         } else
1661                 error = dpm_suspend(state);
1662         return error;
1663 }
1664 EXPORT_SYMBOL_GPL(dpm_suspend_start);
1665
1666 void __suspend_report_result(const char *function, void *fn, int ret)
1667 {
1668         if (ret)
1669                 printk(KERN_ERR "%s(): %pF returns %d\n", function, fn, ret);
1670 }
1671 EXPORT_SYMBOL_GPL(__suspend_report_result);
1672
1673 /**
1674  * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1675  * @dev: Device to wait for.
1676  * @subordinate: Device that needs to wait for @dev.
1677  */
1678 int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1679 {
1680         dpm_wait(dev, subordinate->power.async_suspend);
1681         return async_error;
1682 }
1683 EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1684
1685 /**
1686  * dpm_for_each_dev - device iterator.
1687  * @data: data for the callback.
1688  * @fn: function to be called for each device.
1689  *
1690  * Iterate over devices in dpm_list, and call @fn for each device,
1691  * passing it @data.
1692  */
1693 void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1694 {
1695         struct device *dev;
1696
1697         if (!fn)
1698                 return;
1699
1700         device_pm_lock();
1701         list_for_each_entry(dev, &dpm_list, power.entry)
1702                 fn(dev, data);
1703         device_pm_unlock();
1704 }
1705 EXPORT_SYMBOL_GPL(dpm_for_each_dev);