kernel/irq/manage.c

   1 /*
   2  * linux/kernel/irq/manage.c
   3  *
   4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
   5  * Copyright (C) 2005-2006 Thomas Gleixner
   6  *
   7  * This file contains driver APIs to the irq subsystem.
   8  */
   9
  10 #define pr_fmt(fmt) "genirq: " fmt
  11
  12 #include <linux/irq.h>
  13 #include <linux/kthread.h>
  14 #include <linux/module.h>
  15 #include <linux/random.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/slab.h>
  18 #include <linux/sched.h>
  19 #include <linux/sched/rt.h>
  20 #include <linux/task_work.h>
  21
  22 #include "internals.h"
  23
  24 #ifdef CONFIG_IRQ_FORCED_THREADING
  25 __read_mostly bool force_irqthreads;
  26
  27 static int __init setup_forced_irqthreads(char *arg)
  28 {
  29         force_irqthreads = true;
  30         return 0;
  31 }
  32 early_param("threadirqs", setup_forced_irqthreads);
  33 #endif
  34
  35 static void __synchronize_hardirq(struct irq_desc *desc)
  36 {
  37         bool inprogress;
  38
  39         do {
  40                 unsigned long flags;
  41
  42                 /*
  43                  * Wait until we're out of the critical section.  This might
  44                  * give the wrong answer due to the lack of memory barriers.
  45                  */
  46                 while (irqd_irq_inprogress(&desc->irq_data))
  47                         cpu_relax();
  48
  49                 /* Ok, that indicated we're done: double-check carefully. */
  50                 raw_spin_lock_irqsave(&desc->lock, flags);
  51                 inprogress = irqd_irq_inprogress(&desc->irq_data);
  52                 raw_spin_unlock_irqrestore(&desc->lock, flags);
  53
  54                 /* Oops, that failed? */
  55         } while (inprogress);
  56 }
  57
  58 /**
  59  *      synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
  60  *      @irq: interrupt number to wait for
  61  *
  62  *      This function waits for any pending hard IRQ handlers for this
  63  *      interrupt to complete before returning. If you use this
  64  *      function while holding a resource the IRQ handler may need you
  65  *      will deadlock. It does not take associated threaded handlers
  66  *      into account.
  67  *
  68  *      Do not use this for shutdown scenarios where you must be sure
  69  *      that all parts (hardirq and threaded handler) have completed.
  70  *
  71  *      This function may be called - with care - from IRQ context.
  72  */
  73 void synchronize_hardirq(unsigned int irq)
  74 {
  75         struct irq_desc *desc = irq_to_desc(irq);
  76
  77         if (desc)
  78                 __synchronize_hardirq(desc);
  79 }
  80 EXPORT_SYMBOL(synchronize_hardirq);
  81
  82 /**
  83  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  84  *      @irq: interrupt number to wait for
  85  *
  86  *      This function waits for any pending IRQ handlers for this interrupt
  87  *      to complete before returning. If you use this function while
  88  *      holding a resource the IRQ handler may need you will deadlock.
  89  *
  90  *      This function may be called - with care - from IRQ context.
  91  */
  92 void synchronize_irq(unsigned int irq)
  93 {
  94         struct irq_desc *desc = irq_to_desc(irq);
  95
  96         if (desc) {
  97                 __synchronize_hardirq(desc);
  98                 /*
  99                  * We made sure that no hardirq handler is
 100                  * running. Now verify that no threaded handlers are
 101                  * active.
 102                  */
 103                 wait_event(desc->wait_for_threads,
 104                            !atomic_read(&desc->threads_active));
 105         }
 106 }
 107 EXPORT_SYMBOL(synchronize_irq);
 108
 109 #ifdef CONFIG_SMP
 110 cpumask_var_t irq_default_affinity;
 111
 112 /**
 113  *      irq_can_set_affinity - Check if the affinity of a given irq can be set
 114  *      @irq:           Interrupt to check
 115  *
 116  */
 117 int irq_can_set_affinity(unsigned int irq)
 118 {
 119         struct irq_desc *desc = irq_to_desc(irq);
 120
 121         if (!desc || !irqd_can_balance(&desc->irq_data) ||
 122             !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
 123                 return 0;
 124
 125         return 1;
 126 }
 127
 128 /**
 129  *      irq_set_thread_affinity - Notify irq threads to adjust affinity
 130  *      @desc:          irq descriptor which has affitnity changed
 131  *
 132  *      We just set IRQTF_AFFINITY and delegate the affinity setting
 133  *      to the interrupt thread itself. We can not call
 134  *      set_cpus_allowed_ptr() here as we hold desc->lock and this
 135  *      code can be called from hard interrupt context.
 136  */
 137 void irq_set_thread_affinity(struct irq_desc *desc)
 138 {
 139         struct irqaction *action = desc->action;
 140
 141         while (action) {
 142                 if (action->thread)
 143                         set_bit(IRQTF_AFFINITY, &action->thread_flags);
 144                 action = action->next;
 145         }
 146 }
 147
 148 #ifdef CONFIG_GENERIC_PENDING_IRQ
 149 static inline bool irq_can_move_pcntxt(struct irq_data *data)
 150 {
 151         return irqd_can_move_in_process_context(data);
 152 }
 153 static inline bool irq_move_pending(struct irq_data *data)
 154 {
 155         return irqd_is_setaffinity_pending(data);
 156 }
 157 static inline void
 158 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
 159 {
 160         cpumask_copy(desc->pending_mask, mask);
 161 }
 162 static inline void
 163 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
 164 {
 165         cpumask_copy(mask, desc->pending_mask);
 166 }
 167 #else
 168 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
 169 static inline bool irq_move_pending(struct irq_data *data) { return false; }
 170 static inline void
 171 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
 172 static inline void
 173 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
 174 #endif
 175
 176 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 177                         bool force)
 178 {
 179         struct irq_desc *desc = irq_data_to_desc(data);
 180         struct irq_chip *chip = irq_data_get_irq_chip(data);
 181         int ret;
 182
 183         ret = chip->irq_set_affinity(data, mask, force);
 184         switch (ret) {
 185         case IRQ_SET_MASK_OK:
 186         case IRQ_SET_MASK_OK_DONE:
 187                 cpumask_copy(data->affinity, mask);
 188         case IRQ_SET_MASK_OK_NOCOPY:
 189                 irq_set_thread_affinity(desc);
 190                 ret = 0;
 191         }
 192
 193         return ret;
 194 }
 195
 196 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
 197                             bool force)
 198 {
 199         struct irq_chip *chip = irq_data_get_irq_chip(data);
 200         struct irq_desc *desc = irq_data_to_desc(data);
 201         int ret = 0;
 202
 203         if (!chip || !chip->irq_set_affinity)
 204                 return -EINVAL;
 205
 206         if (irq_can_move_pcntxt(data)) {
 207                 ret = irq_do_set_affinity(data, mask, force);
 208         } else {
 209                 irqd_set_move_pending(data);
 210                 irq_copy_pending(desc, mask);
 211         }
 212
 213         if (desc->affinity_notify) {
 214                 kref_get(&desc->affinity_notify->kref);
 215                 schedule_work(&desc->affinity_notify->work);
 216         }
 217         irqd_set(data, IRQD_AFFINITY_SET);
 218
 219         return ret;
 220 }
 221
 222 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
 223 {
 224         struct irq_desc *desc = irq_to_desc(irq);
 225         unsigned long flags;
 226         int ret;
 227
 228         if (!desc)
 229                 return -EINVAL;
 230
 231         raw_spin_lock_irqsave(&desc->lock, flags);
 232         ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
 233         raw_spin_unlock_irqrestore(&desc->lock, flags);
 234         return ret;
 235 }
 236
 237 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
 238 {
 239         unsigned long flags;
 240         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 241
 242         if (!desc)
 243                 return -EINVAL;
 244         desc->affinity_hint = m;
 245         irq_put_desc_unlock(desc, flags);
 246         return 0;
 247 }
 248 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
 249
 250 static void irq_affinity_notify(struct work_struct *work)
 251 {
 252         struct irq_affinity_notify *notify =
 253                 container_of(work, struct irq_affinity_notify, work);
 254         struct irq_desc *desc = irq_to_desc(notify->irq);
 255         cpumask_var_t cpumask;
 256         unsigned long flags;
 257
 258         if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
 259                 goto out;
 260
 261         raw_spin_lock_irqsave(&desc->lock, flags);
 262         if (irq_move_pending(&desc->irq_data))
 263                 irq_get_pending(cpumask, desc);
 264         else
 265                 cpumask_copy(cpumask, desc->irq_data.affinity);
 266         raw_spin_unlock_irqrestore(&desc->lock, flags);
 267
 268         notify->notify(notify, cpumask);
 269
 270         free_cpumask_var(cpumask);
 271 out:
 272         kref_put(&notify->kref, notify->release);
 273 }
 274
 275 /**
 276  *      irq_set_affinity_notifier - control notification of IRQ affinity changes
 277  *      @irq:           Interrupt for which to enable/disable notification
 278  *      @notify:        Context for notification, or %NULL to disable
 279  *                      notification.  Function pointers must be initialised;
 280  *                      the other fields will be initialised by this function.
 281  *
 282  *      Must be called in process context.  Notification may only be enabled
 283  *      after the IRQ is allocated and must be disabled before the IRQ is
 284  *      freed using free_irq().
 285  */
 286 int
 287 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 288 {
 289         struct irq_desc *desc = irq_to_desc(irq);
 290         struct irq_affinity_notify *old_notify;
 291         unsigned long flags;
 292
 293         /* The release function is promised process context */
 294         might_sleep();
 295
 296         if (!desc)
 297                 return -EINVAL;
 298
 299         /* Complete initialisation of *notify */
 300         if (notify) {
 301                 notify->irq = irq;
 302                 kref_init(&notify->kref);
 303                 INIT_WORK(&notify->work, irq_affinity_notify);
 304         }
 305
 306         raw_spin_lock_irqsave(&desc->lock, flags);
 307         old_notify = desc->affinity_notify;
 308         desc->affinity_notify = notify;
 309         raw_spin_unlock_irqrestore(&desc->lock, flags);
 310
 311         if (old_notify)
 312                 kref_put(&old_notify->kref, old_notify->release);
 313
 314         return 0;
 315 }
 316 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
 317
 318 #ifndef CONFIG_AUTO_IRQ_AFFINITY
 319 /*
 320  * Generic version of the affinity autoselector.
 321  */
 322 static int
 323 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 324 {
 325         struct cpumask *set = irq_default_affinity;
 326         int node = desc->irq_data.node;
 327
 328         /* Excludes PER_CPU and NO_BALANCE interrupts */
 329         if (!irq_can_set_affinity(irq))
 330                 return 0;
 331
 332         /*
 333          * Preserve an userspace affinity setup, but make sure that
 334          * one of the targets is online.
 335          */
 336         if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
 337                 if (cpumask_intersects(desc->irq_data.affinity,
 338                                        cpu_online_mask))
 339                         set = desc->irq_data.affinity;
 340                 else
 341                         irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
 342         }
 343
 344         cpumask_and(mask, cpu_online_mask, set);
 345         if (node != NUMA_NO_NODE) {
 346                 const struct cpumask *nodemask = cpumask_of_node(node);
 347
 348                 /* make sure at least one of the cpus in nodemask is online */
 349                 if (cpumask_intersects(mask, nodemask))
 350                         cpumask_and(mask, mask, nodemask);
 351         }
 352         irq_do_set_affinity(&desc->irq_data, mask, false);
 353         return 0;
 354 }
 355 #else
 356 static inline int
 357 setup_affinity(unsigned int irq, struct irq_desc *d, struct cpumask *mask)
 358 {
 359         return irq_select_affinity(irq);
 360 }
 361 #endif
 362
 363 /*
 364  * Called when affinity is set via /proc/irq
 365  */
 366 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
 367 {
 368         struct irq_desc *desc = irq_to_desc(irq);
 369         unsigned long flags;
 370         int ret;
 371
 372         raw_spin_lock_irqsave(&desc->lock, flags);
 373         ret = setup_affinity(irq, desc, mask);
 374         raw_spin_unlock_irqrestore(&desc->lock, flags);
 375         return ret;
 376 }
 377
 378 #else
 379 static inline int
 380 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 381 {
 382         return 0;
 383 }
 384 #endif
 385
 386 void __disable_irq(struct irq_desc *desc, unsigned int irq)
 387 {
 388         if (!desc->depth++)
 389                 irq_disable(desc);
 390 }
 391
 392 static int __disable_irq_nosync(unsigned int irq)
 393 {
 394         unsigned long flags;
 395         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 396
 397         if (!desc)
 398                 return -EINVAL;
 399         __disable_irq(desc, irq);
 400         irq_put_desc_busunlock(desc, flags);
 401         return 0;
 402 }
 403
 404 /**
 405  *      disable_irq_nosync - disable an irq without waiting
 406  *      @irq: Interrupt to disable
 407  *
 408  *      Disable the selected interrupt line.  Disables and Enables are
 409  *      nested.
 410  *      Unlike disable_irq(), this function does not ensure existing
 411  *      instances of the IRQ handler have completed before returning.
 412  *
 413  *      This function may be called from IRQ context.
 414  */
 415 void disable_irq_nosync(unsigned int irq)
 416 {
 417         __disable_irq_nosync(irq);
 418 }
 419 EXPORT_SYMBOL(disable_irq_nosync);
 420
 421 /**
 422  *      disable_irq - disable an irq and wait for completion
 423  *      @irq: Interrupt to disable
 424  *
 425  *      Disable the selected interrupt line.  Enables and Disables are
 426  *      nested.
 427  *      This function waits for any pending IRQ handlers for this interrupt
 428  *      to complete before returning. If you use this function while
 429  *      holding a resource the IRQ handler may need you will deadlock.
 430  *
 431  *      This function may be called - with care - from IRQ context.
 432  */
 433 void disable_irq(unsigned int irq)
 434 {
 435         if (!__disable_irq_nosync(irq))
 436                 synchronize_irq(irq);
 437 }
 438 EXPORT_SYMBOL(disable_irq);
 439
 440 void __enable_irq(struct irq_desc *desc, unsigned int irq)
 441 {
 442         switch (desc->depth) {
 443         case 0:
 444  err_out:
 445                 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
 446                 break;
 447         case 1: {
 448                 if (desc->istate & IRQS_SUSPENDED)
 449                         goto err_out;
 450                 /* Prevent probing on this irq: */
 451                 irq_settings_set_noprobe(desc);
 452                 irq_enable(desc);
 453                 check_irq_resend(desc, irq);
 454                 /* fall-through */
 455         }
 456         default:
 457                 desc->depth--;
 458         }
 459 }
 460
 461 /**
 462  *      enable_irq - enable handling of an irq
 463  *      @irq: Interrupt to enable
 464  *
 465  *      Undoes the effect of one call to disable_irq().  If this
 466  *      matches the last disable, processing of interrupts on this
 467  *      IRQ line is re-enabled.
 468  *
 469  *      This function may be called from IRQ context only when
 470  *      desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
 471  */
 472 void enable_irq(unsigned int irq)
 473 {
 474         unsigned long flags;
 475         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 476
 477         if (!desc)
 478                 return;
 479         if (WARN(!desc->irq_data.chip,
 480                  KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
 481                 goto out;
 482
 483         __enable_irq(desc, irq);
 484 out:
 485         irq_put_desc_busunlock(desc, flags);
 486 }
 487 EXPORT_SYMBOL(enable_irq);
 488
 489 static int set_irq_wake_real(unsigned int irq, unsigned int on)
 490 {
 491         struct irq_desc *desc = irq_to_desc(irq);
 492         int ret = -ENXIO;
 493
 494         if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
 495                 return 0;
 496
 497         if (desc->irq_data.chip->irq_set_wake)
 498                 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
 499
 500         return ret;
 501 }
 502
 503 /**
 504  *      irq_set_irq_wake - control irq power management wakeup
 505  *      @irq:   interrupt to control
 506  *      @on:    enable/disable power management wakeup
 507  *
 508  *      Enable/disable power management wakeup mode, which is
 509  *      disabled by default.  Enables and disables must match,
 510  *      just as they match for non-wakeup mode support.
 511  *
 512  *      Wakeup mode lets this IRQ wake the system from sleep
 513  *      states like "suspend to RAM".
 514  */
 515 int irq_set_irq_wake(unsigned int irq, unsigned int on)
 516 {
 517         unsigned long flags;
 518         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 519         int ret = 0;
 520
 521         if (!desc)
 522                 return -EINVAL;
 523
 524         /* wakeup-capable irqs can be shared between drivers that
 525          * don't need to have the same sleep mode behaviors.
 526          */
 527         if (on) {
 528                 if (desc->wake_depth++ == 0) {
 529                         ret = set_irq_wake_real(irq, on);
 530                         if (ret)
 531                                 desc->wake_depth = 0;
 532                         else
 533                                 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
 534                 }
 535         } else {
 536                 if (desc->wake_depth == 0) {
 537                         WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
 538                 } else if (--desc->wake_depth == 0) {
 539                         ret = set_irq_wake_real(irq, on);
 540                         if (ret)
 541                                 desc->wake_depth = 1;
 542                         else
 543                                 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
 544                 }
 545         }
 546         irq_put_desc_busunlock(desc, flags);
 547         return ret;
 548 }
 549 EXPORT_SYMBOL(irq_set_irq_wake);
 550
 551 /*
 552  * Internal function that tells the architecture code whether a
 553  * particular irq has been exclusively allocated or is available
 554  * for driver use.
 555  */
 556 int can_request_irq(unsigned int irq, unsigned long irqflags)
 557 {
 558         unsigned long flags;
 559         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 560         int canrequest = 0;
 561
 562         if (!desc)
 563                 return 0;
 564
 565         if (irq_settings_can_request(desc)) {
 566                 if (!desc->action ||
 567                     irqflags & desc->action->flags & IRQF_SHARED)
 568                         canrequest = 1;
 569         }
 570         irq_put_desc_unlock(desc, flags);
 571         return canrequest;
 572 }
 573
 574 int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
 575                       unsigned long flags)
 576 {
 577         struct irq_chip *chip = desc->irq_data.chip;
 578         int ret, unmask = 0;
 579
 580         if (!chip || !chip->irq_set_type) {
 581                 /*
 582                  * IRQF_TRIGGER_* but the PIC does not support multiple
 583                  * flow-types?
 584                  */
 585                 pr_debug("No set_type function for IRQ %d (%s)\n", irq,
 586                          chip ? (chip->name ? : "unknown") : "unknown");
 587                 return 0;
 588         }
 589
 590         flags &= IRQ_TYPE_SENSE_MASK;
 591
 592         if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
 593                 if (!irqd_irq_masked(&desc->irq_data))
 594                         mask_irq(desc);
 595                 if (!irqd_irq_disabled(&desc->irq_data))
 596                         unmask = 1;
 597         }
 598
 599         /* caller masked out all except trigger mode flags */
 600         ret = chip->irq_set_type(&desc->irq_data, flags);
 601
 602         switch (ret) {
 603         case IRQ_SET_MASK_OK:
 604         case IRQ_SET_MASK_OK_DONE:
 605                 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
 606                 irqd_set(&desc->irq_data, flags);
 607
 608         case IRQ_SET_MASK_OK_NOCOPY:
 609                 flags = irqd_get_trigger_type(&desc->irq_data);
 610                 irq_settings_set_trigger_mask(desc, flags);
 611                 irqd_clear(&desc->irq_data, IRQD_LEVEL);
 612                 irq_settings_clr_level(desc);
 613                 if (flags & IRQ_TYPE_LEVEL_MASK) {
 614                         irq_settings_set_level(desc);
 615                         irqd_set(&desc->irq_data, IRQD_LEVEL);
 616                 }
 617
 618                 ret = 0;
 619                 break;
 620         default:
 621                 pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
 622                        flags, irq, chip->irq_set_type);
 623         }
 624         if (unmask)
 625                 unmask_irq(desc);
 626         return ret;
 627 }
 628
 629 #ifdef CONFIG_HARDIRQS_SW_RESEND
 630 int irq_set_parent(int irq, int parent_irq)
 631 {
 632         unsigned long flags;
 633         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 634
 635         if (!desc)
 636                 return -EINVAL;
 637
 638         desc->parent_irq = parent_irq;
 639
 640         irq_put_desc_unlock(desc, flags);
 641         return 0;
 642 }
 643 #endif
 644
 645 /*
 646  * Default primary interrupt handler for threaded interrupts. Is
 647  * assigned as primary handler when request_threaded_irq is called
 648  * with handler == NULL. Useful for oneshot interrupts.
 649  */
 650 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
 651 {
 652         return IRQ_WAKE_THREAD;
 653 }
 654
 655 /*
 656  * Primary handler for nested threaded interrupts. Should never be
 657  * called.
 658  */
 659 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
 660 {
 661         WARN(1, "Primary handler called for nested irq %d\n", irq);
 662         return IRQ_NONE;
 663 }
 664
 665 static int irq_wait_for_interrupt(struct irqaction *action)
 666 {
 667         set_current_state(TASK_INTERRUPTIBLE);
 668
 669         while (!kthread_should_stop()) {
 670
 671                 if (test_and_clear_bit(IRQTF_RUNTHREAD,
 672                                        &action->thread_flags)) {
 673                         __set_current_state(TASK_RUNNING);
 674                         return 0;
 675                 }
 676                 schedule();
 677                 set_current_state(TASK_INTERRUPTIBLE);
 678         }
 679         __set_current_state(TASK_RUNNING);
 680         return -1;
 681 }
 682
 683 /*
 684  * Oneshot interrupts keep the irq line masked until the threaded
 685  * handler finished. unmask if the interrupt has not been disabled and
 686  * is marked MASKED.
 687  */
 688 static void irq_finalize_oneshot(struct irq_desc *desc,
 689                                  struct irqaction *action)
 690 {
 691         if (!(desc->istate & IRQS_ONESHOT))
 692                 return;
 693 again:
 694         chip_bus_lock(desc);
 695         raw_spin_lock_irq(&desc->lock);
 696
 697         /*
 698          * Implausible though it may be we need to protect us against
 699          * the following scenario:
 700          *
 701          * The thread is faster done than the hard interrupt handler
 702          * on the other CPU. If we unmask the irq line then the
 703          * interrupt can come in again and masks the line, leaves due
 704          * to IRQS_INPROGRESS and the irq line is masked forever.
 705          *
 706          * This also serializes the state of shared oneshot handlers
 707          * versus "desc->threads_onehsot |= action->thread_mask;" in
 708          * irq_wake_thread(). See the comment there which explains the
 709          * serialization.
 710          */
 711         if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
 712                 raw_spin_unlock_irq(&desc->lock);
 713                 chip_bus_sync_unlock(desc);
 714                 cpu_relax();
 715                 goto again;
 716         }
 717
 718         /*
 719          * Now check again, whether the thread should run. Otherwise
 720          * we would clear the threads_oneshot bit of this thread which
 721          * was just set.
 722          */
 723         if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 724                 goto out_unlock;
 725
 726         desc->threads_oneshot &= ~action->thread_mask;
 727
 728         if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
 729             irqd_irq_masked(&desc->irq_data))
 730                 unmask_threaded_irq(desc);
 731
 732 out_unlock:
 733         raw_spin_unlock_irq(&desc->lock);
 734         chip_bus_sync_unlock(desc);
 735 }
 736
 737 #ifdef CONFIG_SMP
 738 /*
 739  * Check whether we need to change the affinity of the interrupt thread.
 740  */
 741 static void
 742 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
 743 {
 744         cpumask_var_t mask;
 745         bool valid = true;
 746
 747         if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
 748                 return;
 749
 750         /*
 751          * In case we are out of memory we set IRQTF_AFFINITY again and
 752          * try again next time
 753          */
 754         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
 755                 set_bit(IRQTF_AFFINITY, &action->thread_flags);
 756                 return;
 757         }
 758
 759         raw_spin_lock_irq(&desc->lock);
 760         /*
 761          * This code is triggered unconditionally. Check the affinity
 762          * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
 763          */
 764         if (desc->irq_data.affinity)
 765                 cpumask_copy(mask, desc->irq_data.affinity);
 766         else
 767                 valid = false;
 768         raw_spin_unlock_irq(&desc->lock);
 769
 770         if (valid)
 771                 set_cpus_allowed_ptr(current, mask);
 772         free_cpumask_var(mask);
 773 }
 774 #else
 775 static inline void
 776 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
 777 #endif
 778
 779 /*
 780  * Interrupts which are not explicitely requested as threaded
 781  * interrupts rely on the implicit bh/preempt disable of the hard irq
 782  * context. So we need to disable bh here to avoid deadlocks and other
 783  * side effects.
 784  */
 785 static irqreturn_t
 786 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
 787 {
 788         irqreturn_t ret;
 789
 790         local_bh_disable();
 791         ret = action->thread_fn(action->irq, action->dev_id);
 792         irq_finalize_oneshot(desc, action);
 793         local_bh_enable();
 794         return ret;
 795 }
 796
 797 /*
 798  * Interrupts explicitly requested as threaded interrupts want to be
 799  * preemtible - many of them need to sleep and wait for slow busses to
 800  * complete.
 801  */
 802 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
 803                 struct irqaction *action)
 804 {
 805         irqreturn_t ret;
 806
 807         ret = action->thread_fn(action->irq, action->dev_id);
 808         irq_finalize_oneshot(desc, action);
 809         return ret;
 810 }
 811
 812 static void wake_threads_waitq(struct irq_desc *desc)
 813 {
 814         if (atomic_dec_and_test(&desc->threads_active))
 815                 wake_up(&desc->wait_for_threads);
 816 }
 817
 818 static void irq_thread_dtor(struct callback_head *unused)
 819 {
 820         struct task_struct *tsk = current;
 821         struct irq_desc *desc;
 822         struct irqaction *action;
 823
 824         if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
 825                 return;
 826
 827         action = kthread_data(tsk);
 828
 829         pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
 830                tsk->comm, tsk->pid, action->irq);
 831
 832
 833         desc = irq_to_desc(action->irq);
 834         /*
 835          * If IRQTF_RUNTHREAD is set, we need to decrement
 836          * desc->threads_active and wake possible waiters.
 837          */
 838         if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 839                 wake_threads_waitq(desc);
 840
 841         /* Prevent a stale desc->threads_oneshot */
 842         irq_finalize_oneshot(desc, action);
 843 }
 844
 845 /*
 846  * Interrupt handler thread
 847  */
 848 static int irq_thread(void *data)
 849 {
 850         struct callback_head on_exit_work;
 851         struct irqaction *action = data;
 852         struct irq_desc *desc = irq_to_desc(action->irq);
 853         irqreturn_t (*handler_fn)(struct irq_desc *desc,
 854                         struct irqaction *action);
 855
 856         if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
 857                                         &action->thread_flags))
 858                 handler_fn = irq_forced_thread_fn;
 859         else
 860                 handler_fn = irq_thread_fn;
 861
 862         init_task_work(&on_exit_work, irq_thread_dtor);
 863         task_work_add(current, &on_exit_work, false);
 864
 865         irq_thread_check_affinity(desc, action);
 866
 867         while (!irq_wait_for_interrupt(action)) {
 868                 irqreturn_t action_ret;
 869
 870                 irq_thread_check_affinity(desc, action);
 871
 872                 action_ret = handler_fn(desc, action);
 873                 if (action_ret == IRQ_HANDLED)
 874                         atomic_inc(&desc->threads_handled);
 875
 876                 wake_threads_waitq(desc);
 877         }
 878
 879         /*
 880          * This is the regular exit path. __free_irq() is stopping the
 881          * thread via kthread_stop() after calling
 882          * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
 883          * oneshot mask bit can be set. We cannot verify that as we
 884          * cannot touch the oneshot mask at this point anymore as
 885          * __setup_irq() might have given out currents thread_mask
 886          * again.
 887          */
 888         task_work_cancel(current, irq_thread_dtor);
 889         return 0;
 890 }
 891
 892 /**
 893  *      irq_wake_thread - wake the irq thread for the action identified by dev_id
 894  *      @irq:           Interrupt line
 895  *      @dev_id:        Device identity for which the thread should be woken
 896  *
 897  */
 898 void irq_wake_thread(unsigned int irq, void *dev_id)
 899 {
 900         struct irq_desc *desc = irq_to_desc(irq);
 901         struct irqaction *action;
 902         unsigned long flags;
 903
 904         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 905                 return;
 906
 907         raw_spin_lock_irqsave(&desc->lock, flags);
 908         for (action = desc->action; action; action = action->next) {
 909                 if (action->dev_id == dev_id) {
 910                         if (action->thread)
 911                                 __irq_wake_thread(desc, action);
 912                         break;
 913                 }
 914         }
 915         raw_spin_unlock_irqrestore(&desc->lock, flags);
 916 }
 917 EXPORT_SYMBOL_GPL(irq_wake_thread);
 918
 919 static void irq_setup_forced_threading(struct irqaction *new)
 920 {
 921         if (!force_irqthreads)
 922                 return;
 923         if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
 924                 return;
 925
 926         new->flags |= IRQF_ONESHOT;
 927
 928         if (!new->thread_fn) {
 929                 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
 930                 new->thread_fn = new->handler;
 931                 new->handler = irq_default_primary_handler;
 932         }
 933 }
 934
 935 static int irq_request_resources(struct irq_desc *desc)
 936 {
 937         struct irq_data *d = &desc->irq_data;
 938         struct irq_chip *c = d->chip;
 939
 940         return c->irq_request_resources ? c->irq_request_resources(d) : 0;
 941 }
 942
 943 static void irq_release_resources(struct irq_desc *desc)
 944 {
 945         struct irq_data *d = &desc->irq_data;
 946         struct irq_chip *c = d->chip;
 947
 948         if (c->irq_release_resources)
 949                 c->irq_release_resources(d);
 950 }
 951
 952 /*
 953  * Internal function to register an irqaction - typically used to
 954  * allocate special interrupts that are part of the architecture.
 955  */
 956 static int
 957 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 958 {
 959         struct irqaction *old, **old_ptr;
 960         unsigned long flags, thread_mask = 0;
 961         int ret, nested, shared = 0;
 962         cpumask_var_t mask;
 963
 964         if (!desc)
 965                 return -EINVAL;
 966
 967         if (desc->irq_data.chip == &no_irq_chip)
 968                 return -ENOSYS;
 969         if (!try_module_get(desc->owner))
 970                 return -ENODEV;
 971
 972         /*
 973          * Check whether the interrupt nests into another interrupt
 974          * thread.
 975          */
 976         nested = irq_settings_is_nested_thread(desc);
 977         if (nested) {
 978                 if (!new->thread_fn) {
 979                         ret = -EINVAL;
 980                         goto out_mput;
 981                 }
 982                 /*
 983                  * Replace the primary handler which was provided from
 984                  * the driver for non nested interrupt handling by the
 985                  * dummy function which warns when called.
 986                  */
 987                 new->handler = irq_nested_primary_handler;
 988         } else {
 989                 if (irq_settings_can_thread(desc))
 990                         irq_setup_forced_threading(new);
 991         }
 992
 993         /*
 994          * Create a handler thread when a thread function is supplied
 995          * and the interrupt does not nest into another interrupt
 996          * thread.
 997          */
 998         if (new->thread_fn && !nested) {
 999                 struct task_struct *t;
1000                 static const struct sched_param param = {
1001                         .sched_priority = MAX_USER_RT_PRIO/2,
1002                 };
1003
1004                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1005                                    new->name);
1006                 if (IS_ERR(t)) {
1007                         ret = PTR_ERR(t);
1008                         goto out_mput;
1009                 }
1010
1011                 sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
1012
1013                 /*
1014                  * We keep the reference to the task struct even if
1015                  * the thread dies to avoid that the interrupt code
1016                  * references an already freed task_struct.
1017                  */
1018                 get_task_struct(t);
1019                 new->thread = t;
1020                 /*
1021                  * Tell the thread to set its affinity. This is
1022                  * important for shared interrupt handlers as we do
1023                  * not invoke setup_affinity() for the secondary
1024                  * handlers as everything is already set up. Even for
1025                  * interrupts marked with IRQF_NO_BALANCE this is
1026                  * correct as we want the thread to move to the cpu(s)
1027                  * on which the requesting code placed the interrupt.
1028                  */
1029                 set_bit(IRQTF_AFFINITY, &new->thread_flags);
1030         }
1031
1032         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1033                 ret = -ENOMEM;
1034                 goto out_thread;
1035         }
1036
1037         /*
1038          * Drivers are often written to work w/o knowledge about the
1039          * underlying irq chip implementation, so a request for a
1040          * threaded irq without a primary hard irq context handler
1041          * requires the ONESHOT flag to be set. Some irq chips like
1042          * MSI based interrupts are per se one shot safe. Check the
1043          * chip flags, so we can avoid the unmask dance at the end of
1044          * the threaded handler for those.
1045          */
1046         if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1047                 new->flags &= ~IRQF_ONESHOT;
1048
1049         /*
1050          * The following block of code has to be executed atomically
1051          */
1052         raw_spin_lock_irqsave(&desc->lock, flags);
1053         old_ptr = &desc->action;
1054         old = *old_ptr;
1055         if (old) {
1056                 /*
1057                  * Can't share interrupts unless both agree to and are
1058                  * the same type (level, edge, polarity). So both flag
1059                  * fields must have IRQF_SHARED set and the bits which
1060                  * set the trigger type must match. Also all must
1061                  * agree on ONESHOT.
1062                  */
1063                 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1064                     ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
1065                     ((old->flags ^ new->flags) & IRQF_ONESHOT))
1066                         goto mismatch;
1067
1068                 /* All handlers must agree on per-cpuness */
1069                 if ((old->flags & IRQF_PERCPU) !=
1070                     (new->flags & IRQF_PERCPU))
1071                         goto mismatch;
1072
1073                 /* add new interrupt at end of irq queue */
1074                 do {
1075                         /*
1076                          * Or all existing action->thread_mask bits,
1077                          * so we can find the next zero bit for this
1078                          * new action.
1079                          */
1080                         thread_mask |= old->thread_mask;
1081                         old_ptr = &old->next;
1082                         old = *old_ptr;
1083                 } while (old);
1084                 shared = 1;
1085         }
1086
1087         /*
1088          * Setup the thread mask for this irqaction for ONESHOT. For
1089          * !ONESHOT irqs the thread mask is 0 so we can avoid a
1090          * conditional in irq_wake_thread().
1091          */
1092         if (new->flags & IRQF_ONESHOT) {
1093                 /*
1094                  * Unlikely to have 32 resp 64 irqs sharing one line,
1095                  * but who knows.
1096                  */
1097                 if (thread_mask == ~0UL) {
1098                         ret = -EBUSY;
1099                         goto out_mask;
1100                 }
1101                 /*
1102                  * The thread_mask for the action is or'ed to
1103                  * desc->thread_active to indicate that the
1104                  * IRQF_ONESHOT thread handler has been woken, but not
1105                  * yet finished. The bit is cleared when a thread
1106                  * completes. When all threads of a shared interrupt
1107                  * line have completed desc->threads_active becomes
1108                  * zero and the interrupt line is unmasked. See
1109                  * handle.c:irq_wake_thread() for further information.
1110                  *
1111                  * If no thread is woken by primary (hard irq context)
1112                  * interrupt handlers, then desc->threads_active is
1113                  * also checked for zero to unmask the irq line in the
1114                  * affected hard irq flow handlers
1115                  * (handle_[fasteoi|level]_irq).
1116                  *
1117                  * The new action gets the first zero bit of
1118                  * thread_mask assigned. See the loop above which or's
1119                  * all existing action->thread_mask bits.
1120                  */
1121                 new->thread_mask = 1 << ffz(thread_mask);
1122
1123         } else if (new->handler == irq_default_primary_handler &&
1124                    !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1125                 /*
1126                  * The interrupt was requested with handler = NULL, so
1127                  * we use the default primary handler for it. But it
1128                  * does not have the oneshot flag set. In combination
1129                  * with level interrupts this is deadly, because the
1130                  * default primary handler just wakes the thread, then
1131                  * the irq lines is reenabled, but the device still
1132                  * has the level irq asserted. Rinse and repeat....
1133                  *
1134                  * While this works for edge type interrupts, we play
1135                  * it safe and reject unconditionally because we can't
1136                  * say for sure which type this interrupt really
1137                  * has. The type flags are unreliable as the
1138                  * underlying chip implementation can override them.
1139                  */
1140                 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1141                        irq);
1142                 ret = -EINVAL;
1143                 goto out_mask;
1144         }
1145
1146         if (!shared) {
1147                 ret = irq_request_resources(desc);
1148                 if (ret) {
1149                         pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1150                                new->name, irq, desc->irq_data.chip->name);
1151                         goto out_mask;
1152                 }
1153
1154                 init_waitqueue_head(&desc->wait_for_threads);
1155
1156                 /* Setup the type (level, edge polarity) if configured: */
1157                 if (new->flags & IRQF_TRIGGER_MASK) {
1158                         ret = __irq_set_trigger(desc, irq,
1159                                         new->flags & IRQF_TRIGGER_MASK);
1160
1161                         if (ret)
1162                                 goto out_mask;
1163                 }
1164
1165                 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1166                                   IRQS_ONESHOT | IRQS_WAITING);
1167                 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1168
1169                 if (new->flags & IRQF_PERCPU) {
1170                         irqd_set(&desc->irq_data, IRQD_PER_CPU);
1171                         irq_settings_set_per_cpu(desc);
1172                 }
1173
1174                 if (new->flags & IRQF_ONESHOT)
1175                         desc->istate |= IRQS_ONESHOT;
1176
1177                 if (irq_settings_can_autoenable(desc))
1178                         irq_startup(desc, true);
1179                 else
1180                         /* Undo nested disables: */
1181                         desc->depth = 1;
1182
1183                 /* Exclude IRQ from balancing if requested */
1184                 if (new->flags & IRQF_NOBALANCING) {
1185                         irq_settings_set_no_balancing(desc);
1186                         irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1187                 }
1188
1189                 /* Set default affinity mask once everything is setup */
1190                 setup_affinity(irq, desc, mask);
1191
1192         } else if (new->flags & IRQF_TRIGGER_MASK) {
1193                 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1194                 unsigned int omsk = irq_settings_get_trigger_mask(desc);
1195
1196                 if (nmsk != omsk)
1197                         /* hope the handler works with current  trigger mode */
1198                         pr_warning("irq %d uses trigger mode %u; requested %u\n",
1199                                    irq, nmsk, omsk);
1200         }
1201
1202         new->irq = irq;
1203         *old_ptr = new;
1204
1205         irq_pm_install_action(desc, new);
1206
1207         /* Reset broken irq detection when installing new handler */
1208         desc->irq_count = 0;
1209         desc->irqs_unhandled = 0;
1210
1211         /*
1212          * Check whether we disabled the irq via the spurious handler
1213          * before. Reenable it and give it another chance.
1214          */
1215         if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1216                 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1217                 __enable_irq(desc, irq);
1218         }
1219
1220         raw_spin_unlock_irqrestore(&desc->lock, flags);
1221
1222         /*
1223          * Strictly no need to wake it up, but hung_task complains
1224          * when no hard interrupt wakes the thread up.
1225          */
1226         if (new->thread)
1227                 wake_up_process(new->thread);
1228
1229         register_irq_proc(irq, desc);
1230         new->dir = NULL;
1231         register_handler_proc(irq, new);
1232         free_cpumask_var(mask);
1233
1234         return 0;
1235
1236 mismatch:
1237         if (!(new->flags & IRQF_PROBE_SHARED)) {
1238                 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1239                        irq, new->flags, new->name, old->flags, old->name);
1240 #ifdef CONFIG_DEBUG_SHIRQ
1241                 dump_stack();
1242 #endif
1243         }
1244         ret = -EBUSY;
1245
1246 out_mask:
1247         raw_spin_unlock_irqrestore(&desc->lock, flags);
1248         free_cpumask_var(mask);
1249
1250 out_thread:
1251         if (new->thread) {
1252                 struct task_struct *t = new->thread;
1253
1254                 new->thread = NULL;
1255                 kthread_stop(t);
1256                 put_task_struct(t);
1257         }
1258 out_mput:
1259         module_put(desc->owner);
1260         return ret;
1261 }
1262
1263 /**
1264  *      setup_irq - setup an interrupt
1265  *      @irq: Interrupt line to setup
1266  *      @act: irqaction for the interrupt
1267  *
1268  * Used to statically setup interrupts in the early boot process.
1269  */
1270 int setup_irq(unsigned int irq, struct irqaction *act)
1271 {
1272         int retval;
1273         struct irq_desc *desc = irq_to_desc(irq);
1274
1275         if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1276                 return -EINVAL;
1277         chip_bus_lock(desc);
1278         retval = __setup_irq(irq, desc, act);
1279         chip_bus_sync_unlock(desc);
1280
1281         return retval;
1282 }
1283 EXPORT_SYMBOL_GPL(setup_irq);
1284
1285 /*
1286  * Internal function to unregister an irqaction - used to free
1287  * regular and special interrupts that are part of the architecture.
1288  */
1289 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1290 {
1291         struct irq_desc *desc = irq_to_desc(irq);
1292         struct irqaction *action, **action_ptr;
1293         unsigned long flags;
1294
1295         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1296
1297         if (!desc)
1298                 return NULL;
1299
1300         raw_spin_lock_irqsave(&desc->lock, flags);
1301
1302         /*
1303          * There can be multiple actions per IRQ descriptor, find the right
1304          * one based on the dev_id:
1305          */
1306         action_ptr = &desc->action;
1307         for (;;) {
1308                 action = *action_ptr;
1309
1310                 if (!action) {
1311                         WARN(1, "Trying to free already-free IRQ %d\n", irq);
1312                         raw_spin_unlock_irqrestore(&desc->lock, flags);
1313
1314                         return NULL;
1315                 }
1316
1317                 if (action->dev_id == dev_id)
1318                         break;
1319                 action_ptr = &action->next;
1320         }
1321
1322         /* Found it - now remove it from the list of entries: */
1323         *action_ptr = action->next;
1324
1325         irq_pm_remove_action(desc, action);
1326
1327         /* If this was the last handler, shut down the IRQ line: */
1328         if (!desc->action) {
1329                 irq_shutdown(desc);
1330                 irq_release_resources(desc);
1331         }
1332
1333 #ifdef CONFIG_SMP
1334         /* make sure affinity_hint is cleaned up */
1335         if (WARN_ON_ONCE(desc->affinity_hint))
1336                 desc->affinity_hint = NULL;
1337 #endif
1338
1339         raw_spin_unlock_irqrestore(&desc->lock, flags);
1340
1341         unregister_handler_proc(irq, action);
1342
1343         /* Make sure it's not being used on another CPU: */
1344         synchronize_irq(irq);
1345
1346 #ifdef CONFIG_DEBUG_SHIRQ
1347         /*
1348          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1349          * event to happen even now it's being freed, so let's make sure that
1350          * is so by doing an extra call to the handler ....
1351          *
1352          * ( We do this after actually deregistering it, to make sure that a
1353          *   'real' IRQ doesn't run in * parallel with our fake. )
1354          */
1355         if (action->flags & IRQF_SHARED) {
1356                 local_irq_save(flags);
1357                 action->handler(irq, dev_id);
1358                 local_irq_restore(flags);
1359         }
1360 #endif
1361
1362         if (action->thread) {
1363                 kthread_stop(action->thread);
1364                 put_task_struct(action->thread);
1365         }
1366
1367         module_put(desc->owner);
1368         return action;
1369 }
1370
1371 /**
1372  *      remove_irq - free an interrupt
1373  *      @irq: Interrupt line to free
1374  *      @act: irqaction for the interrupt
1375  *
1376  * Used to remove interrupts statically setup by the early boot process.
1377  */
1378 void remove_irq(unsigned int irq, struct irqaction *act)
1379 {
1380         struct irq_desc *desc = irq_to_desc(irq);
1381
1382         if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1383             __free_irq(irq, act->dev_id);
1384 }
1385 EXPORT_SYMBOL_GPL(remove_irq);
1386
1387 /**
1388  *      free_irq - free an interrupt allocated with request_irq
1389  *      @irq: Interrupt line to free
1390  *      @dev_id: Device identity to free
1391  *
1392  *      Remove an interrupt handler. The handler is removed and if the
1393  *      interrupt line is no longer in use by any driver it is disabled.
1394  *      On a shared IRQ the caller must ensure the interrupt is disabled
1395  *      on the card it drives before calling this function. The function
1396  *      does not return until any executing interrupts for this IRQ
1397  *      have completed.
1398  *
1399  *      This function must not be called from interrupt context.
1400  */
1401 void free_irq(unsigned int irq, void *dev_id)
1402 {
1403         struct irq_desc *desc = irq_to_desc(irq);
1404
1405         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1406                 return;
1407
1408 #ifdef CONFIG_SMP
1409         if (WARN_ON(desc->affinity_notify))
1410                 desc->affinity_notify = NULL;
1411 #endif
1412
1413         chip_bus_lock(desc);
1414         kfree(__free_irq(irq, dev_id));
1415         chip_bus_sync_unlock(desc);
1416 }
1417 EXPORT_SYMBOL(free_irq);
1418
1419 /**
1420  *      request_threaded_irq - allocate an interrupt line
1421  *      @irq: Interrupt line to allocate
1422  *      @handler: Function to be called when the IRQ occurs.
1423  *                Primary handler for threaded interrupts
1424  *                If NULL and thread_fn != NULL the default
1425  *                primary handler is installed
1426  *      @thread_fn: Function called from the irq handler thread
1427  *                  If NULL, no irq thread is created
1428  *      @irqflags: Interrupt type flags
1429  *      @devname: An ascii name for the claiming device
1430  *      @dev_id: A cookie passed back to the handler function
1431  *
1432  *      This call allocates interrupt resources and enables the
1433  *      interrupt line and IRQ handling. From the point this
1434  *      call is made your handler function may be invoked. Since
1435  *      your handler function must clear any interrupt the board
1436  *      raises, you must take care both to initialise your hardware
1437  *      and to set up the interrupt handler in the right order.
1438  *
1439  *      If you want to set up a threaded irq handler for your device
1440  *      then you need to supply @handler and @thread_fn. @handler is
1441  *      still called in hard interrupt context and has to check
1442  *      whether the interrupt originates from the device. If yes it
1443  *      needs to disable the interrupt on the device and return
1444  *      IRQ_WAKE_THREAD which will wake up the handler thread and run
1445  *      @thread_fn. This split handler design is necessary to support
1446  *      shared interrupts.
1447  *
1448  *      Dev_id must be globally unique. Normally the address of the
1449  *      device data structure is used as the cookie. Since the handler
1450  *      receives this value it makes sense to use it.
1451  *
1452  *      If your interrupt is shared you must pass a non NULL dev_id
1453  *      as this is required when freeing the interrupt.
1454  *
1455  *      Flags:
1456  *
1457  *      IRQF_SHARED             Interrupt is shared
1458  *      IRQF_TRIGGER_*          Specify active edge(s) or level
1459  *
1460  */
1461 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1462                          irq_handler_t thread_fn, unsigned long irqflags,
1463                          const char *devname, void *dev_id)
1464 {
1465         struct irqaction *action;
1466         struct irq_desc *desc;
1467         int retval;
1468
1469         /*
1470          * Sanity-check: shared interrupts must pass in a real dev-ID,
1471          * otherwise we'll have trouble later trying to figure out
1472          * which interrupt is which (messes up the interrupt freeing
1473          * logic etc).
1474          */
1475         if ((irqflags & IRQF_SHARED) && !dev_id)
1476                 return -EINVAL;
1477
1478         desc = irq_to_desc(irq);
1479         if (!desc)
1480                 return -EINVAL;
1481
1482         if (!irq_settings_can_request(desc) ||
1483             WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1484                 return -EINVAL;
1485
1486         if (!handler) {
1487                 if (!thread_fn)
1488                         return -EINVAL;
1489                 handler = irq_default_primary_handler;
1490         }
1491
1492         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1493         if (!action)
1494                 return -ENOMEM;
1495
1496         action->handler = handler;
1497         action->thread_fn = thread_fn;
1498         action->flags = irqflags;
1499         action->name = devname;
1500         action->dev_id = dev_id;
1501
1502         chip_bus_lock(desc);
1503         retval = __setup_irq(irq, desc, action);
1504         chip_bus_sync_unlock(desc);
1505
1506         if (retval)
1507                 kfree(action);
1508
1509 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1510         if (!retval && (irqflags & IRQF_SHARED)) {
1511                 /*
1512                  * It's a shared IRQ -- the driver ought to be prepared for it
1513                  * to happen immediately, so let's make sure....
1514                  * We disable the irq to make sure that a 'real' IRQ doesn't
1515                  * run in parallel with our fake.
1516                  */
1517                 unsigned long flags;
1518
1519                 disable_irq(irq);
1520                 local_irq_save(flags);
1521
1522                 handler(irq, dev_id);
1523
1524                 local_irq_restore(flags);
1525                 enable_irq(irq);
1526         }
1527 #endif
1528         return retval;
1529 }
1530 EXPORT_SYMBOL(request_threaded_irq);
1531
1532 /**
1533  *      request_any_context_irq - allocate an interrupt line
1534  *      @irq: Interrupt line to allocate
1535  *      @handler: Function to be called when the IRQ occurs.
1536  *                Threaded handler for threaded interrupts.
1537  *      @flags: Interrupt type flags
1538  *      @name: An ascii name for the claiming device
1539  *      @dev_id: A cookie passed back to the handler function
1540  *
1541  *      This call allocates interrupt resources and enables the
1542  *      interrupt line and IRQ handling. It selects either a
1543  *      hardirq or threaded handling method depending on the
1544  *      context.
1545  *
1546  *      On failure, it returns a negative value. On success,
1547  *      it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1548  */
1549 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1550                             unsigned long flags, const char *name, void *dev_id)
1551 {
1552         struct irq_desc *desc = irq_to_desc(irq);
1553         int ret;
1554
1555         if (!desc)
1556                 return -EINVAL;
1557
1558         if (irq_settings_is_nested_thread(desc)) {
1559                 ret = request_threaded_irq(irq, NULL, handler,
1560                                            flags, name, dev_id);
1561                 return !ret ? IRQC_IS_NESTED : ret;
1562         }
1563
1564         ret = request_irq(irq, handler, flags, name, dev_id);
1565         return !ret ? IRQC_IS_HARDIRQ : ret;
1566 }
1567 EXPORT_SYMBOL_GPL(request_any_context_irq);
1568
1569 void enable_percpu_irq(unsigned int irq, unsigned int type)
1570 {
1571         unsigned int cpu = smp_processor_id();
1572         unsigned long flags;
1573         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1574
1575         if (!desc)
1576                 return;
1577
1578         type &= IRQ_TYPE_SENSE_MASK;
1579         if (type != IRQ_TYPE_NONE) {
1580                 int ret;
1581
1582                 ret = __irq_set_trigger(desc, irq, type);
1583
1584                 if (ret) {
1585                         WARN(1, "failed to set type for IRQ%d\n", irq);
1586                         goto out;
1587                 }
1588         }
1589
1590         irq_percpu_enable(desc, cpu);
1591 out:
1592         irq_put_desc_unlock(desc, flags);
1593 }
1594 EXPORT_SYMBOL_GPL(enable_percpu_irq);
1595
1596 void disable_percpu_irq(unsigned int irq)
1597 {
1598         unsigned int cpu = smp_processor_id();
1599         unsigned long flags;
1600         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1601
1602         if (!desc)
1603                 return;
1604
1605         irq_percpu_disable(desc, cpu);
1606         irq_put_desc_unlock(desc, flags);
1607 }
1608 EXPORT_SYMBOL_GPL(disable_percpu_irq);
1609
1610 /*
1611  * Internal function to unregister a percpu irqaction.
1612  */
1613 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1614 {
1615         struct irq_desc *desc = irq_to_desc(irq);
1616         struct irqaction *action;
1617         unsigned long flags;
1618
1619         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1620
1621         if (!desc)
1622                 return NULL;
1623
1624         raw_spin_lock_irqsave(&desc->lock, flags);
1625
1626         action = desc->action;
1627         if (!action || action->percpu_dev_id != dev_id) {
1628                 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1629                 goto bad;
1630         }
1631
1632         if (!cpumask_empty(desc->percpu_enabled)) {
1633                 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
1634                      irq, cpumask_first(desc->percpu_enabled));
1635                 goto bad;
1636         }
1637
1638         /* Found it - now remove it from the list of entries: */
1639         desc->action = NULL;
1640
1641         raw_spin_unlock_irqrestore(&desc->lock, flags);
1642
1643         unregister_handler_proc(irq, action);
1644
1645         module_put(desc->owner);
1646         return action;
1647
1648 bad:
1649         raw_spin_unlock_irqrestore(&desc->lock, flags);
1650         return NULL;
1651 }
1652
1653 /**
1654  *      remove_percpu_irq - free a per-cpu interrupt
1655  *      @irq: Interrupt line to free
1656  *      @act: irqaction for the interrupt
1657  *
1658  * Used to remove interrupts statically setup by the early boot process.
1659  */
1660 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
1661 {
1662         struct irq_desc *desc = irq_to_desc(irq);
1663
1664         if (desc && irq_settings_is_per_cpu_devid(desc))
1665             __free_percpu_irq(irq, act->percpu_dev_id);
1666 }
1667
1668 /**
1669  *      free_percpu_irq - free an interrupt allocated with request_percpu_irq
1670  *      @irq: Interrupt line to free
1671  *      @dev_id: Device identity to free
1672  *
1673  *      Remove a percpu interrupt handler. The handler is removed, but
1674  *      the interrupt line is not disabled. This must be done on each
1675  *      CPU before calling this function. The function does not return
1676  *      until any executing interrupts for this IRQ have completed.
1677  *
1678  *      This function must not be called from interrupt context.
1679  */
1680 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1681 {
1682         struct irq_desc *desc = irq_to_desc(irq);
1683
1684         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1685                 return;
1686
1687         chip_bus_lock(desc);
1688         kfree(__free_percpu_irq(irq, dev_id));
1689         chip_bus_sync_unlock(desc);
1690 }
1691
1692 /**
1693  *      setup_percpu_irq - setup a per-cpu interrupt
1694  *      @irq: Interrupt line to setup
1695  *      @act: irqaction for the interrupt
1696  *
1697  * Used to statically setup per-cpu interrupts in the early boot process.
1698  */
1699 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
1700 {
1701         struct irq_desc *desc = irq_to_desc(irq);
1702         int retval;
1703
1704         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1705                 return -EINVAL;
1706         chip_bus_lock(desc);
1707         retval = __setup_irq(irq, desc, act);
1708         chip_bus_sync_unlock(desc);
1709
1710         return retval;
1711 }
1712
1713 /**
1714  *      request_percpu_irq - allocate a percpu interrupt line
1715  *      @irq: Interrupt line to allocate
1716  *      @handler: Function to be called when the IRQ occurs.
1717  *      @devname: An ascii name for the claiming device
1718  *      @dev_id: A percpu cookie passed back to the handler function
1719  *
1720  *      This call allocates interrupt resources, but doesn't
1721  *      automatically enable the interrupt. It has to be done on each
1722  *      CPU using enable_percpu_irq().
1723  *
1724  *      Dev_id must be globally unique. It is a per-cpu variable, and
1725  *      the handler gets called with the interrupted CPU's instance of
1726  *      that variable.
1727  */
1728 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
1729                        const char *devname, void __percpu *dev_id)
1730 {
1731         struct irqaction *action;
1732         struct irq_desc *desc;
1733         int retval;
1734
1735         if (!dev_id)
1736                 return -EINVAL;
1737
1738         desc = irq_to_desc(irq);
1739         if (!desc || !irq_settings_can_request(desc) ||
1740             !irq_settings_is_per_cpu_devid(desc))
1741                 return -EINVAL;
1742
1743         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1744         if (!action)
1745                 return -ENOMEM;
1746
1747         action->handler = handler;
1748         action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
1749         action->name = devname;
1750         action->percpu_dev_id = dev_id;
1751
1752         chip_bus_lock(desc);
1753         retval = __setup_irq(irq, desc, action);
1754         chip_bus_sync_unlock(desc);
1755
1756         if (retval)
1757                 kfree(action);
1758
1759         return retval;
1760 }