kernel/irq/irqdesc.c

   1 /*
   2  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
   3  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
   4  *
   5  * This file contains the interrupt descriptor management code
   6  *
   7  * Detailed information is available in Documentation/DocBook/genericirq
   8  *
   9  */
  10 #include <linux/irq.h>
  11 #include <linux/slab.h>
  12 #include <linux/export.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/radix-tree.h>
  16 #include <linux/bitmap.h>
  17 #include <linux/irqdomain.h>
  18 #include <linux/sysfs.h>
  19
  20 #include "internals.h"
  21
  22 /*
  23  * lockdep: we want to handle all irq_desc locks as a single lock-class:
  24  */
  25 static struct lock_class_key irq_desc_lock_class;
  26
  27 #if defined(CONFIG_SMP)
  28 static int __init irq_affinity_setup(char *str)
  29 {
  30         zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
  31         cpulist_parse(str, irq_default_affinity);
  32         /*
  33          * Set at least the boot cpu. We don't want to end up with
  34          * bugreports caused by random comandline masks
  35          */
  36         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
  37         return 1;
  38 }
  39 __setup("irqaffinity=", irq_affinity_setup);
  40
  41 static void __init init_irq_default_affinity(void)
  42 {
  43 #ifdef CONFIG_CPUMASK_OFFSTACK
  44         if (!irq_default_affinity)
  45                 zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
  46 #endif
  47         if (cpumask_empty(irq_default_affinity))
  48                 cpumask_setall(irq_default_affinity);
  49 }
  50 #else
  51 static void __init init_irq_default_affinity(void)
  52 {
  53 }
  54 #endif
  55
  56 #ifdef CONFIG_SMP
  57 static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
  58 {
  59         if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
  60                                      gfp, node))
  61                 return -ENOMEM;
  62
  63 #ifdef CONFIG_GENERIC_PENDING_IRQ
  64         if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
  65                 free_cpumask_var(desc->irq_common_data.affinity);
  66                 return -ENOMEM;
  67         }
  68 #endif
  69         return 0;
  70 }
  71
  72 static void desc_smp_init(struct irq_desc *desc, int node,
  73                           const struct cpumask *affinity)
  74 {
  75         if (!affinity)
  76                 affinity = irq_default_affinity;
  77         cpumask_copy(desc->irq_common_data.affinity, affinity);
  78
  79 #ifdef CONFIG_GENERIC_PENDING_IRQ
  80         cpumask_clear(desc->pending_mask);
  81 #endif
  82 #ifdef CONFIG_NUMA
  83         desc->irq_common_data.node = node;
  84 #endif
  85 }
  86
  87 #else
  88 static inline int
  89 alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
  90 static inline void
  91 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
  92 #endif
  93
  94 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
  95                               const struct cpumask *affinity, struct module *owner)
  96 {
  97         int cpu;
  98
  99         desc->irq_common_data.handler_data = NULL;
 100         desc->irq_common_data.msi_desc = NULL;
 101
 102         desc->irq_data.common = &desc->irq_common_data;
 103         desc->irq_data.irq = irq;
 104         desc->irq_data.chip = &no_irq_chip;
 105         desc->irq_data.chip_data = NULL;
 106         irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
 107         irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
 108         desc->handle_irq = handle_bad_irq;
 109         desc->depth = 1;
 110         desc->irq_count = 0;
 111         desc->irqs_unhandled = 0;
 112         desc->name = NULL;
 113         desc->owner = owner;
 114         for_each_possible_cpu(cpu)
 115                 *per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
 116         desc_smp_init(desc, node, affinity);
 117 }
 118
 119 int nr_irqs = NR_IRQS;
 120 EXPORT_SYMBOL_GPL(nr_irqs);
 121
 122 static DEFINE_MUTEX(sparse_irq_lock);
 123 static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
 124
 125 #ifdef CONFIG_SPARSE_IRQ
 126
 127 static void irq_kobj_release(struct kobject *kobj);
 128
 129 #ifdef CONFIG_SYSFS
 130 static struct kobject *irq_kobj_base;
 131
 132 #define IRQ_ATTR_RO(_name) \
 133 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 134
 135 static ssize_t per_cpu_count_show(struct kobject *kobj,
 136                                   struct kobj_attribute *attr, char *buf)
 137 {
 138         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 139         int cpu, irq = desc->irq_data.irq;
 140         ssize_t ret = 0;
 141         char *p = "";
 142
 143         for_each_possible_cpu(cpu) {
 144                 unsigned int c = kstat_irqs_cpu(irq, cpu);
 145
 146                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c);
 147                 p = ",";
 148         }
 149
 150         ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
 151         return ret;
 152 }
 153 IRQ_ATTR_RO(per_cpu_count);
 154
 155 static ssize_t chip_name_show(struct kobject *kobj,
 156                               struct kobj_attribute *attr, char *buf)
 157 {
 158         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 159         ssize_t ret = 0;
 160
 161         raw_spin_lock_irq(&desc->lock);
 162         if (desc->irq_data.chip && desc->irq_data.chip->name) {
 163                 ret = scnprintf(buf, PAGE_SIZE, "%s\n",
 164                                 desc->irq_data.chip->name);
 165         }
 166         raw_spin_unlock_irq(&desc->lock);
 167
 168         return ret;
 169 }
 170 IRQ_ATTR_RO(chip_name);
 171
 172 static ssize_t hwirq_show(struct kobject *kobj,
 173                           struct kobj_attribute *attr, char *buf)
 174 {
 175         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 176         ssize_t ret = 0;
 177
 178         raw_spin_lock_irq(&desc->lock);
 179         if (desc->irq_data.domain)
 180                 ret = sprintf(buf, "%d\n", (int)desc->irq_data.hwirq);
 181         raw_spin_unlock_irq(&desc->lock);
 182
 183         return ret;
 184 }
 185 IRQ_ATTR_RO(hwirq);
 186
 187 static ssize_t type_show(struct kobject *kobj,
 188                          struct kobj_attribute *attr, char *buf)
 189 {
 190         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 191         ssize_t ret = 0;
 192
 193         raw_spin_lock_irq(&desc->lock);
 194         ret = sprintf(buf, "%s\n",
 195                       irqd_is_level_type(&desc->irq_data) ? "level" : "edge");
 196         raw_spin_unlock_irq(&desc->lock);
 197
 198         return ret;
 199
 200 }
 201 IRQ_ATTR_RO(type);
 202
 203 static ssize_t name_show(struct kobject *kobj,
 204                          struct kobj_attribute *attr, char *buf)
 205 {
 206         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 207         ssize_t ret = 0;
 208
 209         raw_spin_lock_irq(&desc->lock);
 210         if (desc->name)
 211                 ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name);
 212         raw_spin_unlock_irq(&desc->lock);
 213
 214         return ret;
 215 }
 216 IRQ_ATTR_RO(name);
 217
 218 static ssize_t actions_show(struct kobject *kobj,
 219                             struct kobj_attribute *attr, char *buf)
 220 {
 221         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 222         struct irqaction *action;
 223         ssize_t ret = 0;
 224         char *p = "";
 225
 226         raw_spin_lock_irq(&desc->lock);
 227         for (action = desc->action; action != NULL; action = action->next) {
 228                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
 229                                  p, action->name);
 230                 p = ",";
 231         }
 232         raw_spin_unlock_irq(&desc->lock);
 233
 234         if (ret)
 235                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
 236
 237         return ret;
 238 }
 239 IRQ_ATTR_RO(actions);
 240
 241 static struct attribute *irq_attrs[] = {
 242         &per_cpu_count_attr.attr,
 243         &chip_name_attr.attr,
 244         &hwirq_attr.attr,
 245         &type_attr.attr,
 246         &name_attr.attr,
 247         &actions_attr.attr,
 248         NULL
 249 };
 250
 251 static struct kobj_type irq_kobj_type = {
 252         .release        = irq_kobj_release,
 253         .sysfs_ops      = &kobj_sysfs_ops,
 254         .default_attrs  = irq_attrs,
 255 };
 256
 257 static void irq_sysfs_add(int irq, struct irq_desc *desc)
 258 {
 259         if (irq_kobj_base) {
 260                 /*
 261                  * Continue even in case of failure as this is nothing
 262                  * crucial.
 263                  */
 264                 if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))
 265                         pr_warn("Failed to add kobject for irq %d\n", irq);
 266         }
 267 }
 268
 269 static int __init irq_sysfs_init(void)
 270 {
 271         struct irq_desc *desc;
 272         int irq;
 273
 274         /* Prevent concurrent irq alloc/free */
 275         irq_lock_sparse();
 276
 277         irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);
 278         if (!irq_kobj_base) {
 279                 irq_unlock_sparse();
 280                 return -ENOMEM;
 281         }
 282
 283         /* Add the already allocated interrupts */
 284         for_each_irq_desc(irq, desc)
 285                 irq_sysfs_add(irq, desc);
 286         irq_unlock_sparse();
 287
 288         return 0;
 289 }
 290 postcore_initcall(irq_sysfs_init);
 291
 292 #else /* !CONFIG_SYSFS */
 293
 294 static struct kobj_type irq_kobj_type = {
 295         .release        = irq_kobj_release,
 296 };
 297
 298 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}
 299
 300 #endif /* CONFIG_SYSFS */
 301
 302 static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
 303
 304 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
 305 {
 306         radix_tree_insert(&irq_desc_tree, irq, desc);
 307 }
 308
 309 struct irq_desc *irq_to_desc(unsigned int irq)
 310 {
 311         return radix_tree_lookup(&irq_desc_tree, irq);
 312 }
 313 EXPORT_SYMBOL(irq_to_desc);
 314
 315 static void delete_irq_desc(unsigned int irq)
 316 {
 317         radix_tree_delete(&irq_desc_tree, irq);
 318 }
 319
 320 #ifdef CONFIG_SMP
 321 static void free_masks(struct irq_desc *desc)
 322 {
 323 #ifdef CONFIG_GENERIC_PENDING_IRQ
 324         free_cpumask_var(desc->pending_mask);
 325 #endif
 326         free_cpumask_var(desc->irq_common_data.affinity);
 327 }
 328 #else
 329 static inline void free_masks(struct irq_desc *desc) { }
 330 #endif
 331
 332 void irq_lock_sparse(void)
 333 {
 334         mutex_lock(&sparse_irq_lock);
 335 }
 336
 337 void irq_unlock_sparse(void)
 338 {
 339         mutex_unlock(&sparse_irq_lock);
 340 }
 341
 342 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
 343                                    const struct cpumask *affinity,
 344                                    struct module *owner)
 345 {
 346         struct irq_desc *desc;
 347         gfp_t gfp = GFP_KERNEL;
 348
 349         desc = kzalloc_node(sizeof(*desc), gfp, node);
 350         if (!desc)
 351                 return NULL;
 352         /* allocate based on nr_cpu_ids */
 353         desc->kstat_irqs = alloc_percpu(unsigned int);
 354         if (!desc->kstat_irqs)
 355                 goto err_desc;
 356
 357         if (alloc_masks(desc, gfp, node))
 358                 goto err_kstat;
 359
 360         raw_spin_lock_init(&desc->lock);
 361         lockdep_set_class(&desc->lock, &irq_desc_lock_class);
 362         init_rcu_head(&desc->rcu);
 363
 364         desc_set_defaults(irq, desc, node, affinity, owner);
 365         irqd_set(&desc->irq_data, flags);
 366         kobject_init(&desc->kobj, &irq_kobj_type);
 367
 368         return desc;
 369
 370 err_kstat:
 371         free_percpu(desc->kstat_irqs);
 372 err_desc:
 373         kfree(desc);
 374         return NULL;
 375 }
 376
 377 static void irq_kobj_release(struct kobject *kobj)
 378 {
 379         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 380
 381         free_masks(desc);
 382         free_percpu(desc->kstat_irqs);
 383         kfree(desc);
 384 }
 385
 386 static void delayed_free_desc(struct rcu_head *rhp)
 387 {
 388         struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);
 389
 390         kobject_put(&desc->kobj);
 391 }
 392
 393 static void free_desc(unsigned int irq)
 394 {
 395         struct irq_desc *desc = irq_to_desc(irq);
 396
 397         unregister_irq_proc(irq, desc);
 398
 399         /*
 400          * sparse_irq_lock protects also show_interrupts() and
 401          * kstat_irq_usr(). Once we deleted the descriptor from the
 402          * sparse tree we can free it. Access in proc will fail to
 403          * lookup the descriptor.
 404          *
 405          * The sysfs entry must be serialized against a concurrent
 406          * irq_sysfs_init() as well.
 407          */
 408         mutex_lock(&sparse_irq_lock);
 409         kobject_del(&desc->kobj);
 410         delete_irq_desc(irq);
 411         mutex_unlock(&sparse_irq_lock);
 412
 413         /*
 414          * We free the descriptor, masks and stat fields via RCU. That
 415          * allows demultiplex interrupts to do rcu based management of
 416          * the child interrupts.
 417          */
 418         call_rcu(&desc->rcu, delayed_free_desc);
 419 }
 420
 421 static int alloc_descs(unsigned int start, unsigned int cnt, int node,
 422                        const struct cpumask *affinity, struct module *owner)
 423 {
 424         const struct cpumask *mask = NULL;
 425         struct irq_desc *desc;
 426         unsigned int flags;
 427         int i;
 428
 429         /* Validate affinity mask(s) */
 430         if (affinity) {
 431                 for (i = 0, mask = affinity; i < cnt; i++, mask++) {
 432                         if (cpumask_empty(mask))
 433                                 return -EINVAL;
 434                 }
 435         }
 436
 437         flags = affinity ? IRQD_AFFINITY_MANAGED : 0;
 438         mask = NULL;
 439
 440         for (i = 0; i < cnt; i++) {
 441                 if (affinity) {
 442                         node = cpu_to_node(cpumask_first(affinity));
 443                         mask = affinity;
 444                         affinity++;
 445                 }
 446                 desc = alloc_desc(start + i, node, flags, mask, owner);
 447                 if (!desc)
 448                         goto err;
 449                 mutex_lock(&sparse_irq_lock);
 450                 irq_insert_desc(start + i, desc);
 451                 irq_sysfs_add(start + i, desc);
 452                 mutex_unlock(&sparse_irq_lock);
 453         }
 454         return start;
 455
 456 err:
 457         for (i--; i >= 0; i--)
 458                 free_desc(start + i);
 459
 460         mutex_lock(&sparse_irq_lock);
 461         bitmap_clear(allocated_irqs, start, cnt);
 462         mutex_unlock(&sparse_irq_lock);
 463         return -ENOMEM;
 464 }
 465
 466 static int irq_expand_nr_irqs(unsigned int nr)
 467 {
 468         if (nr > IRQ_BITMAP_BITS)
 469                 return -ENOMEM;
 470         nr_irqs = nr;
 471         return 0;
 472 }
 473
 474 int __init early_irq_init(void)
 475 {
 476         int i, initcnt, node = first_online_node;
 477         struct irq_desc *desc;
 478
 479         init_irq_default_affinity();
 480
 481         /* Let arch update nr_irqs and return the nr of preallocated irqs */
 482         initcnt = arch_probe_nr_irqs();
 483         printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);
 484
 485         if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
 486                 nr_irqs = IRQ_BITMAP_BITS;
 487
 488         if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
 489                 initcnt = IRQ_BITMAP_BITS;
 490
 491         if (initcnt > nr_irqs)
 492                 nr_irqs = initcnt;
 493
 494         for (i = 0; i < initcnt; i++) {
 495                 desc = alloc_desc(i, node, 0, NULL, NULL);
 496                 set_bit(i, allocated_irqs);
 497                 irq_insert_desc(i, desc);
 498         }
 499         return arch_early_irq_init();
 500 }
 501
 502 #else /* !CONFIG_SPARSE_IRQ */
 503
 504 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
 505         [0 ... NR_IRQS-1] = {
 506                 .handle_irq     = handle_bad_irq,
 507                 .depth          = 1,
 508                 .lock           = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
 509         }
 510 };
 511
 512 int __init early_irq_init(void)
 513 {
 514         int count, i, node = first_online_node;
 515         struct irq_desc *desc;
 516
 517         init_irq_default_affinity();
 518
 519         printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
 520
 521         desc = irq_desc;
 522         count = ARRAY_SIZE(irq_desc);
 523
 524         for (i = 0; i < count; i++) {
 525                 desc[i].kstat_irqs = alloc_percpu(unsigned int);
 526                 alloc_masks(&desc[i], GFP_KERNEL, node);
 527                 raw_spin_lock_init(&desc[i].lock);
 528                 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
 529                 desc_set_defaults(i, &desc[i], node, NULL, NULL);
 530         }
 531         return arch_early_irq_init();
 532 }
 533
 534 struct irq_desc *irq_to_desc(unsigned int irq)
 535 {
 536         return (irq < NR_IRQS) ? irq_desc + irq : NULL;
 537 }
 538 EXPORT_SYMBOL(irq_to_desc);
 539
 540 static void free_desc(unsigned int irq)
 541 {
 542         struct irq_desc *desc = irq_to_desc(irq);
 543         unsigned long flags;
 544
 545         raw_spin_lock_irqsave(&desc->lock, flags);
 546         desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);
 547         raw_spin_unlock_irqrestore(&desc->lock, flags);
 548 }
 549
 550 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
 551                               const struct cpumask *affinity,
 552                               struct module *owner)
 553 {
 554         u32 i;
 555
 556         for (i = 0; i < cnt; i++) {
 557                 struct irq_desc *desc = irq_to_desc(start + i);
 558
 559                 desc->owner = owner;
 560         }
 561         return start;
 562 }
 563
 564 static int irq_expand_nr_irqs(unsigned int nr)
 565 {
 566         return -ENOMEM;
 567 }
 568
 569 void irq_mark_irq(unsigned int irq)
 570 {
 571         mutex_lock(&sparse_irq_lock);
 572         bitmap_set(allocated_irqs, irq, 1);
 573         mutex_unlock(&sparse_irq_lock);
 574 }
 575
 576 #ifdef CONFIG_GENERIC_IRQ_LEGACY
 577 void irq_init_desc(unsigned int irq)
 578 {
 579         free_desc(irq);
 580 }
 581 #endif
 582
 583 #endif /* !CONFIG_SPARSE_IRQ */
 584
 585 /**
 586  * generic_handle_irq - Invoke the handler for a particular irq
 587  * @irq:        The irq number to handle
 588  *
 589  */
 590 int generic_handle_irq(unsigned int irq)
 591 {
 592         struct irq_desc *desc = irq_to_desc(irq);
 593
 594         if (!desc)
 595                 return -EINVAL;
 596         generic_handle_irq_desc(desc);
 597         return 0;
 598 }
 599 EXPORT_SYMBOL_GPL(generic_handle_irq);
 600
 601 #ifdef CONFIG_HANDLE_DOMAIN_IRQ
 602 /**
 603  * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
 604  * @domain:     The domain where to perform the lookup
 605  * @hwirq:      The HW irq number to convert to a logical one
 606  * @lookup:     Whether to perform the domain lookup or not
 607  * @regs:       Register file coming from the low-level handling code
 608  *
 609  * Returns:     0 on success, or -EINVAL if conversion has failed
 610  */
 611 int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
 612                         bool lookup, struct pt_regs *regs)
 613 {
 614         struct pt_regs *old_regs = set_irq_regs(regs);
 615         unsigned int irq = hwirq;
 616         int ret = 0;
 617
 618         irq_enter();
 619
 620 #ifdef CONFIG_IRQ_DOMAIN
 621         if (lookup)
 622                 irq = irq_find_mapping(domain, hwirq);
 623 #endif
 624
 625         /*
 626          * Some hardware gives randomly wrong interrupts.  Rather
 627          * than crashing, do something sensible.
 628          */
 629         if (unlikely(!irq || irq >= nr_irqs)) {
 630                 ack_bad_irq(irq);
 631                 ret = -EINVAL;
 632         } else {
 633                 generic_handle_irq(irq);
 634         }
 635
 636         irq_exit();
 637         set_irq_regs(old_regs);
 638         return ret;
 639 }
 640 #endif
 641
 642 /* Dynamic interrupt handling */
 643
 644 /**
 645  * irq_free_descs - free irq descriptors
 646  * @from:       Start of descriptor range
 647  * @cnt:        Number of consecutive irqs to free
 648  */
 649 void irq_free_descs(unsigned int from, unsigned int cnt)
 650 {
 651         int i;
 652
 653         if (from >= nr_irqs || (from + cnt) > nr_irqs)
 654                 return;
 655
 656         for (i = 0; i < cnt; i++)
 657                 free_desc(from + i);
 658
 659         mutex_lock(&sparse_irq_lock);
 660         bitmap_clear(allocated_irqs, from, cnt);
 661         mutex_unlock(&sparse_irq_lock);
 662 }
 663 EXPORT_SYMBOL_GPL(irq_free_descs);
 664
 665 /**
 666  * irq_alloc_descs - allocate and initialize a range of irq descriptors
 667  * @irq:        Allocate for specific irq number if irq >= 0
 668  * @from:       Start the search from this irq number
 669  * @cnt:        Number of consecutive irqs to allocate.
 670  * @node:       Preferred node on which the irq descriptor should be allocated
 671  * @owner:      Owning module (can be NULL)
 672  * @affinity:   Optional pointer to an affinity mask array of size @cnt which
 673  *              hints where the irq descriptors should be allocated and which
 674  *              default affinities to use
 675  *
 676  * Returns the first irq number or error code
 677  */
 678 int __ref
 679 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
 680                   struct module *owner, const struct cpumask *affinity)
 681 {
 682         int start, ret;
 683
 684         if (!cnt)
 685                 return -EINVAL;
 686
 687         if (irq >= 0) {
 688                 if (from > irq)
 689                         return -EINVAL;
 690                 from = irq;
 691         } else {
 692                 /*
 693                  * For interrupts which are freely allocated the
 694                  * architecture can force a lower bound to the @from
 695                  * argument. x86 uses this to exclude the GSI space.
 696                  */
 697                 from = arch_dynirq_lower_bound(from);
 698         }
 699
 700         mutex_lock(&sparse_irq_lock);
 701
 702         start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
 703                                            from, cnt, 0);
 704         ret = -EEXIST;
 705         if (irq >=0 && start != irq)
 706                 goto err;
 707
 708         if (start + cnt > nr_irqs) {
 709                 ret = irq_expand_nr_irqs(start + cnt);
 710                 if (ret)
 711                         goto err;
 712         }
 713
 714         bitmap_set(allocated_irqs, start, cnt);
 715         mutex_unlock(&sparse_irq_lock);
 716         return alloc_descs(start, cnt, node, affinity, owner);
 717
 718 err:
 719         mutex_unlock(&sparse_irq_lock);
 720         return ret;
 721 }
 722 EXPORT_SYMBOL_GPL(__irq_alloc_descs);
 723
 724 #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
 725 /**
 726  * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
 727  * @cnt:        number of interrupts to allocate
 728  * @node:       node on which to allocate
 729  *
 730  * Returns an interrupt number > 0 or 0, if the allocation fails.
 731  */
 732 unsigned int irq_alloc_hwirqs(int cnt, int node)
 733 {
 734         int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL, NULL);
 735
 736         if (irq < 0)
 737                 return 0;
 738
 739         for (i = irq; cnt > 0; i++, cnt--) {
 740                 if (arch_setup_hwirq(i, node))
 741                         goto err;
 742                 irq_clear_status_flags(i, _IRQ_NOREQUEST);
 743         }
 744         return irq;
 745
 746 err:
 747         for (i--; i >= irq; i--) {
 748                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
 749                 arch_teardown_hwirq(i);
 750         }
 751         irq_free_descs(irq, cnt);
 752         return 0;
 753 }
 754 EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);
 755
 756 /**
 757  * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
 758  * @from:       Free from irq number
 759  * @cnt:        number of interrupts to free
 760  *
 761  */
 762 void irq_free_hwirqs(unsigned int from, int cnt)
 763 {
 764         int i, j;
 765
 766         for (i = from, j = cnt; j > 0; i++, j--) {
 767                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
 768                 arch_teardown_hwirq(i);
 769         }
 770         irq_free_descs(from, cnt);
 771 }
 772 EXPORT_SYMBOL_GPL(irq_free_hwirqs);
 773 #endif
 774
 775 /**
 776  * irq_get_next_irq - get next allocated irq number
 777  * @offset:     where to start the search
 778  *
 779  * Returns next irq number after offset or nr_irqs if none is found.
 780  */
 781 unsigned int irq_get_next_irq(unsigned int offset)
 782 {
 783         return find_next_bit(allocated_irqs, nr_irqs, offset);
 784 }
 785
 786 struct irq_desc *
 787 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
 788                     unsigned int check)
 789 {
 790         struct irq_desc *desc = irq_to_desc(irq);
 791
 792         if (desc) {
 793                 if (check & _IRQ_DESC_CHECK) {
 794                         if ((check & _IRQ_DESC_PERCPU) &&
 795                             !irq_settings_is_per_cpu_devid(desc))
 796                                 return NULL;
 797
 798                         if (!(check & _IRQ_DESC_PERCPU) &&
 799                             irq_settings_is_per_cpu_devid(desc))
 800                                 return NULL;
 801                 }
 802
 803                 if (bus)
 804                         chip_bus_lock(desc);
 805                 raw_spin_lock_irqsave(&desc->lock, *flags);
 806         }
 807         return desc;
 808 }
 809
 810 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
 811 {
 812         raw_spin_unlock_irqrestore(&desc->lock, flags);
 813         if (bus)
 814                 chip_bus_sync_unlock(desc);
 815 }
 816
 817 int irq_set_percpu_devid_partition(unsigned int irq,
 818                                    const struct cpumask *affinity)
 819 {
 820         struct irq_desc *desc = irq_to_desc(irq);
 821
 822         if (!desc)
 823                 return -EINVAL;
 824
 825         if (desc->percpu_enabled)
 826                 return -EINVAL;
 827
 828         desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
 829
 830         if (!desc->percpu_enabled)
 831                 return -ENOMEM;
 832
 833         if (affinity)
 834                 desc->percpu_affinity = affinity;
 835         else
 836                 desc->percpu_affinity = cpu_possible_mask;
 837
 838         irq_set_percpu_devid_flags(irq);
 839         return 0;
 840 }
 841
 842 int irq_set_percpu_devid(unsigned int irq)
 843 {
 844         return irq_set_percpu_devid_partition(irq, NULL);
 845 }
 846
 847 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)
 848 {
 849         struct irq_desc *desc = irq_to_desc(irq);
 850
 851         if (!desc || !desc->percpu_enabled)
 852                 return -EINVAL;
 853
 854         if (affinity)
 855                 cpumask_copy(affinity, desc->percpu_affinity);
 856
 857         return 0;
 858 }
 859
 860 void kstat_incr_irq_this_cpu(unsigned int irq)
 861 {
 862         kstat_incr_irqs_this_cpu(irq_to_desc(irq));
 863 }
 864
 865 /**
 866  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
 867  * @irq:        The interrupt number
 868  * @cpu:        The cpu number
 869  *
 870  * Returns the sum of interrupt counts on @cpu since boot for
 871  * @irq. The caller must ensure that the interrupt is not removed
 872  * concurrently.
 873  */
 874 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
 875 {
 876         struct irq_desc *desc = irq_to_desc(irq);
 877
 878         return desc && desc->kstat_irqs ?
 879                         *per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
 880 }
 881
 882 /**
 883  * kstat_irqs - Get the statistics for an interrupt
 884  * @irq:        The interrupt number
 885  *
 886  * Returns the sum of interrupt counts on all cpus since boot for
 887  * @irq. The caller must ensure that the interrupt is not removed
 888  * concurrently.
 889  */
 890 unsigned int kstat_irqs(unsigned int irq)
 891 {
 892         struct irq_desc *desc = irq_to_desc(irq);
 893         int cpu;
 894         unsigned int sum = 0;
 895
 896         if (!desc || !desc->kstat_irqs)
 897                 return 0;
 898         for_each_possible_cpu(cpu)
 899                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 900         return sum;
 901 }
 902
 903 /**
 904  * kstat_irqs_usr - Get the statistics for an interrupt
 905  * @irq:        The interrupt number
 906  *
 907  * Returns the sum of interrupt counts on all cpus since boot for
 908  * @irq. Contrary to kstat_irqs() this can be called from any
 909  * preemptible context. It's protected against concurrent removal of
 910  * an interrupt descriptor when sparse irqs are enabled.
 911  */
 912 unsigned int kstat_irqs_usr(unsigned int irq)
 913 {
 914         unsigned int sum;
 915
 916         irq_lock_sparse();
 917         sum = kstat_irqs(irq);
 918         irq_unlock_sparse();
 919         return sum;
 920 }