drivers/irqchip/irq-mips-gic.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
   8  */
   9 #include <linux/bitmap.h>
  10 #include <linux/clocksource.h>
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/irq.h>
  14 #include <linux/irqchip.h>
  15 #include <linux/irqchip/mips-gic.h>
  16 #include <linux/of_address.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19
  20 #include <asm/mips-cm.h>
  21 #include <asm/setup.h>
  22 #include <asm/traps.h>
  23
  24 #include <dt-bindings/interrupt-controller/mips-gic.h>
  25
  26 unsigned int gic_present;
  27
  28 struct gic_pcpu_mask {
  29         DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
  30 };
  31
  32 struct gic_irq_spec {
  33         enum {
  34                 GIC_DEVICE,
  35                 GIC_IPI
  36         } type;
  37
  38         union {
  39                 struct cpumask *ipimask;
  40                 unsigned int hwirq;
  41         };
  42 };
  43
  44 static unsigned long __gic_base_addr;
  45
  46 static void __iomem *gic_base;
  47 static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
  48 static DEFINE_SPINLOCK(gic_lock);
  49 static struct irq_domain *gic_irq_domain;
  50 static struct irq_domain *gic_dev_domain;
  51 static struct irq_domain *gic_ipi_domain;
  52 static int gic_shared_intrs;
  53 static int gic_vpes;
  54 static unsigned int gic_cpu_pin;
  55 static unsigned int timer_cpu_pin;
  56 static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
  57 DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
  58
  59 static void __gic_irq_dispatch(void);
  60
  61 static inline u32 gic_read32(unsigned int reg)
  62 {
  63         return __raw_readl(gic_base + reg);
  64 }
  65
  66 static inline u64 gic_read64(unsigned int reg)
  67 {
  68         return __raw_readq(gic_base + reg);
  69 }
  70
  71 static inline unsigned long gic_read(unsigned int reg)
  72 {
  73         if (!mips_cm_is64)
  74                 return gic_read32(reg);
  75         else
  76                 return gic_read64(reg);
  77 }
  78
  79 static inline void gic_write32(unsigned int reg, u32 val)
  80 {
  81         return __raw_writel(val, gic_base + reg);
  82 }
  83
  84 static inline void gic_write64(unsigned int reg, u64 val)
  85 {
  86         return __raw_writeq(val, gic_base + reg);
  87 }
  88
  89 static inline void gic_write(unsigned int reg, unsigned long val)
  90 {
  91         if (!mips_cm_is64)
  92                 return gic_write32(reg, (u32)val);
  93         else
  94                 return gic_write64(reg, (u64)val);
  95 }
  96
  97 static inline void gic_update_bits(unsigned int reg, unsigned long mask,
  98                                    unsigned long val)
  99 {
 100         unsigned long regval;
 101
 102         regval = gic_read(reg);
 103         regval &= ~mask;
 104         regval |= val;
 105         gic_write(reg, regval);
 106 }
 107
 108 static inline void gic_reset_mask(unsigned int intr)
 109 {
 110         gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
 111                   1ul << GIC_INTR_BIT(intr));
 112 }
 113
 114 static inline void gic_set_mask(unsigned int intr)
 115 {
 116         gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
 117                   1ul << GIC_INTR_BIT(intr));
 118 }
 119
 120 static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
 121 {
 122         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
 123                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 124                         (unsigned long)pol << GIC_INTR_BIT(intr));
 125 }
 126
 127 static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
 128 {
 129         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
 130                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 131                         (unsigned long)trig << GIC_INTR_BIT(intr));
 132 }
 133
 134 static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
 135 {
 136         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
 137                         1ul << GIC_INTR_BIT(intr),
 138                         (unsigned long)dual << GIC_INTR_BIT(intr));
 139 }
 140
 141 static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
 142 {
 143         gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
 144                     GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
 145 }
 146
 147 static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
 148 {
 149         gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
 150                   GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
 151                   GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
 152 }
 153
 154 #ifdef CONFIG_CLKSRC_MIPS_GIC
 155 cycle_t gic_read_count(void)
 156 {
 157         unsigned int hi, hi2, lo;
 158
 159         if (mips_cm_is64)
 160                 return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
 161
 162         do {
 163                 hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 164                 lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
 165                 hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 166         } while (hi2 != hi);
 167
 168         return (((cycle_t) hi) << 32) + lo;
 169 }
 170
 171 unsigned int gic_get_count_width(void)
 172 {
 173         unsigned int bits, config;
 174
 175         config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 176         bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
 177                          GIC_SH_CONFIG_COUNTBITS_SHF);
 178
 179         return bits;
 180 }
 181
 182 void gic_write_compare(cycle_t cnt)
 183 {
 184         if (mips_cm_is64) {
 185                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
 186         } else {
 187                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
 188                                         (int)(cnt >> 32));
 189                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
 190                                         (int)(cnt & 0xffffffff));
 191         }
 192 }
 193
 194 void gic_write_cpu_compare(cycle_t cnt, int cpu)
 195 {
 196         unsigned long flags;
 197
 198         local_irq_save(flags);
 199
 200         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), mips_cm_vp_id(cpu));
 201
 202         if (mips_cm_is64) {
 203                 gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
 204         } else {
 205                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
 206                                         (int)(cnt >> 32));
 207                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
 208                                         (int)(cnt & 0xffffffff));
 209         }
 210
 211         local_irq_restore(flags);
 212 }
 213
 214 cycle_t gic_read_compare(void)
 215 {
 216         unsigned int hi, lo;
 217
 218         if (mips_cm_is64)
 219                 return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
 220
 221         hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
 222         lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
 223
 224         return (((cycle_t) hi) << 32) + lo;
 225 }
 226
 227 void gic_start_count(void)
 228 {
 229         u32 gicconfig;
 230
 231         /* Start the counter */
 232         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 233         gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
 234         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 235 }
 236
 237 void gic_stop_count(void)
 238 {
 239         u32 gicconfig;
 240
 241         /* Stop the counter */
 242         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 243         gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
 244         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 245 }
 246
 247 #endif
 248
 249 unsigned gic_read_local_vp_id(void)
 250 {
 251         unsigned long ident;
 252
 253         ident = gic_read(GIC_REG(VPE_LOCAL, GIC_VP_IDENT));
 254         return ident & GIC_VP_IDENT_VCNUM_MSK;
 255 }
 256
 257 static bool gic_local_irq_is_routable(int intr)
 258 {
 259         u32 vpe_ctl;
 260
 261         /* All local interrupts are routable in EIC mode. */
 262         if (cpu_has_veic)
 263                 return true;
 264
 265         vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
 266         switch (intr) {
 267         case GIC_LOCAL_INT_TIMER:
 268                 return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
 269         case GIC_LOCAL_INT_PERFCTR:
 270                 return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
 271         case GIC_LOCAL_INT_FDC:
 272                 return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
 273         case GIC_LOCAL_INT_SWINT0:
 274         case GIC_LOCAL_INT_SWINT1:
 275                 return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
 276         default:
 277                 return true;
 278         }
 279 }
 280
 281 static void gic_bind_eic_interrupt(int irq, int set)
 282 {
 283         /* Convert irq vector # to hw int # */
 284         irq -= GIC_PIN_TO_VEC_OFFSET;
 285
 286         /* Set irq to use shadow set */
 287         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
 288                   GIC_VPE_EIC_SS(irq), set);
 289 }
 290
 291 static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
 292 {
 293         irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
 294
 295         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(hwirq));
 296 }
 297
 298 int gic_get_c0_compare_int(void)
 299 {
 300         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
 301                 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
 302         return irq_create_mapping(gic_irq_domain,
 303                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
 304 }
 305
 306 int gic_get_c0_perfcount_int(void)
 307 {
 308         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
 309                 /* Is the performance counter shared with the timer? */
 310                 if (cp0_perfcount_irq < 0)
 311                         return -1;
 312                 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
 313         }
 314         return irq_create_mapping(gic_irq_domain,
 315                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
 316 }
 317
 318 int gic_get_c0_fdc_int(void)
 319 {
 320         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
 321                 /* Is the FDC IRQ even present? */
 322                 if (cp0_fdc_irq < 0)
 323                         return -1;
 324                 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
 325         }
 326
 327         return irq_create_mapping(gic_irq_domain,
 328                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
 329 }
 330
 331 int gic_get_usm_range(struct resource *gic_usm_res)
 332 {
 333         if (!gic_present)
 334                 return -1;
 335
 336         gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
 337         gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
 338
 339         return 0;
 340 }
 341
 342 static void gic_handle_shared_int(bool chained)
 343 {
 344         unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
 345         unsigned long *pcpu_mask;
 346         unsigned long pending_reg, intrmask_reg;
 347         DECLARE_BITMAP(pending, GIC_MAX_INTRS);
 348         DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
 349
 350         /* Get per-cpu bitmaps */
 351         pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
 352
 353         pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
 354         intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
 355
 356         for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
 357                 pending[i] = gic_read(pending_reg);
 358                 intrmask[i] = gic_read(intrmask_reg);
 359                 pending_reg += gic_reg_step;
 360                 intrmask_reg += gic_reg_step;
 361
 362                 if (!IS_ENABLED(CONFIG_64BIT) || mips_cm_is64)
 363                         continue;
 364
 365                 pending[i] |= (u64)gic_read(pending_reg) << 32;
 366                 intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
 367                 pending_reg += gic_reg_step;
 368                 intrmask_reg += gic_reg_step;
 369         }
 370
 371         bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 372         bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 373
 374         intr = find_first_bit(pending, gic_shared_intrs);
 375         while (intr != gic_shared_intrs) {
 376                 virq = irq_linear_revmap(gic_irq_domain,
 377                                          GIC_SHARED_TO_HWIRQ(intr));
 378                 if (chained)
 379                         generic_handle_irq(virq);
 380                 else
 381                         do_IRQ(virq);
 382
 383                 /* go to next pending bit */
 384                 bitmap_clear(pending, intr, 1);
 385                 intr = find_first_bit(pending, gic_shared_intrs);
 386         }
 387 }
 388
 389 static void gic_mask_irq(struct irq_data *d)
 390 {
 391         gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 392 }
 393
 394 static void gic_unmask_irq(struct irq_data *d)
 395 {
 396         gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 397 }
 398
 399 static void gic_ack_irq(struct irq_data *d)
 400 {
 401         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 402
 403         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
 404 }
 405
 406 static int gic_set_type(struct irq_data *d, unsigned int type)
 407 {
 408         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 409         unsigned long flags;
 410         bool is_edge;
 411
 412         spin_lock_irqsave(&gic_lock, flags);
 413         switch (type & IRQ_TYPE_SENSE_MASK) {
 414         case IRQ_TYPE_EDGE_FALLING:
 415                 gic_set_polarity(irq, GIC_POL_NEG);
 416                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 417                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 418                 is_edge = true;
 419                 break;
 420         case IRQ_TYPE_EDGE_RISING:
 421                 gic_set_polarity(irq, GIC_POL_POS);
 422                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 423                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 424                 is_edge = true;
 425                 break;
 426         case IRQ_TYPE_EDGE_BOTH:
 427                 /* polarity is irrelevant in this case */
 428                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 429                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
 430                 is_edge = true;
 431                 break;
 432         case IRQ_TYPE_LEVEL_LOW:
 433                 gic_set_polarity(irq, GIC_POL_NEG);
 434                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 435                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 436                 is_edge = false;
 437                 break;
 438         case IRQ_TYPE_LEVEL_HIGH:
 439         default:
 440                 gic_set_polarity(irq, GIC_POL_POS);
 441                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 442                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 443                 is_edge = false;
 444                 break;
 445         }
 446
 447         if (is_edge)
 448                 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
 449                                                  handle_edge_irq, NULL);
 450         else
 451                 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
 452                                                  handle_level_irq, NULL);
 453         spin_unlock_irqrestore(&gic_lock, flags);
 454
 455         return 0;
 456 }
 457
 458 #ifdef CONFIG_SMP
 459 static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 460                             bool force)
 461 {
 462         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 463         cpumask_t       tmp = CPU_MASK_NONE;
 464         unsigned long   flags;
 465         int             i;
 466
 467         cpumask_and(&tmp, cpumask, cpu_online_mask);
 468         if (cpumask_empty(&tmp))
 469                 return -EINVAL;
 470
 471         /* Assumption : cpumask refers to a single CPU */
 472         spin_lock_irqsave(&gic_lock, flags);
 473
 474         /* Re-route this IRQ */
 475         gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
 476
 477         /* Update the pcpu_masks */
 478         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 479                 clear_bit(irq, pcpu_masks[i].pcpu_mask);
 480         set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
 481
 482         cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
 483         spin_unlock_irqrestore(&gic_lock, flags);
 484
 485         return IRQ_SET_MASK_OK_NOCOPY;
 486 }
 487 #endif
 488
 489 static struct irq_chip gic_level_irq_controller = {
 490         .name                   =       "MIPS GIC",
 491         .irq_mask               =       gic_mask_irq,
 492         .irq_unmask             =       gic_unmask_irq,
 493         .irq_set_type           =       gic_set_type,
 494 #ifdef CONFIG_SMP
 495         .irq_set_affinity       =       gic_set_affinity,
 496 #endif
 497 };
 498
 499 static struct irq_chip gic_edge_irq_controller = {
 500         .name                   =       "MIPS GIC",
 501         .irq_ack                =       gic_ack_irq,
 502         .irq_mask               =       gic_mask_irq,
 503         .irq_unmask             =       gic_unmask_irq,
 504         .irq_set_type           =       gic_set_type,
 505 #ifdef CONFIG_SMP
 506         .irq_set_affinity       =       gic_set_affinity,
 507 #endif
 508         .ipi_send_single        =       gic_send_ipi,
 509 };
 510
 511 static void gic_handle_local_int(bool chained)
 512 {
 513         unsigned long pending, masked;
 514         unsigned int intr, virq;
 515
 516         pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
 517         masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
 518
 519         bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 520
 521         intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 522         while (intr != GIC_NUM_LOCAL_INTRS) {
 523                 virq = irq_linear_revmap(gic_irq_domain,
 524                                          GIC_LOCAL_TO_HWIRQ(intr));
 525                 if (chained)
 526                         generic_handle_irq(virq);
 527                 else
 528                         do_IRQ(virq);
 529
 530                 /* go to next pending bit */
 531                 bitmap_clear(&pending, intr, 1);
 532                 intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 533         }
 534 }
 535
 536 static void gic_mask_local_irq(struct irq_data *d)
 537 {
 538         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 539
 540         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
 541 }
 542
 543 static void gic_unmask_local_irq(struct irq_data *d)
 544 {
 545         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 546
 547         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
 548 }
 549
 550 static struct irq_chip gic_local_irq_controller = {
 551         .name                   =       "MIPS GIC Local",
 552         .irq_mask               =       gic_mask_local_irq,
 553         .irq_unmask             =       gic_unmask_local_irq,
 554 };
 555
 556 static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 557 {
 558         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 559         int i;
 560         unsigned long flags;
 561
 562         spin_lock_irqsave(&gic_lock, flags);
 563         for (i = 0; i < gic_vpes; i++) {
 564                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 565                           mips_cm_vp_id(i));
 566                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
 567         }
 568         spin_unlock_irqrestore(&gic_lock, flags);
 569 }
 570
 571 static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 572 {
 573         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 574         int i;
 575         unsigned long flags;
 576
 577         spin_lock_irqsave(&gic_lock, flags);
 578         for (i = 0; i < gic_vpes; i++) {
 579                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 580                           mips_cm_vp_id(i));
 581                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
 582         }
 583         spin_unlock_irqrestore(&gic_lock, flags);
 584 }
 585
 586 static struct irq_chip gic_all_vpes_local_irq_controller = {
 587         .name                   =       "MIPS GIC Local",
 588         .irq_mask               =       gic_mask_local_irq_all_vpes,
 589         .irq_unmask             =       gic_unmask_local_irq_all_vpes,
 590 };
 591
 592 static void __gic_irq_dispatch(void)
 593 {
 594         gic_handle_local_int(false);
 595         gic_handle_shared_int(false);
 596 }
 597
 598 static void gic_irq_dispatch(struct irq_desc *desc)
 599 {
 600         gic_handle_local_int(true);
 601         gic_handle_shared_int(true);
 602 }
 603
 604 static void __init gic_basic_init(void)
 605 {
 606         unsigned int i;
 607
 608         board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 609
 610         /* Setup defaults */
 611         for (i = 0; i < gic_shared_intrs; i++) {
 612                 gic_set_polarity(i, GIC_POL_POS);
 613                 gic_set_trigger(i, GIC_TRIG_LEVEL);
 614                 gic_reset_mask(i);
 615         }
 616
 617         for (i = 0; i < gic_vpes; i++) {
 618                 unsigned int j;
 619
 620                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 621                           mips_cm_vp_id(i));
 622                 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
 623                         if (!gic_local_irq_is_routable(j))
 624                                 continue;
 625                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
 626                 }
 627         }
 628 }
 629
 630 static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 631                                     irq_hw_number_t hw)
 632 {
 633         int intr = GIC_HWIRQ_TO_LOCAL(hw);
 634         int ret = 0;
 635         int i;
 636         unsigned long flags;
 637
 638         if (!gic_local_irq_is_routable(intr))
 639                 return -EPERM;
 640
 641         spin_lock_irqsave(&gic_lock, flags);
 642         for (i = 0; i < gic_vpes; i++) {
 643                 u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
 644
 645                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 646                           mips_cm_vp_id(i));
 647
 648                 switch (intr) {
 649                 case GIC_LOCAL_INT_WD:
 650                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
 651                         break;
 652                 case GIC_LOCAL_INT_COMPARE:
 653                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
 654                                     val);
 655                         break;
 656                 case GIC_LOCAL_INT_TIMER:
 657                         /* CONFIG_MIPS_CMP workaround (see __gic_init) */
 658                         val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
 659                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
 660                                     val);
 661                         break;
 662                 case GIC_LOCAL_INT_PERFCTR:
 663                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
 664                                     val);
 665                         break;
 666                 case GIC_LOCAL_INT_SWINT0:
 667                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
 668                                     val);
 669                         break;
 670                 case GIC_LOCAL_INT_SWINT1:
 671                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
 672                                     val);
 673                         break;
 674                 case GIC_LOCAL_INT_FDC:
 675                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
 676                         break;
 677                 default:
 678                         pr_err("Invalid local IRQ %d\n", intr);
 679                         ret = -EINVAL;
 680                         break;
 681                 }
 682         }
 683         spin_unlock_irqrestore(&gic_lock, flags);
 684
 685         return ret;
 686 }
 687
 688 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 689                                      irq_hw_number_t hw, unsigned int vpe)
 690 {
 691         int intr = GIC_HWIRQ_TO_SHARED(hw);
 692         unsigned long flags;
 693         int i;
 694
 695         spin_lock_irqsave(&gic_lock, flags);
 696         gic_map_to_pin(intr, gic_cpu_pin);
 697         gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
 698         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 699                 clear_bit(intr, pcpu_masks[i].pcpu_mask);
 700         set_bit(intr, pcpu_masks[vpe].pcpu_mask);
 701         spin_unlock_irqrestore(&gic_lock, flags);
 702
 703         return 0;
 704 }
 705
 706 static int gic_setup_dev_chip(struct irq_domain *d, unsigned int virq,
 707                               unsigned int hwirq)
 708 {
 709         struct irq_chip *chip;
 710         int err;
 711
 712         if (hwirq >= GIC_SHARED_HWIRQ_BASE) {
 713                 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
 714                                                     &gic_level_irq_controller,
 715                                                     NULL);
 716         } else {
 717                 switch (GIC_HWIRQ_TO_LOCAL(hwirq)) {
 718                 case GIC_LOCAL_INT_TIMER:
 719                 case GIC_LOCAL_INT_PERFCTR:
 720                 case GIC_LOCAL_INT_FDC:
 721                         /*
 722                          * HACK: These are all really percpu interrupts, but
 723                          * the rest of the MIPS kernel code does not use the
 724                          * percpu IRQ API for them.
 725                          */
 726                         chip = &gic_all_vpes_local_irq_controller;
 727                         irq_set_handler(virq, handle_percpu_irq);
 728                         break;
 729
 730                 default:
 731                         chip = &gic_local_irq_controller;
 732                         irq_set_handler(virq, handle_percpu_devid_irq);
 733                         irq_set_percpu_devid(virq);
 734                         break;
 735                 }
 736
 737                 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
 738                                                     chip, NULL);
 739         }
 740
 741         return err;
 742 }
 743
 744 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 745                                 unsigned int nr_irqs, void *arg)
 746 {
 747         struct gic_irq_spec *spec = arg;
 748         irq_hw_number_t hwirq, base_hwirq;
 749         int cpu, ret, i;
 750
 751         if (spec->type == GIC_DEVICE) {
 752                 /* verify that shared irqs don't conflict with an IPI irq */
 753                 if ((spec->hwirq >= GIC_SHARED_HWIRQ_BASE) &&
 754                     test_bit(GIC_HWIRQ_TO_SHARED(spec->hwirq), ipi_resrv))
 755                         return -EBUSY;
 756
 757                 return gic_setup_dev_chip(d, virq, spec->hwirq);
 758         } else {
 759                 base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs);
 760                 if (base_hwirq == gic_shared_intrs) {
 761                         return -ENOMEM;
 762                 }
 763
 764                 /* check that we have enough space */
 765                 for (i = base_hwirq; i < nr_irqs; i++) {
 766                         if (!test_bit(i, ipi_resrv))
 767                                 return -EBUSY;
 768                 }
 769                 bitmap_clear(ipi_resrv, base_hwirq, nr_irqs);
 770
 771                 /* map the hwirq for each cpu consecutively */
 772                 i = 0;
 773                 for_each_cpu(cpu, spec->ipimask) {
 774                         hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
 775
 776                         ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
 777                                                             &gic_level_irq_controller,
 778                                                             NULL);
 779                         if (ret)
 780                                 goto error;
 781
 782                         irq_set_handler(virq + i, handle_level_irq);
 783
 784                         ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
 785                         if (ret)
 786                                 goto error;
 787
 788                         i++;
 789                 }
 790
 791                 /*
 792                  * tell the parent about the base hwirq we allocated so it can
 793                  * set its own domain data
 794                  */
 795                 spec->hwirq = base_hwirq;
 796         }
 797
 798         return 0;
 799 error:
 800         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 801         return ret;
 802 }
 803
 804 void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
 805                          unsigned int nr_irqs)
 806 {
 807         irq_hw_number_t base_hwirq;
 808         struct irq_data *data;
 809
 810         data = irq_get_irq_data(virq);
 811         if (!data)
 812                 return;
 813
 814         base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
 815         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 816 }
 817
 818 int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
 819                          enum irq_domain_bus_token bus_token)
 820 {
 821         /* this domain should'nt be accessed directly */
 822         return 0;
 823 }
 824
 825 static const struct irq_domain_ops gic_irq_domain_ops = {
 826         .alloc = gic_irq_domain_alloc,
 827         .free = gic_irq_domain_free,
 828         .match = gic_irq_domain_match,
 829 };
 830
 831 static int gic_dev_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 832                                 const u32 *intspec, unsigned int intsize,
 833                                 irq_hw_number_t *out_hwirq,
 834                                 unsigned int *out_type)
 835 {
 836         if (intsize != 3)
 837                 return -EINVAL;
 838
 839         if (intspec[0] == GIC_SHARED)
 840                 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
 841         else if (intspec[0] == GIC_LOCAL)
 842                 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
 843         else
 844                 return -EINVAL;
 845         *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
 846
 847         return 0;
 848 }
 849
 850 static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
 851                                 unsigned int nr_irqs, void *arg)
 852 {
 853         struct irq_fwspec *fwspec = arg;
 854         struct gic_irq_spec spec = {
 855                 .type = GIC_DEVICE,
 856         };
 857         int i, ret;
 858
 859         if (fwspec->param[0] == GIC_SHARED)
 860                 spec.hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]);
 861         else
 862                 spec.hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]);
 863
 864         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 865         if (ret)
 866                 return ret;
 867
 868         for (i = 0; i < nr_irqs; i++) {
 869                 ret = gic_setup_dev_chip(d, virq + i, spec.hwirq + i);
 870                 if (ret)
 871                         goto error;
 872         }
 873
 874         return 0;
 875
 876 error:
 877         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 878         return ret;
 879 }
 880
 881 void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
 882                          unsigned int nr_irqs)
 883 {
 884         /* no real allocation is done for dev irqs, so no need to free anything */
 885         return;
 886 }
 887
 888 static void gic_dev_domain_activate(struct irq_domain *domain,
 889                                     struct irq_data *d)
 890 {
 891         if (GIC_HWIRQ_TO_LOCAL(d->hwirq) < GIC_NUM_LOCAL_INTRS)
 892                 gic_local_irq_domain_map(domain, d->irq, d->hwirq);
 893         else
 894                 gic_shared_irq_domain_map(domain, d->irq, d->hwirq, 0);
 895 }
 896
 897 static struct irq_domain_ops gic_dev_domain_ops = {
 898         .xlate = gic_dev_domain_xlate,
 899         .alloc = gic_dev_domain_alloc,
 900         .free = gic_dev_domain_free,
 901         .activate = gic_dev_domain_activate,
 902 };
 903
 904 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 905                                 const u32 *intspec, unsigned int intsize,
 906                                 irq_hw_number_t *out_hwirq,
 907                                 unsigned int *out_type)
 908 {
 909         /*
 910          * There's nothing to translate here. hwirq is dynamically allocated and
 911          * the irq type is always edge triggered.
 912          * */
 913         *out_hwirq = 0;
 914         *out_type = IRQ_TYPE_EDGE_RISING;
 915
 916         return 0;
 917 }
 918
 919 static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
 920                                 unsigned int nr_irqs, void *arg)
 921 {
 922         struct cpumask *ipimask = arg;
 923         struct gic_irq_spec spec = {
 924                 .type = GIC_IPI,
 925                 .ipimask = ipimask
 926         };
 927         int ret, i;
 928
 929         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 930         if (ret)
 931                 return ret;
 932
 933         /* the parent should have set spec.hwirq to the base_hwirq it allocated */
 934         for (i = 0; i < nr_irqs; i++) {
 935                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 936                                                     GIC_SHARED_TO_HWIRQ(spec.hwirq + i),
 937                                                     &gic_edge_irq_controller,
 938                                                     NULL);
 939                 if (ret)
 940                         goto error;
 941
 942                 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
 943                 if (ret)
 944                         goto error;
 945         }
 946
 947         return 0;
 948 error:
 949         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 950         return ret;
 951 }
 952
 953 void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
 954                          unsigned int nr_irqs)
 955 {
 956         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 957 }
 958
 959 int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
 960                          enum irq_domain_bus_token bus_token)
 961 {
 962         bool is_ipi;
 963
 964         switch (bus_token) {
 965         case DOMAIN_BUS_IPI:
 966                 is_ipi = d->bus_token == bus_token;
 967                 return (!node || to_of_node(d->fwnode) == node) && is_ipi;
 968                 break;
 969         default:
 970                 return 0;
 971         }
 972 }
 973
 974 static struct irq_domain_ops gic_ipi_domain_ops = {
 975         .xlate = gic_ipi_domain_xlate,
 976         .alloc = gic_ipi_domain_alloc,
 977         .free = gic_ipi_domain_free,
 978         .match = gic_ipi_domain_match,
 979 };
 980
 981 static void __init __gic_init(unsigned long gic_base_addr,
 982                               unsigned long gic_addrspace_size,
 983                               unsigned int cpu_vec, unsigned int irqbase,
 984                               struct device_node *node)
 985 {
 986         unsigned int gicconfig, cpu;
 987         unsigned int v[2];
 988
 989         __gic_base_addr = gic_base_addr;
 990
 991         gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
 992
 993         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 994         gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
 995                    GIC_SH_CONFIG_NUMINTRS_SHF;
 996         gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
 997
 998         gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
 999                   GIC_SH_CONFIG_NUMVPES_SHF;
1000         gic_vpes = gic_vpes + 1;
1001
1002         if (cpu_has_veic) {
1003                 /* Set EIC mode for all VPEs */
1004                 for_each_present_cpu(cpu) {
1005                         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
1006                                   mips_cm_vp_id(cpu));
1007                         gic_write(GIC_REG(VPE_OTHER, GIC_VPE_CTL),
1008                                   GIC_VPE_CTL_EIC_MODE_MSK);
1009                 }
1010
1011                 /* Always use vector 1 in EIC mode */
1012                 gic_cpu_pin = 0;
1013                 timer_cpu_pin = gic_cpu_pin;
1014                 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
1015                                __gic_irq_dispatch);
1016         } else {
1017                 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
1018                 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
1019                                         gic_irq_dispatch);
1020                 /*
1021                  * With the CMP implementation of SMP (deprecated), other CPUs
1022                  * are started by the bootloader and put into a timer based
1023                  * waiting poll loop. We must not re-route those CPU's local
1024                  * timer interrupts as the wait instruction will never finish,
1025                  * so just handle whatever CPU interrupt it is routed to by
1026                  * default.
1027                  *
1028                  * This workaround should be removed when CMP support is
1029                  * dropped.
1030                  */
1031                 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
1032                     gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
1033                         timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
1034                                                          GIC_VPE_TIMER_MAP)) &
1035                                         GIC_MAP_MSK;
1036                         irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
1037                                                 GIC_CPU_PIN_OFFSET +
1038                                                 timer_cpu_pin,
1039                                                 gic_irq_dispatch);
1040                 } else {
1041                         timer_cpu_pin = gic_cpu_pin;
1042                 }
1043         }
1044
1045         gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
1046                                                gic_shared_intrs, irqbase,
1047                                                &gic_irq_domain_ops, NULL);
1048         if (!gic_irq_domain)
1049                 panic("Failed to add GIC IRQ domain");
1050         gic_irq_domain->name = "mips-gic-irq";
1051
1052         gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0,
1053                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1054                                                   node, &gic_dev_domain_ops, NULL);
1055         if (!gic_dev_domain)
1056                 panic("Failed to add GIC DEV domain");
1057         gic_dev_domain->name = "mips-gic-dev";
1058
1059         gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
1060                                                   IRQ_DOMAIN_FLAG_IPI_PER_CPU,
1061                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1062                                                   node, &gic_ipi_domain_ops, NULL);
1063         if (!gic_ipi_domain)
1064                 panic("Failed to add GIC IPI domain");
1065
1066         gic_ipi_domain->name = "mips-gic-ipi";
1067         gic_ipi_domain->bus_token = DOMAIN_BUS_IPI;
1068
1069         if (node &&
1070             !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
1071                 bitmap_set(ipi_resrv, v[0], v[1]);
1072         } else {
1073                 /* Make the last 2 * gic_vpes available for IPIs */
1074                 bitmap_set(ipi_resrv,
1075                            gic_shared_intrs - 2 * gic_vpes,
1076                            2 * gic_vpes);
1077         }
1078
1079         gic_basic_init();
1080 }
1081
1082 void __init gic_init(unsigned long gic_base_addr,
1083                      unsigned long gic_addrspace_size,
1084                      unsigned int cpu_vec, unsigned int irqbase)
1085 {
1086         __gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
1087 }
1088
1089 static int __init gic_of_init(struct device_node *node,
1090                               struct device_node *parent)
1091 {
1092         struct resource res;
1093         unsigned int cpu_vec, i = 0, reserved = 0;
1094         phys_addr_t gic_base;
1095         size_t gic_len;
1096
1097         /* Find the first available CPU vector. */
1098         while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
1099                                            i++, &cpu_vec))
1100                 reserved |= BIT(cpu_vec);
1101         for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
1102                 if (!(reserved & BIT(cpu_vec)))
1103                         break;
1104         }
1105         if (cpu_vec == 8) {
1106                 pr_err("No CPU vectors available for GIC\n");
1107                 return -ENODEV;
1108         }
1109
1110         if (of_address_to_resource(node, 0, &res)) {
1111                 /*
1112                  * Probe the CM for the GIC base address if not specified
1113                  * in the device-tree.
1114                  */
1115                 if (mips_cm_present()) {
1116                         gic_base = read_gcr_gic_base() &
1117                                 ~CM_GCR_GIC_BASE_GICEN_MSK;
1118                         gic_len = 0x20000;
1119                 } else {
1120                         pr_err("Failed to get GIC memory range\n");
1121                         return -ENODEV;
1122                 }
1123         } else {
1124                 gic_base = res.start;
1125                 gic_len = resource_size(&res);
1126         }
1127
1128         if (mips_cm_present())
1129                 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
1130         gic_present = true;
1131
1132         __gic_init(gic_base, gic_len, cpu_vec, 0, node);
1133
1134         return 0;
1135 }
1136 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);