drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
  13
  14 #include <linux/init.h>
  15 #include <linux/kernel.h>
  16 #include <linux/device.h>
  17 #include <linux/smp.h>
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/clockchips.h>
  21 #include <linux/clocksource.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_address.h>
  25 #include <linux/io.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29
  30 #include <asm/arch_timer.h>
  31 #include <asm/virt.h>
  32
  33 #include <clocksource/arm_arch_timer.h>
  34
  35 #define CNTTIDR         0x08
  36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  37
  38 #define CNTACR(n)       (0x40 + ((n) * 4))
  39 #define CNTACR_RPCT     BIT(0)
  40 #define CNTACR_RVCT     BIT(1)
  41 #define CNTACR_RFRQ     BIT(2)
  42 #define CNTACR_RVOFF    BIT(3)
  43 #define CNTACR_RWVT     BIT(4)
  44 #define CNTACR_RWPT     BIT(5)
  45
  46 #define CNTVCT_LO       0x08
  47 #define CNTVCT_HI       0x0c
  48 #define CNTFRQ          0x10
  49 #define CNTP_TVAL       0x28
  50 #define CNTP_CTL        0x2c
  51 #define CNTV_TVAL       0x38
  52 #define CNTV_CTL        0x3c
  53
  54 #define ARCH_CP15_TIMER BIT(0)
  55 #define ARCH_MEM_TIMER  BIT(1)
  56 static unsigned arch_timers_present __initdata;
  57
  58 static void __iomem *arch_counter_base;
  59
  60 struct arch_timer {
  61         void __iomem *base;
  62         struct clock_event_device evt;
  63 };
  64
  65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  66
  67 static u32 arch_timer_rate;
  68
  69 enum ppi_nr {
  70         PHYS_SECURE_PPI,
  71         PHYS_NONSECURE_PPI,
  72         VIRT_PPI,
  73         HYP_PPI,
  74         MAX_TIMER_PPI
  75 };
  76
  77 static int arch_timer_ppi[MAX_TIMER_PPI];
  78
  79 static struct clock_event_device __percpu *arch_timer_evt;
  80
  81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  82 static bool arch_timer_c3stop;
  83 static bool arch_timer_mem_use_virtual;
  84
  85 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  86
  87 static int __init early_evtstrm_cfg(char *buf)
  88 {
  89         return strtobool(buf, &evtstrm_enable);
  90 }
  91 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  92
  93 /*
  94  * Architected system timer support.
  95  */
  96
  97 static __always_inline
  98 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  99                           struct clock_event_device *clk)
 100 {
 101         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 102                 struct arch_timer *timer = to_arch_timer(clk);
 103                 switch (reg) {
 104                 case ARCH_TIMER_REG_CTRL:
 105                         writel_relaxed(val, timer->base + CNTP_CTL);
 106                         break;
 107                 case ARCH_TIMER_REG_TVAL:
 108                         writel_relaxed(val, timer->base + CNTP_TVAL);
 109                         break;
 110                 }
 111         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 112                 struct arch_timer *timer = to_arch_timer(clk);
 113                 switch (reg) {
 114                 case ARCH_TIMER_REG_CTRL:
 115                         writel_relaxed(val, timer->base + CNTV_CTL);
 116                         break;
 117                 case ARCH_TIMER_REG_TVAL:
 118                         writel_relaxed(val, timer->base + CNTV_TVAL);
 119                         break;
 120                 }
 121         } else {
 122                 arch_timer_reg_write_cp15(access, reg, val);
 123         }
 124 }
 125
 126 static __always_inline
 127 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 128                         struct clock_event_device *clk)
 129 {
 130         u32 val;
 131
 132         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 133                 struct arch_timer *timer = to_arch_timer(clk);
 134                 switch (reg) {
 135                 case ARCH_TIMER_REG_CTRL:
 136                         val = readl_relaxed(timer->base + CNTP_CTL);
 137                         break;
 138                 case ARCH_TIMER_REG_TVAL:
 139                         val = readl_relaxed(timer->base + CNTP_TVAL);
 140                         break;
 141                 }
 142         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 143                 struct arch_timer *timer = to_arch_timer(clk);
 144                 switch (reg) {
 145                 case ARCH_TIMER_REG_CTRL:
 146                         val = readl_relaxed(timer->base + CNTV_CTL);
 147                         break;
 148                 case ARCH_TIMER_REG_TVAL:
 149                         val = readl_relaxed(timer->base + CNTV_TVAL);
 150                         break;
 151                 }
 152         } else {
 153                 val = arch_timer_reg_read_cp15(access, reg);
 154         }
 155
 156         return val;
 157 }
 158
 159 static __always_inline irqreturn_t timer_handler(const int access,
 160                                         struct clock_event_device *evt)
 161 {
 162         unsigned long ctrl;
 163
 164         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 165         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 166                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 167                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 168                 evt->event_handler(evt);
 169                 return IRQ_HANDLED;
 170         }
 171
 172         return IRQ_NONE;
 173 }
 174
 175 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 176 {
 177         struct clock_event_device *evt = dev_id;
 178
 179         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 180 }
 181
 182 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 183 {
 184         struct clock_event_device *evt = dev_id;
 185
 186         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 187 }
 188
 189 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 190 {
 191         struct clock_event_device *evt = dev_id;
 192
 193         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 194 }
 195
 196 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 197 {
 198         struct clock_event_device *evt = dev_id;
 199
 200         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 201 }
 202
 203 static __always_inline int timer_shutdown(const int access,
 204                                           struct clock_event_device *clk)
 205 {
 206         unsigned long ctrl;
 207
 208         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 209         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 210         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 211
 212         return 0;
 213 }
 214
 215 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 216 {
 217         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 218 }
 219
 220 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 221 {
 222         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 223 }
 224
 225 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 226 {
 227         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 228 }
 229
 230 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 231 {
 232         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 233 }
 234
 235 static __always_inline void set_next_event(const int access, unsigned long evt,
 236                                            struct clock_event_device *clk)
 237 {
 238         unsigned long ctrl;
 239         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 240         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 241         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 242         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 243         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 244 }
 245
 246 static int arch_timer_set_next_event_virt(unsigned long evt,
 247                                           struct clock_event_device *clk)
 248 {
 249         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 250         return 0;
 251 }
 252
 253 static int arch_timer_set_next_event_phys(unsigned long evt,
 254                                           struct clock_event_device *clk)
 255 {
 256         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 257         return 0;
 258 }
 259
 260 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 261                                               struct clock_event_device *clk)
 262 {
 263         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 264         return 0;
 265 }
 266
 267 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 268                                               struct clock_event_device *clk)
 269 {
 270         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 271         return 0;
 272 }
 273
 274 static void __arch_timer_setup(unsigned type,
 275                                struct clock_event_device *clk)
 276 {
 277         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 278
 279         if (type == ARCH_CP15_TIMER) {
 280                 if (arch_timer_c3stop)
 281                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 282                 clk->name = "arch_sys_timer";
 283                 clk->rating = 450;
 284                 clk->cpumask = cpumask_of(smp_processor_id());
 285                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 286                 switch (arch_timer_uses_ppi) {
 287                 case VIRT_PPI:
 288                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 289                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 290                         clk->set_next_event = arch_timer_set_next_event_virt;
 291                         break;
 292                 case PHYS_SECURE_PPI:
 293                 case PHYS_NONSECURE_PPI:
 294                 case HYP_PPI:
 295                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 296                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 297                         clk->set_next_event = arch_timer_set_next_event_phys;
 298                         break;
 299                 default:
 300                         BUG();
 301                 }
 302         } else {
 303                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 304                 clk->name = "arch_mem_timer";
 305                 clk->rating = 400;
 306                 clk->cpumask = cpu_all_mask;
 307                 if (arch_timer_mem_use_virtual) {
 308                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 309                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 310                         clk->set_next_event =
 311                                 arch_timer_set_next_event_virt_mem;
 312                 } else {
 313                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 314                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 315                         clk->set_next_event =
 316                                 arch_timer_set_next_event_phys_mem;
 317                 }
 318         }
 319
 320         clk->set_state_shutdown(clk);
 321
 322         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 323 }
 324
 325 static void arch_timer_evtstrm_enable(int divider)
 326 {
 327         u32 cntkctl = arch_timer_get_cntkctl();
 328
 329         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 330         /* Set the divider and enable virtual event stream */
 331         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 332                         | ARCH_TIMER_VIRT_EVT_EN;
 333         arch_timer_set_cntkctl(cntkctl);
 334         elf_hwcap |= HWCAP_EVTSTRM;
 335 #ifdef CONFIG_COMPAT
 336         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 337 #endif
 338 }
 339
 340 static void arch_timer_configure_evtstream(void)
 341 {
 342         int evt_stream_div, pos;
 343
 344         /* Find the closest power of two to the divisor */
 345         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 346         pos = fls(evt_stream_div);
 347         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 348                 pos--;
 349         /* enable event stream */
 350         arch_timer_evtstrm_enable(min(pos, 15));
 351 }
 352
 353 static void arch_counter_set_user_access(void)
 354 {
 355         u32 cntkctl = arch_timer_get_cntkctl();
 356
 357         /* Disable user access to the timers and the physical counter */
 358         /* Also disable virtual event stream */
 359         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 360                         | ARCH_TIMER_USR_VT_ACCESS_EN
 361                         | ARCH_TIMER_VIRT_EVT_EN
 362                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 363
 364         /* Enable user access to the virtual counter */
 365         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 366
 367         arch_timer_set_cntkctl(cntkctl);
 368 }
 369
 370 static bool arch_timer_has_nonsecure_ppi(void)
 371 {
 372         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 373                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
 374 }
 375
 376 static u32 check_ppi_trigger(int irq)
 377 {
 378         u32 flags = irq_get_trigger_type(irq);
 379
 380         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 381                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 382                 pr_warn("WARNING: Please fix your firmware\n");
 383                 flags = IRQF_TRIGGER_LOW;
 384         }
 385
 386         return flags;
 387 }
 388
 389 static int arch_timer_starting_cpu(unsigned int cpu)
 390 {
 391         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 392         u32 flags;
 393
 394         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 395
 396         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 397         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 398
 399         if (arch_timer_has_nonsecure_ppi()) {
 400                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 401                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
 402         }
 403
 404         arch_counter_set_user_access();
 405         if (evtstrm_enable)
 406                 arch_timer_configure_evtstream();
 407
 408         return 0;
 409 }
 410
 411 static void
 412 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 413 {
 414         /* Who has more than one independent system counter? */
 415         if (arch_timer_rate)
 416                 return;
 417
 418         /*
 419          * Try to determine the frequency from the device tree or CNTFRQ,
 420          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 421          */
 422         if (!acpi_disabled ||
 423             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 424                 if (cntbase)
 425                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 426                 else
 427                         arch_timer_rate = arch_timer_get_cntfrq();
 428         }
 429
 430         /* Check the timer frequency. */
 431         if (arch_timer_rate == 0)
 432                 pr_warn("Architected timer frequency not available\n");
 433 }
 434
 435 static void arch_timer_banner(unsigned type)
 436 {
 437         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 438                      type & ARCH_CP15_TIMER ? "cp15" : "",
 439                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 440                      type & ARCH_MEM_TIMER ? "mmio" : "",
 441                      (unsigned long)arch_timer_rate / 1000000,
 442                      (unsigned long)(arch_timer_rate / 10000) % 100,
 443                      type & ARCH_CP15_TIMER ?
 444                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 445                         "",
 446                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 447                      type & ARCH_MEM_TIMER ?
 448                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 449                         "");
 450 }
 451
 452 u32 arch_timer_get_rate(void)
 453 {
 454         return arch_timer_rate;
 455 }
 456
 457 static u64 arch_counter_get_cntvct_mem(void)
 458 {
 459         u32 vct_lo, vct_hi, tmp_hi;
 460
 461         do {
 462                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 463                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 464                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 465         } while (vct_hi != tmp_hi);
 466
 467         return ((u64) vct_hi << 32) | vct_lo;
 468 }
 469
 470 /*
 471  * Default to cp15 based access because arm64 uses this function for
 472  * sched_clock() before DT is probed and the cp15 method is guaranteed
 473  * to exist on arm64. arm doesn't use this before DT is probed so even
 474  * if we don't have the cp15 accessors we won't have a problem.
 475  */
 476 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 477
 478 static cycle_t arch_counter_read(struct clocksource *cs)
 479 {
 480         return arch_timer_read_counter();
 481 }
 482
 483 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 484 {
 485         return arch_timer_read_counter();
 486 }
 487
 488 static struct clocksource clocksource_counter = {
 489         .name   = "arch_sys_counter",
 490         .rating = 400,
 491         .read   = arch_counter_read,
 492         .mask   = CLOCKSOURCE_MASK(56),
 493         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 494 };
 495
 496 static struct cyclecounter cyclecounter = {
 497         .read   = arch_counter_read_cc,
 498         .mask   = CLOCKSOURCE_MASK(56),
 499 };
 500
 501 static struct arch_timer_kvm_info arch_timer_kvm_info;
 502
 503 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 504 {
 505         return &arch_timer_kvm_info;
 506 }
 507
 508 static void __init arch_counter_register(unsigned type)
 509 {
 510         u64 start_count;
 511
 512         /* Register the CP15 based counter if we have one */
 513         if (type & ARCH_CP15_TIMER) {
 514                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 515                         arch_timer_read_counter = arch_counter_get_cntvct;
 516                 else
 517                         arch_timer_read_counter = arch_counter_get_cntpct;
 518         } else {
 519                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 520
 521                 /* If the clocksource name is "arch_sys_counter" the
 522                  * VDSO will attempt to read the CP15-based counter.
 523                  * Ensure this does not happen when CP15-based
 524                  * counter is not available.
 525                  */
 526                 clocksource_counter.name = "arch_mem_counter";
 527         }
 528
 529         start_count = arch_timer_read_counter();
 530         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 531         cyclecounter.mult = clocksource_counter.mult;
 532         cyclecounter.shift = clocksource_counter.shift;
 533         timecounter_init(&arch_timer_kvm_info.timecounter,
 534                          &cyclecounter, start_count);
 535
 536         /* 56 bits minimum, so we assume worst case rollover */
 537         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 538 }
 539
 540 static void arch_timer_stop(struct clock_event_device *clk)
 541 {
 542         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 543                  clk->irq, smp_processor_id());
 544
 545         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 546         if (arch_timer_has_nonsecure_ppi())
 547                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 548
 549         clk->set_state_shutdown(clk);
 550 }
 551
 552 static int arch_timer_dying_cpu(unsigned int cpu)
 553 {
 554         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 555
 556         arch_timer_stop(clk);
 557         return 0;
 558 }
 559
 560 #ifdef CONFIG_CPU_PM
 561 static unsigned int saved_cntkctl;
 562 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 563                                     unsigned long action, void *hcpu)
 564 {
 565         if (action == CPU_PM_ENTER)
 566                 saved_cntkctl = arch_timer_get_cntkctl();
 567         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 568                 arch_timer_set_cntkctl(saved_cntkctl);
 569         return NOTIFY_OK;
 570 }
 571
 572 static struct notifier_block arch_timer_cpu_pm_notifier = {
 573         .notifier_call = arch_timer_cpu_pm_notify,
 574 };
 575
 576 static int __init arch_timer_cpu_pm_init(void)
 577 {
 578         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 579 }
 580
 581 static void __init arch_timer_cpu_pm_deinit(void)
 582 {
 583         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
 584 }
 585
 586 #else
 587 static int __init arch_timer_cpu_pm_init(void)
 588 {
 589         return 0;
 590 }
 591
 592 static void __init arch_timer_cpu_pm_deinit(void)
 593 {
 594 }
 595 #endif
 596
 597 static int __init arch_timer_register(void)
 598 {
 599         int err;
 600         int ppi;
 601
 602         arch_timer_evt = alloc_percpu(struct clock_event_device);
 603         if (!arch_timer_evt) {
 604                 err = -ENOMEM;
 605                 goto out;
 606         }
 607
 608         ppi = arch_timer_ppi[arch_timer_uses_ppi];
 609         switch (arch_timer_uses_ppi) {
 610         case VIRT_PPI:
 611                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 612                                          "arch_timer", arch_timer_evt);
 613                 break;
 614         case PHYS_SECURE_PPI:
 615         case PHYS_NONSECURE_PPI:
 616                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 617                                          "arch_timer", arch_timer_evt);
 618                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 619                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 620                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 621                                                  "arch_timer", arch_timer_evt);
 622                         if (err)
 623                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 624                                                 arch_timer_evt);
 625                 }
 626                 break;
 627         case HYP_PPI:
 628                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 629                                          "arch_timer", arch_timer_evt);
 630                 break;
 631         default:
 632                 BUG();
 633         }
 634
 635         if (err) {
 636                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 637                        ppi, err);
 638                 goto out_free;
 639         }
 640
 641         err = arch_timer_cpu_pm_init();
 642         if (err)
 643                 goto out_unreg_notify;
 644
 645
 646         /* Register and immediately configure the timer on the boot CPU */
 647         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
 648                                 "AP_ARM_ARCH_TIMER_STARTING",
 649                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
 650         if (err)
 651                 goto out_unreg_cpupm;
 652         return 0;
 653
 654 out_unreg_cpupm:
 655         arch_timer_cpu_pm_deinit();
 656
 657 out_unreg_notify:
 658         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 659         if (arch_timer_has_nonsecure_ppi())
 660                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 661                                 arch_timer_evt);
 662
 663 out_free:
 664         free_percpu(arch_timer_evt);
 665 out:
 666         return err;
 667 }
 668
 669 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 670 {
 671         int ret;
 672         irq_handler_t func;
 673         struct arch_timer *t;
 674
 675         t = kzalloc(sizeof(*t), GFP_KERNEL);
 676         if (!t)
 677                 return -ENOMEM;
 678
 679         t->base = base;
 680         t->evt.irq = irq;
 681         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 682
 683         if (arch_timer_mem_use_virtual)
 684                 func = arch_timer_handler_virt_mem;
 685         else
 686                 func = arch_timer_handler_phys_mem;
 687
 688         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 689         if (ret) {
 690                 pr_err("arch_timer: Failed to request mem timer irq\n");
 691                 kfree(t);
 692         }
 693
 694         return ret;
 695 }
 696
 697 static const struct of_device_id arch_timer_of_match[] __initconst = {
 698         { .compatible   = "arm,armv7-timer",    },
 699         { .compatible   = "arm,armv8-timer",    },
 700         {},
 701 };
 702
 703 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 704         { .compatible   = "arm,armv7-timer-mem", },
 705         {},
 706 };
 707
 708 static bool __init
 709 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 710 {
 711         struct device_node *dn;
 712         bool needs_probing = false;
 713
 714         dn = of_find_matching_node(NULL, matches);
 715         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 716                 needs_probing = true;
 717         of_node_put(dn);
 718
 719         return needs_probing;
 720 }
 721
 722 static int __init arch_timer_common_init(void)
 723 {
 724         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 725
 726         /* Wait until both nodes are probed if we have two timers */
 727         if ((arch_timers_present & mask) != mask) {
 728                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 729                         return 0;
 730                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 731                         return 0;
 732         }
 733
 734         arch_timer_banner(arch_timers_present);
 735         arch_counter_register(arch_timers_present);
 736         return arch_timer_arch_init();
 737 }
 738
 739 static int __init arch_timer_init(void)
 740 {
 741         int ret;
 742         /*
 743          * If HYP mode is available, we know that the physical timer
 744          * has been configured to be accessible from PL1. Use it, so
 745          * that a guest can use the virtual timer instead.
 746          *
 747          * If no interrupt provided for virtual timer, we'll have to
 748          * stick to the physical timer. It'd better be accessible...
 749          *
 750          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 751          * accesses to CNTP_*_EL1 registers are silently redirected to
 752          * their CNTHP_*_EL2 counterparts, and use a different PPI
 753          * number.
 754          */
 755         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 756                 bool has_ppi;
 757
 758                 if (is_kernel_in_hyp_mode()) {
 759                         arch_timer_uses_ppi = HYP_PPI;
 760                         has_ppi = !!arch_timer_ppi[HYP_PPI];
 761                 } else {
 762                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
 763                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 764                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 765                 }
 766
 767                 if (!has_ppi) {
 768                         pr_warn("arch_timer: No interrupt available, giving up\n");
 769                         return -EINVAL;
 770                 }
 771         }
 772
 773         ret = arch_timer_register();
 774         if (ret)
 775                 return ret;
 776
 777         ret = arch_timer_common_init();
 778         if (ret)
 779                 return ret;
 780
 781         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 782
 783         return 0;
 784 }
 785
 786 static int __init arch_timer_of_init(struct device_node *np)
 787 {
 788         int i;
 789
 790         if (arch_timers_present & ARCH_CP15_TIMER) {
 791                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 792                 return 0;
 793         }
 794
 795         arch_timers_present |= ARCH_CP15_TIMER;
 796         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 797                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 798
 799         arch_timer_detect_rate(NULL, np);
 800
 801         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 802
 803         /*
 804          * If we cannot rely on firmware initializing the timer registers then
 805          * we should use the physical timers instead.
 806          */
 807         if (IS_ENABLED(CONFIG_ARM) &&
 808             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 809                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
 810
 811         return arch_timer_init();
 812 }
 813 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 814 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 815
 816 static int __init arch_timer_mem_init(struct device_node *np)
 817 {
 818         struct device_node *frame, *best_frame = NULL;
 819         void __iomem *cntctlbase, *base;
 820         unsigned int irq, ret = -EINVAL;
 821         u32 cnttidr;
 822
 823         arch_timers_present |= ARCH_MEM_TIMER;
 824         cntctlbase = of_iomap(np, 0);
 825         if (!cntctlbase) {
 826                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 827                 return -ENXIO;
 828         }
 829
 830         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 831
 832         /*
 833          * Try to find a virtual capable frame. Otherwise fall back to a
 834          * physical capable frame.
 835          */
 836         for_each_available_child_of_node(np, frame) {
 837                 int n;
 838                 u32 cntacr;
 839
 840                 if (of_property_read_u32(frame, "frame-number", &n)) {
 841                         pr_err("arch_timer: Missing frame-number\n");
 842                         of_node_put(frame);
 843                         goto out;
 844                 }
 845
 846                 /* Try enabling everything, and see what sticks */
 847                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 848                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 849                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 850                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 851
 852                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
 853                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 854                         of_node_put(best_frame);
 855                         best_frame = frame;
 856                         arch_timer_mem_use_virtual = true;
 857                         break;
 858                 }
 859
 860                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 861                         continue;
 862
 863                 of_node_put(best_frame);
 864                 best_frame = of_node_get(frame);
 865         }
 866
 867         ret= -ENXIO;
 868         base = arch_counter_base = of_iomap(best_frame, 0);
 869         if (!base) {
 870                 pr_err("arch_timer: Can't map frame's registers\n");
 871                 goto out;
 872         }
 873
 874         if (arch_timer_mem_use_virtual)
 875                 irq = irq_of_parse_and_map(best_frame, 1);
 876         else
 877                 irq = irq_of_parse_and_map(best_frame, 0);
 878
 879         ret = -EINVAL;
 880         if (!irq) {
 881                 pr_err("arch_timer: Frame missing %s irq",
 882                        arch_timer_mem_use_virtual ? "virt" : "phys");
 883                 goto out;
 884         }
 885
 886         arch_timer_detect_rate(base, np);
 887         ret = arch_timer_mem_register(base, irq);
 888         if (ret)
 889                 goto out;
 890
 891         return arch_timer_common_init();
 892 out:
 893         iounmap(cntctlbase);
 894         of_node_put(best_frame);
 895         return ret;
 896 }
 897 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 898                        arch_timer_mem_init);
 899
 900 #ifdef CONFIG_ACPI
 901 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 902 {
 903         int trigger, polarity;
 904
 905         if (!interrupt)
 906                 return 0;
 907
 908         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
 909                         : ACPI_LEVEL_SENSITIVE;
 910
 911         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
 912                         : ACPI_ACTIVE_HIGH;
 913
 914         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 915 }
 916
 917 /* Initialize per-processor generic timer */
 918 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
 919 {
 920         struct acpi_table_gtdt *gtdt;
 921
 922         if (arch_timers_present & ARCH_CP15_TIMER) {
 923                 pr_warn("arch_timer: already initialized, skipping\n");
 924                 return -EINVAL;
 925         }
 926
 927         gtdt = container_of(table, struct acpi_table_gtdt, header);
 928
 929         arch_timers_present |= ARCH_CP15_TIMER;
 930
 931         arch_timer_ppi[PHYS_SECURE_PPI] =
 932                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
 933                 gtdt->secure_el1_flags);
 934
 935         arch_timer_ppi[PHYS_NONSECURE_PPI] =
 936                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
 937                 gtdt->non_secure_el1_flags);
 938
 939         arch_timer_ppi[VIRT_PPI] =
 940                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
 941                 gtdt->virtual_timer_flags);
 942
 943         arch_timer_ppi[HYP_PPI] =
 944                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
 945                 gtdt->non_secure_el2_flags);
 946
 947         /* Get the frequency from CNTFRQ */
 948         arch_timer_detect_rate(NULL, NULL);
 949
 950         /* Always-on capability */
 951         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
 952
 953         arch_timer_init();
 954         return 0;
 955 }
 956 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
 957 #endif