drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
  13
  14 #include <linux/init.h>
  15 #include <linux/kernel.h>
  16 #include <linux/device.h>
  17 #include <linux/smp.h>
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/clockchips.h>
  21 #include <linux/clocksource.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_address.h>
  25 #include <linux/io.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29
  30 #include <asm/arch_timer.h>
  31 #include <asm/virt.h>
  32
  33 #include <clocksource/arm_arch_timer.h>
  34
  35 #define CNTTIDR         0x08
  36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  37
  38 #define CNTACR(n)       (0x40 + ((n) * 4))
  39 #define CNTACR_RPCT     BIT(0)
  40 #define CNTACR_RVCT     BIT(1)
  41 #define CNTACR_RFRQ     BIT(2)
  42 #define CNTACR_RVOFF    BIT(3)
  43 #define CNTACR_RWVT     BIT(4)
  44 #define CNTACR_RWPT     BIT(5)
  45
  46 #define CNTVCT_LO       0x08
  47 #define CNTVCT_HI       0x0c
  48 #define CNTFRQ          0x10
  49 #define CNTP_TVAL       0x28
  50 #define CNTP_CTL        0x2c
  51 #define CNTV_TVAL       0x38
  52 #define CNTV_CTL        0x3c
  53
  54 #define ARCH_CP15_TIMER BIT(0)
  55 #define ARCH_MEM_TIMER  BIT(1)
  56 static unsigned arch_timers_present __initdata;
  57
  58 static void __iomem *arch_counter_base;
  59
  60 struct arch_timer {
  61         void __iomem *base;
  62         struct clock_event_device evt;
  63 };
  64
  65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  66
  67 static u32 arch_timer_rate;
  68
  69 enum ppi_nr {
  70         PHYS_SECURE_PPI,
  71         PHYS_NONSECURE_PPI,
  72         VIRT_PPI,
  73         HYP_PPI,
  74         MAX_TIMER_PPI
  75 };
  76
  77 static int arch_timer_ppi[MAX_TIMER_PPI];
  78
  79 static struct clock_event_device __percpu *arch_timer_evt;
  80
  81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  82 static bool arch_timer_c3stop;
  83 static bool arch_timer_mem_use_virtual;
  84
  85 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  86
  87 static int __init early_evtstrm_cfg(char *buf)
  88 {
  89         return strtobool(buf, &evtstrm_enable);
  90 }
  91 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  92
  93 /*
  94  * Architected system timer support.
  95  */
  96
  97 #ifdef CONFIG_FSL_ERRATUM_A008585
  98 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
  99 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
 100
 101 static int fsl_a008585_enable = -1;
 102
 103 static int __init early_fsl_a008585_cfg(char *buf)
 104 {
 105         int ret;
 106         bool val;
 107
 108         ret = strtobool(buf, &val);
 109         if (ret)
 110                 return ret;
 111
 112         fsl_a008585_enable = val;
 113         return 0;
 114 }
 115 early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
 116
 117 u32 __fsl_a008585_read_cntp_tval_el0(void)
 118 {
 119         return __fsl_a008585_read_reg(cntp_tval_el0);
 120 }
 121
 122 u32 __fsl_a008585_read_cntv_tval_el0(void)
 123 {
 124         return __fsl_a008585_read_reg(cntv_tval_el0);
 125 }
 126
 127 u64 __fsl_a008585_read_cntvct_el0(void)
 128 {
 129         return __fsl_a008585_read_reg(cntvct_el0);
 130 }
 131 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
 132 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 133
 134 static __always_inline
 135 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 136                           struct clock_event_device *clk)
 137 {
 138         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 139                 struct arch_timer *timer = to_arch_timer(clk);
 140                 switch (reg) {
 141                 case ARCH_TIMER_REG_CTRL:
 142                         writel_relaxed(val, timer->base + CNTP_CTL);
 143                         break;
 144                 case ARCH_TIMER_REG_TVAL:
 145                         writel_relaxed(val, timer->base + CNTP_TVAL);
 146                         break;
 147                 }
 148         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 149                 struct arch_timer *timer = to_arch_timer(clk);
 150                 switch (reg) {
 151                 case ARCH_TIMER_REG_CTRL:
 152                         writel_relaxed(val, timer->base + CNTV_CTL);
 153                         break;
 154                 case ARCH_TIMER_REG_TVAL:
 155                         writel_relaxed(val, timer->base + CNTV_TVAL);
 156                         break;
 157                 }
 158         } else {
 159                 arch_timer_reg_write_cp15(access, reg, val);
 160         }
 161 }
 162
 163 static __always_inline
 164 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 165                         struct clock_event_device *clk)
 166 {
 167         u32 val;
 168
 169         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 170                 struct arch_timer *timer = to_arch_timer(clk);
 171                 switch (reg) {
 172                 case ARCH_TIMER_REG_CTRL:
 173                         val = readl_relaxed(timer->base + CNTP_CTL);
 174                         break;
 175                 case ARCH_TIMER_REG_TVAL:
 176                         val = readl_relaxed(timer->base + CNTP_TVAL);
 177                         break;
 178                 }
 179         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 180                 struct arch_timer *timer = to_arch_timer(clk);
 181                 switch (reg) {
 182                 case ARCH_TIMER_REG_CTRL:
 183                         val = readl_relaxed(timer->base + CNTV_CTL);
 184                         break;
 185                 case ARCH_TIMER_REG_TVAL:
 186                         val = readl_relaxed(timer->base + CNTV_TVAL);
 187                         break;
 188                 }
 189         } else {
 190                 val = arch_timer_reg_read_cp15(access, reg);
 191         }
 192
 193         return val;
 194 }
 195
 196 static __always_inline irqreturn_t timer_handler(const int access,
 197                                         struct clock_event_device *evt)
 198 {
 199         unsigned long ctrl;
 200
 201         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 202         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 203                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 204                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 205                 evt->event_handler(evt);
 206                 return IRQ_HANDLED;
 207         }
 208
 209         return IRQ_NONE;
 210 }
 211
 212 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 213 {
 214         struct clock_event_device *evt = dev_id;
 215
 216         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 217 }
 218
 219 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 220 {
 221         struct clock_event_device *evt = dev_id;
 222
 223         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 224 }
 225
 226 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 227 {
 228         struct clock_event_device *evt = dev_id;
 229
 230         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 231 }
 232
 233 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 234 {
 235         struct clock_event_device *evt = dev_id;
 236
 237         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 238 }
 239
 240 static __always_inline int timer_shutdown(const int access,
 241                                           struct clock_event_device *clk)
 242 {
 243         unsigned long ctrl;
 244
 245         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 246         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 247         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 248
 249         return 0;
 250 }
 251
 252 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 253 {
 254         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 255 }
 256
 257 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 258 {
 259         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 260 }
 261
 262 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 263 {
 264         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 265 }
 266
 267 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 268 {
 269         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 270 }
 271
 272 static __always_inline void set_next_event(const int access, unsigned long evt,
 273                                            struct clock_event_device *clk)
 274 {
 275         unsigned long ctrl;
 276         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 277         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 278         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 279         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 280         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 281 }
 282
 283 #ifdef CONFIG_FSL_ERRATUM_A008585
 284 static __always_inline void fsl_a008585_set_next_event(const int access,
 285                 unsigned long evt, struct clock_event_device *clk)
 286 {
 287         unsigned long ctrl;
 288         u64 cval = evt + arch_counter_get_cntvct();
 289
 290         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 291         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 292         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 293
 294         if (access == ARCH_TIMER_PHYS_ACCESS)
 295                 write_sysreg(cval, cntp_cval_el0);
 296         else if (access == ARCH_TIMER_VIRT_ACCESS)
 297                 write_sysreg(cval, cntv_cval_el0);
 298
 299         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 300 }
 301
 302 static int fsl_a008585_set_next_event_virt(unsigned long evt,
 303                                            struct clock_event_device *clk)
 304 {
 305         fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 306         return 0;
 307 }
 308
 309 static int fsl_a008585_set_next_event_phys(unsigned long evt,
 310                                            struct clock_event_device *clk)
 311 {
 312         fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 313         return 0;
 314 }
 315 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 316
 317 static int arch_timer_set_next_event_virt(unsigned long evt,
 318                                           struct clock_event_device *clk)
 319 {
 320         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 321         return 0;
 322 }
 323
 324 static int arch_timer_set_next_event_phys(unsigned long evt,
 325                                           struct clock_event_device *clk)
 326 {
 327         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 328         return 0;
 329 }
 330
 331 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 332                                               struct clock_event_device *clk)
 333 {
 334         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 335         return 0;
 336 }
 337
 338 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 339                                               struct clock_event_device *clk)
 340 {
 341         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 342         return 0;
 343 }
 344
 345 static void fsl_a008585_set_sne(struct clock_event_device *clk)
 346 {
 347 #ifdef CONFIG_FSL_ERRATUM_A008585
 348         if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
 349                 return;
 350
 351         if (arch_timer_uses_ppi == VIRT_PPI)
 352                 clk->set_next_event = fsl_a008585_set_next_event_virt;
 353         else
 354                 clk->set_next_event = fsl_a008585_set_next_event_phys;
 355 #endif
 356 }
 357
 358 static void __arch_timer_setup(unsigned type,
 359                                struct clock_event_device *clk)
 360 {
 361         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 362
 363         if (type == ARCH_CP15_TIMER) {
 364                 if (arch_timer_c3stop)
 365                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 366                 clk->name = "arch_sys_timer";
 367                 clk->rating = 450;
 368                 clk->cpumask = cpumask_of(smp_processor_id());
 369                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 370                 switch (arch_timer_uses_ppi) {
 371                 case VIRT_PPI:
 372                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 373                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 374                         clk->set_next_event = arch_timer_set_next_event_virt;
 375                         break;
 376                 case PHYS_SECURE_PPI:
 377                 case PHYS_NONSECURE_PPI:
 378                 case HYP_PPI:
 379                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 380                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 381                         clk->set_next_event = arch_timer_set_next_event_phys;
 382                         break;
 383                 default:
 384                         BUG();
 385                 }
 386
 387                 fsl_a008585_set_sne(clk);
 388         } else {
 389                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 390                 clk->name = "arch_mem_timer";
 391                 clk->rating = 400;
 392                 clk->cpumask = cpu_all_mask;
 393                 if (arch_timer_mem_use_virtual) {
 394                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 395                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 396                         clk->set_next_event =
 397                                 arch_timer_set_next_event_virt_mem;
 398                 } else {
 399                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 400                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 401                         clk->set_next_event =
 402                                 arch_timer_set_next_event_phys_mem;
 403                 }
 404         }
 405
 406         clk->set_state_shutdown(clk);
 407
 408         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 409 }
 410
 411 static void arch_timer_evtstrm_enable(int divider)
 412 {
 413         u32 cntkctl = arch_timer_get_cntkctl();
 414
 415         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 416         /* Set the divider and enable virtual event stream */
 417         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 418                         | ARCH_TIMER_VIRT_EVT_EN;
 419         arch_timer_set_cntkctl(cntkctl);
 420         elf_hwcap |= HWCAP_EVTSTRM;
 421 #ifdef CONFIG_COMPAT
 422         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 423 #endif
 424 }
 425
 426 static void arch_timer_configure_evtstream(void)
 427 {
 428         int evt_stream_div, pos;
 429
 430         /* Find the closest power of two to the divisor */
 431         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 432         pos = fls(evt_stream_div);
 433         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 434                 pos--;
 435         /* enable event stream */
 436         arch_timer_evtstrm_enable(min(pos, 15));
 437 }
 438
 439 static void arch_counter_set_user_access(void)
 440 {
 441         u32 cntkctl = arch_timer_get_cntkctl();
 442
 443         /* Disable user access to the timers and the physical counter */
 444         /* Also disable virtual event stream */
 445         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 446                         | ARCH_TIMER_USR_VT_ACCESS_EN
 447                         | ARCH_TIMER_VIRT_EVT_EN
 448                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 449
 450         /* Enable user access to the virtual counter */
 451         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 452
 453         arch_timer_set_cntkctl(cntkctl);
 454 }
 455
 456 static bool arch_timer_has_nonsecure_ppi(void)
 457 {
 458         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 459                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
 460 }
 461
 462 static u32 check_ppi_trigger(int irq)
 463 {
 464         u32 flags = irq_get_trigger_type(irq);
 465
 466         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 467                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 468                 pr_warn("WARNING: Please fix your firmware\n");
 469                 flags = IRQF_TRIGGER_LOW;
 470         }
 471
 472         return flags;
 473 }
 474
 475 static int arch_timer_starting_cpu(unsigned int cpu)
 476 {
 477         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 478         u32 flags;
 479
 480         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 481
 482         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 483         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 484
 485         if (arch_timer_has_nonsecure_ppi()) {
 486                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 487                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
 488         }
 489
 490         arch_counter_set_user_access();
 491         if (evtstrm_enable)
 492                 arch_timer_configure_evtstream();
 493
 494         return 0;
 495 }
 496
 497 static void
 498 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 499 {
 500         /* Who has more than one independent system counter? */
 501         if (arch_timer_rate)
 502                 return;
 503
 504         /*
 505          * Try to determine the frequency from the device tree or CNTFRQ,
 506          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 507          */
 508         if (!acpi_disabled ||
 509             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 510                 if (cntbase)
 511                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 512                 else
 513                         arch_timer_rate = arch_timer_get_cntfrq();
 514         }
 515
 516         /* Check the timer frequency. */
 517         if (arch_timer_rate == 0)
 518                 pr_warn("Architected timer frequency not available\n");
 519 }
 520
 521 static void arch_timer_banner(unsigned type)
 522 {
 523         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 524                      type & ARCH_CP15_TIMER ? "cp15" : "",
 525                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 526                      type & ARCH_MEM_TIMER ? "mmio" : "",
 527                      (unsigned long)arch_timer_rate / 1000000,
 528                      (unsigned long)(arch_timer_rate / 10000) % 100,
 529                      type & ARCH_CP15_TIMER ?
 530                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 531                         "",
 532                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 533                      type & ARCH_MEM_TIMER ?
 534                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 535                         "");
 536 }
 537
 538 u32 arch_timer_get_rate(void)
 539 {
 540         return arch_timer_rate;
 541 }
 542
 543 static u64 arch_counter_get_cntvct_mem(void)
 544 {
 545         u32 vct_lo, vct_hi, tmp_hi;
 546
 547         do {
 548                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 549                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 550                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 551         } while (vct_hi != tmp_hi);
 552
 553         return ((u64) vct_hi << 32) | vct_lo;
 554 }
 555
 556 /*
 557  * Default to cp15 based access because arm64 uses this function for
 558  * sched_clock() before DT is probed and the cp15 method is guaranteed
 559  * to exist on arm64. arm doesn't use this before DT is probed so even
 560  * if we don't have the cp15 accessors we won't have a problem.
 561  */
 562 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 563
 564 static cycle_t arch_counter_read(struct clocksource *cs)
 565 {
 566         return arch_timer_read_counter();
 567 }
 568
 569 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 570 {
 571         return arch_timer_read_counter();
 572 }
 573
 574 static struct clocksource clocksource_counter = {
 575         .name   = "arch_sys_counter",
 576         .rating = 400,
 577         .read   = arch_counter_read,
 578         .mask   = CLOCKSOURCE_MASK(56),
 579         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 580 };
 581
 582 static struct cyclecounter cyclecounter = {
 583         .read   = arch_counter_read_cc,
 584         .mask   = CLOCKSOURCE_MASK(56),
 585 };
 586
 587 static struct arch_timer_kvm_info arch_timer_kvm_info;
 588
 589 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 590 {
 591         return &arch_timer_kvm_info;
 592 }
 593
 594 static void __init arch_counter_register(unsigned type)
 595 {
 596         u64 start_count;
 597
 598         /* Register the CP15 based counter if we have one */
 599         if (type & ARCH_CP15_TIMER) {
 600                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 601                         arch_timer_read_counter = arch_counter_get_cntvct;
 602                 else
 603                         arch_timer_read_counter = arch_counter_get_cntpct;
 604
 605                 clocksource_counter.archdata.vdso_direct = true;
 606
 607 #ifdef CONFIG_FSL_ERRATUM_A008585
 608                 /*
 609                  * Don't use the vdso fastpath if errata require using
 610                  * the out-of-line counter accessor.
 611                  */
 612                 if (static_branch_unlikely(&arch_timer_read_ool_enabled))
 613                         clocksource_counter.archdata.vdso_direct = false;
 614 #endif
 615         } else {
 616                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 617         }
 618
 619         start_count = arch_timer_read_counter();
 620         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 621         cyclecounter.mult = clocksource_counter.mult;
 622         cyclecounter.shift = clocksource_counter.shift;
 623         timecounter_init(&arch_timer_kvm_info.timecounter,
 624                          &cyclecounter, start_count);
 625
 626         /* 56 bits minimum, so we assume worst case rollover */
 627         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 628 }
 629
 630 static void arch_timer_stop(struct clock_event_device *clk)
 631 {
 632         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 633                  clk->irq, smp_processor_id());
 634
 635         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 636         if (arch_timer_has_nonsecure_ppi())
 637                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 638
 639         clk->set_state_shutdown(clk);
 640 }
 641
 642 static int arch_timer_dying_cpu(unsigned int cpu)
 643 {
 644         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 645
 646         arch_timer_stop(clk);
 647         return 0;
 648 }
 649
 650 #ifdef CONFIG_CPU_PM
 651 static unsigned int saved_cntkctl;
 652 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 653                                     unsigned long action, void *hcpu)
 654 {
 655         if (action == CPU_PM_ENTER)
 656                 saved_cntkctl = arch_timer_get_cntkctl();
 657         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 658                 arch_timer_set_cntkctl(saved_cntkctl);
 659         return NOTIFY_OK;
 660 }
 661
 662 static struct notifier_block arch_timer_cpu_pm_notifier = {
 663         .notifier_call = arch_timer_cpu_pm_notify,
 664 };
 665
 666 static int __init arch_timer_cpu_pm_init(void)
 667 {
 668         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 669 }
 670
 671 static void __init arch_timer_cpu_pm_deinit(void)
 672 {
 673         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
 674 }
 675
 676 #else
 677 static int __init arch_timer_cpu_pm_init(void)
 678 {
 679         return 0;
 680 }
 681
 682 static void __init arch_timer_cpu_pm_deinit(void)
 683 {
 684 }
 685 #endif
 686
 687 static int __init arch_timer_register(void)
 688 {
 689         int err;
 690         int ppi;
 691
 692         arch_timer_evt = alloc_percpu(struct clock_event_device);
 693         if (!arch_timer_evt) {
 694                 err = -ENOMEM;
 695                 goto out;
 696         }
 697
 698         ppi = arch_timer_ppi[arch_timer_uses_ppi];
 699         switch (arch_timer_uses_ppi) {
 700         case VIRT_PPI:
 701                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 702                                          "arch_timer", arch_timer_evt);
 703                 break;
 704         case PHYS_SECURE_PPI:
 705         case PHYS_NONSECURE_PPI:
 706                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 707                                          "arch_timer", arch_timer_evt);
 708                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 709                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 710                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 711                                                  "arch_timer", arch_timer_evt);
 712                         if (err)
 713                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 714                                                 arch_timer_evt);
 715                 }
 716                 break;
 717         case HYP_PPI:
 718                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 719                                          "arch_timer", arch_timer_evt);
 720                 break;
 721         default:
 722                 BUG();
 723         }
 724
 725         if (err) {
 726                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 727                        ppi, err);
 728                 goto out_free;
 729         }
 730
 731         err = arch_timer_cpu_pm_init();
 732         if (err)
 733                 goto out_unreg_notify;
 734
 735
 736         /* Register and immediately configure the timer on the boot CPU */
 737         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
 738                                 "AP_ARM_ARCH_TIMER_STARTING",
 739                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
 740         if (err)
 741                 goto out_unreg_cpupm;
 742         return 0;
 743
 744 out_unreg_cpupm:
 745         arch_timer_cpu_pm_deinit();
 746
 747 out_unreg_notify:
 748         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 749         if (arch_timer_has_nonsecure_ppi())
 750                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 751                                 arch_timer_evt);
 752
 753 out_free:
 754         free_percpu(arch_timer_evt);
 755 out:
 756         return err;
 757 }
 758
 759 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 760 {
 761         int ret;
 762         irq_handler_t func;
 763         struct arch_timer *t;
 764
 765         t = kzalloc(sizeof(*t), GFP_KERNEL);
 766         if (!t)
 767                 return -ENOMEM;
 768
 769         t->base = base;
 770         t->evt.irq = irq;
 771         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 772
 773         if (arch_timer_mem_use_virtual)
 774                 func = arch_timer_handler_virt_mem;
 775         else
 776                 func = arch_timer_handler_phys_mem;
 777
 778         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 779         if (ret) {
 780                 pr_err("arch_timer: Failed to request mem timer irq\n");
 781                 kfree(t);
 782         }
 783
 784         return ret;
 785 }
 786
 787 static const struct of_device_id arch_timer_of_match[] __initconst = {
 788         { .compatible   = "arm,armv7-timer",    },
 789         { .compatible   = "arm,armv8-timer",    },
 790         {},
 791 };
 792
 793 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 794         { .compatible   = "arm,armv7-timer-mem", },
 795         {},
 796 };
 797
 798 static bool __init
 799 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 800 {
 801         struct device_node *dn;
 802         bool needs_probing = false;
 803
 804         dn = of_find_matching_node(NULL, matches);
 805         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 806                 needs_probing = true;
 807         of_node_put(dn);
 808
 809         return needs_probing;
 810 }
 811
 812 static int __init arch_timer_common_init(void)
 813 {
 814         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 815
 816         /* Wait until both nodes are probed if we have two timers */
 817         if ((arch_timers_present & mask) != mask) {
 818                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 819                         return 0;
 820                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 821                         return 0;
 822         }
 823
 824         arch_timer_banner(arch_timers_present);
 825         arch_counter_register(arch_timers_present);
 826         return arch_timer_arch_init();
 827 }
 828
 829 static int __init arch_timer_init(void)
 830 {
 831         int ret;
 832         /*
 833          * If HYP mode is available, we know that the physical timer
 834          * has been configured to be accessible from PL1. Use it, so
 835          * that a guest can use the virtual timer instead.
 836          *
 837          * If no interrupt provided for virtual timer, we'll have to
 838          * stick to the physical timer. It'd better be accessible...
 839          *
 840          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 841          * accesses to CNTP_*_EL1 registers are silently redirected to
 842          * their CNTHP_*_EL2 counterparts, and use a different PPI
 843          * number.
 844          */
 845         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 846                 bool has_ppi;
 847
 848                 if (is_kernel_in_hyp_mode()) {
 849                         arch_timer_uses_ppi = HYP_PPI;
 850                         has_ppi = !!arch_timer_ppi[HYP_PPI];
 851                 } else {
 852                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
 853                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 854                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 855                 }
 856
 857                 if (!has_ppi) {
 858                         pr_warn("arch_timer: No interrupt available, giving up\n");
 859                         return -EINVAL;
 860                 }
 861         }
 862
 863         ret = arch_timer_register();
 864         if (ret)
 865                 return ret;
 866
 867         ret = arch_timer_common_init();
 868         if (ret)
 869                 return ret;
 870
 871         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 872
 873         return 0;
 874 }
 875
 876 static int __init arch_timer_of_init(struct device_node *np)
 877 {
 878         int i;
 879
 880         if (arch_timers_present & ARCH_CP15_TIMER) {
 881                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 882                 return 0;
 883         }
 884
 885         arch_timers_present |= ARCH_CP15_TIMER;
 886         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 887                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 888
 889         arch_timer_detect_rate(NULL, np);
 890
 891         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 892
 893 #ifdef CONFIG_FSL_ERRATUM_A008585
 894         if (fsl_a008585_enable < 0)
 895                 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
 896         if (fsl_a008585_enable) {
 897                 static_branch_enable(&arch_timer_read_ool_enabled);
 898                 pr_info("Enabling workaround for FSL erratum A-008585\n");
 899         }
 900 #endif
 901
 902         /*
 903          * If we cannot rely on firmware initializing the timer registers then
 904          * we should use the physical timers instead.
 905          */
 906         if (IS_ENABLED(CONFIG_ARM) &&
 907             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 908                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
 909
 910         return arch_timer_init();
 911 }
 912 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 913 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 914
 915 static int __init arch_timer_mem_init(struct device_node *np)
 916 {
 917         struct device_node *frame, *best_frame = NULL;
 918         void __iomem *cntctlbase, *base;
 919         unsigned int irq, ret = -EINVAL;
 920         u32 cnttidr;
 921
 922         arch_timers_present |= ARCH_MEM_TIMER;
 923         cntctlbase = of_iomap(np, 0);
 924         if (!cntctlbase) {
 925                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 926                 return -ENXIO;
 927         }
 928
 929         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 930
 931         /*
 932          * Try to find a virtual capable frame. Otherwise fall back to a
 933          * physical capable frame.
 934          */
 935         for_each_available_child_of_node(np, frame) {
 936                 int n;
 937                 u32 cntacr;
 938
 939                 if (of_property_read_u32(frame, "frame-number", &n)) {
 940                         pr_err("arch_timer: Missing frame-number\n");
 941                         of_node_put(frame);
 942                         goto out;
 943                 }
 944
 945                 /* Try enabling everything, and see what sticks */
 946                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 947                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 948                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 949                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 950
 951                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
 952                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 953                         of_node_put(best_frame);
 954                         best_frame = frame;
 955                         arch_timer_mem_use_virtual = true;
 956                         break;
 957                 }
 958
 959                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 960                         continue;
 961
 962                 of_node_put(best_frame);
 963                 best_frame = of_node_get(frame);
 964         }
 965
 966         ret= -ENXIO;
 967         base = arch_counter_base = of_iomap(best_frame, 0);
 968         if (!base) {
 969                 pr_err("arch_timer: Can't map frame's registers\n");
 970                 goto out;
 971         }
 972
 973         if (arch_timer_mem_use_virtual)
 974                 irq = irq_of_parse_and_map(best_frame, 1);
 975         else
 976                 irq = irq_of_parse_and_map(best_frame, 0);
 977
 978         ret = -EINVAL;
 979         if (!irq) {
 980                 pr_err("arch_timer: Frame missing %s irq",
 981                        arch_timer_mem_use_virtual ? "virt" : "phys");
 982                 goto out;
 983         }
 984
 985         arch_timer_detect_rate(base, np);
 986         ret = arch_timer_mem_register(base, irq);
 987         if (ret)
 988                 goto out;
 989
 990         return arch_timer_common_init();
 991 out:
 992         iounmap(cntctlbase);
 993         of_node_put(best_frame);
 994         return ret;
 995 }
 996 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 997                        arch_timer_mem_init);
 998
 999 #ifdef CONFIG_ACPI
1000 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
1001 {
1002         int trigger, polarity;
1003
1004         if (!interrupt)
1005                 return 0;
1006
1007         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1008                         : ACPI_LEVEL_SENSITIVE;
1009
1010         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1011                         : ACPI_ACTIVE_HIGH;
1012
1013         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1014 }
1015
1016 /* Initialize per-processor generic timer */
1017 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1018 {
1019         struct acpi_table_gtdt *gtdt;
1020
1021         if (arch_timers_present & ARCH_CP15_TIMER) {
1022                 pr_warn("arch_timer: already initialized, skipping\n");
1023                 return -EINVAL;
1024         }
1025
1026         gtdt = container_of(table, struct acpi_table_gtdt, header);
1027
1028         arch_timers_present |= ARCH_CP15_TIMER;
1029
1030         arch_timer_ppi[PHYS_SECURE_PPI] =
1031                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1032                 gtdt->secure_el1_flags);
1033
1034         arch_timer_ppi[PHYS_NONSECURE_PPI] =
1035                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1036                 gtdt->non_secure_el1_flags);
1037
1038         arch_timer_ppi[VIRT_PPI] =
1039                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1040                 gtdt->virtual_timer_flags);
1041
1042         arch_timer_ppi[HYP_PPI] =
1043                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1044                 gtdt->non_secure_el2_flags);
1045
1046         /* Get the frequency from CNTFRQ */
1047         arch_timer_detect_rate(NULL, NULL);
1048
1049         /* Always-on capability */
1050         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1051
1052         arch_timer_init();
1053         return 0;
1054 }
1055 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1056 #endif