arch/arm/mach-exynos/mcpm-exynos.c

   1 /*
   2  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
   3  *              http://www.samsung.com
   4  *
   5  * arch/arm/mach-exynos/mcpm-exynos.c
   6  *
   7  * Based on arch/arm/mach-vexpress/dcscb.c
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/arm-cci.h>
  15 #include <linux/delay.h>
  16 #include <linux/io.h>
  17 #include <linux/of_address.h>
  18 #include <linux/syscore_ops.h>
  19
  20 #include <asm/cputype.h>
  21 #include <asm/cp15.h>
  22 #include <asm/mcpm.h>
  23
  24 #include "regs-pmu.h"
  25 #include "common.h"
  26
  27 #define EXYNOS5420_CPUS_PER_CLUSTER     4
  28 #define EXYNOS5420_NR_CLUSTERS          2
  29
  30 #define EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN   BIT(9)
  31 #define EXYNOS5420_USE_ARM_CORE_DOWN_STATE      BIT(29)
  32 #define EXYNOS5420_USE_L2_COMMON_UP_STATE       BIT(30)
  33
  34 static void __iomem *ns_sram_base_addr;
  35
  36 /*
  37  * The common v7_exit_coherency_flush API could not be used because of the
  38  * Erratum 799270 workaround. This macro is the same as the common one (in
  39  * arch/arm/include/asm/cacheflush.h) except for the erratum handling.
  40  */
  41 #define exynos_v7_exit_coherency_flush(level) \
  42         asm volatile( \
  43         "stmfd  sp!, {fp, ip}\n\t"\
  44         "mrc    p15, 0, r0, c1, c0, 0   @ get SCTLR\n\t" \
  45         "bic    r0, r0, #"__stringify(CR_C)"\n\t" \
  46         "mcr    p15, 0, r0, c1, c0, 0   @ set SCTLR\n\t" \
  47         "isb\n\t"\
  48         "bl     v7_flush_dcache_"__stringify(level)"\n\t" \
  49         "mrc    p15, 0, r0, c1, c0, 1   @ get ACTLR\n\t" \
  50         "bic    r0, r0, #(1 << 6)       @ disable local coherency\n\t" \
  51         /* Dummy Load of a device register to avoid Erratum 799270 */ \
  52         "ldr    r4, [%0]\n\t" \
  53         "and    r4, r4, #0\n\t" \
  54         "orr    r0, r0, r4\n\t" \
  55         "mcr    p15, 0, r0, c1, c0, 1   @ set ACTLR\n\t" \
  56         "isb\n\t" \
  57         "dsb\n\t" \
  58         "ldmfd  sp!, {fp, ip}" \
  59         : \
  60         : "Ir" (pmu_base_addr + S5P_INFORM0) \
  61         : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \
  62           "r9", "r10", "lr", "memory")
  63
  64 /*
  65  * We can't use regular spinlocks. In the switcher case, it is possible
  66  * for an outbound CPU to call power_down() after its inbound counterpart
  67  * is already live using the same logical CPU number which trips lockdep
  68  * debugging.
  69  */
  70 static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  71 static int
  72 cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS];
  73
  74 #define exynos_cluster_usecnt(cluster) \
  75         (cpu_use_count[0][cluster] +   \
  76          cpu_use_count[1][cluster] +   \
  77          cpu_use_count[2][cluster] +   \
  78          cpu_use_count[3][cluster])
  79
  80 #define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster)
  81
  82 static int exynos_power_up(unsigned int cpu, unsigned int cluster)
  83 {
  84         unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
  85
  86         pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
  87         if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
  88                 cluster >= EXYNOS5420_NR_CLUSTERS)
  89                 return -EINVAL;
  90
  91         /*
  92          * Since this is called with IRQs enabled, and no arch_spin_lock_irq
  93          * variant exists, we need to disable IRQs manually here.
  94          */
  95         local_irq_disable();
  96         arch_spin_lock(&exynos_mcpm_lock);
  97
  98         cpu_use_count[cpu][cluster]++;
  99         if (cpu_use_count[cpu][cluster] == 1) {
 100                 bool was_cluster_down =
 101                         (exynos_cluster_usecnt(cluster) == 1);
 102
 103                 /*
 104                  * Turn on the cluster (L2/COMMON) and then power on the
 105                  * cores.
 106                  */
 107                 if (was_cluster_down)
 108                         exynos_cluster_power_up(cluster);
 109
 110                 exynos_cpu_power_up(cpunr);
 111         } else if (cpu_use_count[cpu][cluster] != 2) {
 112                 /*
 113                  * The only possible values are:
 114                  * 0 = CPU down
 115                  * 1 = CPU (still) up
 116                  * 2 = CPU requested to be up before it had a chance
 117                  *     to actually make itself down.
 118                  * Any other value is a bug.
 119                  */
 120                 BUG();
 121         }
 122
 123         arch_spin_unlock(&exynos_mcpm_lock);
 124         local_irq_enable();
 125
 126         return 0;
 127 }
 128
 129 /*
 130  * NOTE: This function requires the stack data to be visible through power down
 131  * and can only be executed on processors like A15 and A7 that hit the cache
 132  * with the C bit clear in the SCTLR register.
 133  */
 134 static void exynos_power_down(void)
 135 {
 136         unsigned int mpidr, cpu, cluster;
 137         bool last_man = false, skip_wfi = false;
 138         unsigned int cpunr;
 139
 140         mpidr = read_cpuid_mpidr();
 141         cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 142         cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 143         cpunr =  cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
 144
 145         pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
 146         BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
 147                         cluster >= EXYNOS5420_NR_CLUSTERS);
 148
 149         __mcpm_cpu_going_down(cpu, cluster);
 150
 151         arch_spin_lock(&exynos_mcpm_lock);
 152         BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
 153         cpu_use_count[cpu][cluster]--;
 154         if (cpu_use_count[cpu][cluster] == 0) {
 155                 exynos_cpu_power_down(cpunr);
 156
 157                 if (exynos_cluster_unused(cluster)) {
 158                         exynos_cluster_power_down(cluster);
 159                         last_man = true;
 160                 }
 161         } else if (cpu_use_count[cpu][cluster] == 1) {
 162                 /*
 163                  * A power_up request went ahead of us.
 164                  * Even if we do not want to shut this CPU down,
 165                  * the caller expects a certain state as if the WFI
 166                  * was aborted.  So let's continue with cache cleaning.
 167                  */
 168                 skip_wfi = true;
 169         } else {
 170                 BUG();
 171         }
 172
 173         if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
 174                 arch_spin_unlock(&exynos_mcpm_lock);
 175
 176                 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) {
 177                         /*
 178                          * On the Cortex-A15 we need to disable
 179                          * L2 prefetching before flushing the cache.
 180                          */
 181                         asm volatile(
 182                         "mcr    p15, 1, %0, c15, c0, 3\n\t"
 183                         "isb\n\t"
 184                         "dsb"
 185                         : : "r" (0x400));
 186                 }
 187
 188                 /* Flush all cache levels for this cluster. */
 189                 exynos_v7_exit_coherency_flush(all);
 190
 191                 /*
 192                  * Disable cluster-level coherency by masking
 193                  * incoming snoops and DVM messages:
 194                  */
 195                 cci_disable_port_by_cpu(mpidr);
 196
 197                 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
 198         } else {
 199                 arch_spin_unlock(&exynos_mcpm_lock);
 200
 201                 /* Disable and flush the local CPU cache. */
 202                 exynos_v7_exit_coherency_flush(louis);
 203         }
 204
 205         __mcpm_cpu_down(cpu, cluster);
 206
 207         /* Now we are prepared for power-down, do it: */
 208         if (!skip_wfi)
 209                 wfi();
 210
 211         /* Not dead at this point?  Let our caller cope. */
 212 }
 213
 214 static int exynos_wait_for_powerdown(unsigned int cpu, unsigned int cluster)
 215 {
 216         unsigned int tries = 100;
 217         unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
 218
 219         pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
 220         BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
 221                         cluster >= EXYNOS5420_NR_CLUSTERS);
 222
 223         /* Wait for the core state to be OFF */
 224         while (tries--) {
 225                 if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) {
 226                         if ((exynos_cpu_power_state(cpunr) == 0))
 227                                 return 0; /* success: the CPU is halted */
 228                 }
 229
 230                 /* Otherwise, wait and retry: */
 231                 msleep(1);
 232         }
 233
 234         return -ETIMEDOUT; /* timeout */
 235 }
 236
 237 static void exynos_powered_up(void)
 238 {
 239         unsigned int mpidr, cpu, cluster;
 240
 241         mpidr = read_cpuid_mpidr();
 242         cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 243         cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 244
 245         arch_spin_lock(&exynos_mcpm_lock);
 246         if (cpu_use_count[cpu][cluster] == 0)
 247                 cpu_use_count[cpu][cluster] = 1;
 248         arch_spin_unlock(&exynos_mcpm_lock);
 249 }
 250
 251 static void exynos_suspend(u64 residency)
 252 {
 253         unsigned int mpidr, cpunr;
 254
 255         exynos_power_down();
 256
 257         /*
 258          * Execution reaches here only if cpu did not power down.
 259          * Hence roll back the changes done in exynos_power_down function.
 260          *
 261          * CAUTION: "This function requires the stack data to be visible through
 262          * power down and can only be executed on processors like A15 and A7
 263          * that hit the cache with the C bit clear in the SCTLR register."
 264         */
 265         mpidr = read_cpuid_mpidr();
 266         cpunr = exynos_pmu_cpunr(mpidr);
 267
 268         exynos_cpu_power_up(cpunr);
 269 }
 270
 271 static const struct mcpm_platform_ops exynos_power_ops = {
 272         .power_up               = exynos_power_up,
 273         .power_down             = exynos_power_down,
 274         .wait_for_powerdown     = exynos_wait_for_powerdown,
 275         .suspend                = exynos_suspend,
 276         .powered_up             = exynos_powered_up,
 277 };
 278
 279 static void __init exynos_mcpm_usage_count_init(void)
 280 {
 281         unsigned int mpidr, cpu, cluster;
 282
 283         mpidr = read_cpuid_mpidr();
 284         cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 285         cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 286
 287         pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
 288         BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER  ||
 289                         cluster >= EXYNOS5420_NR_CLUSTERS);
 290
 291         cpu_use_count[cpu][cluster] = 1;
 292 }
 293
 294 /*
 295  * Enable cluster-level coherency, in preparation for turning on the MMU.
 296  */
 297 static void __naked exynos_pm_power_up_setup(unsigned int affinity_level)
 298 {
 299         asm volatile ("\n"
 300         "cmp    r0, #1\n"
 301         "bxne   lr\n"
 302         "b      cci_enable_port_for_self");
 303 }
 304
 305 static void __init exynos_cache_off(void)
 306 {
 307         if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) {
 308                 /* disable L2 prefetching on the Cortex-A15 */
 309                 asm volatile(
 310                 "mcr    p15, 1, %0, c15, c0, 3\n\t"
 311                 "isb\n\t"
 312                 "dsb"
 313                 : : "r" (0x400));
 314         }
 315         exynos_v7_exit_coherency_flush(all);
 316 }
 317
 318 static const struct of_device_id exynos_dt_mcpm_match[] = {
 319         { .compatible = "samsung,exynos5420" },
 320         { .compatible = "samsung,exynos5800" },
 321         {},
 322 };
 323
 324 static void exynos_mcpm_setup_entry_point(void)
 325 {
 326         /*
 327          * U-Boot SPL is hardcoded to jump to the start of ns_sram_base_addr
 328          * as part of secondary_cpu_start().  Let's redirect it to the
 329          * mcpm_entry_point(). This is done during both secondary boot-up as
 330          * well as system resume.
 331          */
 332         __raw_writel(0xe59f0000, ns_sram_base_addr);     /* ldr r0, [pc, #0] */
 333         __raw_writel(0xe12fff10, ns_sram_base_addr + 4); /* bx  r0 */
 334         __raw_writel(virt_to_phys(mcpm_entry_point), ns_sram_base_addr + 8);
 335 }
 336
 337 static struct syscore_ops exynos_mcpm_syscore_ops = {
 338         .resume = exynos_mcpm_setup_entry_point,
 339 };
 340
 341 static int __init exynos_mcpm_init(void)
 342 {
 343         struct device_node *node;
 344         unsigned int value, i;
 345         int ret;
 346
 347         node = of_find_matching_node(NULL, exynos_dt_mcpm_match);
 348         if (!node)
 349                 return -ENODEV;
 350         of_node_put(node);
 351
 352         if (!cci_probed())
 353                 return -ENODEV;
 354
 355         node = of_find_compatible_node(NULL, NULL,
 356                         "samsung,exynos4210-sysram-ns");
 357         if (!node)
 358                 return -ENODEV;
 359
 360         ns_sram_base_addr = of_iomap(node, 0);
 361         of_node_put(node);
 362         if (!ns_sram_base_addr) {
 363                 pr_err("failed to map non-secure iRAM base address\n");
 364                 return -ENOMEM;
 365         }
 366
 367         /*
 368          * To increase the stability of KFC reset we need to program
 369          * the PMU SPARE3 register
 370          */
 371         pmu_raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3);
 372
 373         exynos_mcpm_usage_count_init();
 374
 375         ret = mcpm_platform_register(&exynos_power_ops);
 376         if (!ret)
 377                 ret = mcpm_sync_init(exynos_pm_power_up_setup);
 378         if (!ret)
 379                 ret = mcpm_loopback(exynos_cache_off); /* turn on the CCI */
 380         if (ret) {
 381                 iounmap(ns_sram_base_addr);
 382                 return ret;
 383         }
 384
 385         mcpm_smp_set_ops();
 386
 387         pr_info("Exynos MCPM support installed\n");
 388
 389         /*
 390          * On Exynos5420/5800 for the A15 and A7 clusters:
 391          *
 392          * EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN ensures that all the cores
 393          * in a cluster are turned off before turning off the cluster L2.
 394          *
 395          * EXYNOS5420_USE_ARM_CORE_DOWN_STATE ensures that a cores is powered
 396          * off before waking it up.
 397          *
 398          * EXYNOS5420_USE_L2_COMMON_UP_STATE ensures that cluster L2 will be
 399          * turned on before the first man is powered up.
 400          */
 401         for (i = 0; i < EXYNOS5420_NR_CLUSTERS; i++) {
 402                 value = pmu_raw_readl(EXYNOS_COMMON_OPTION(i));
 403                 value |= EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN |
 404                          EXYNOS5420_USE_ARM_CORE_DOWN_STATE    |
 405                          EXYNOS5420_USE_L2_COMMON_UP_STATE;
 406                 pmu_raw_writel(value, EXYNOS_COMMON_OPTION(i));
 407         }
 408
 409         exynos_mcpm_setup_entry_point();
 410
 411         register_syscore_ops(&exynos_mcpm_syscore_ops);
 412
 413         return ret;
 414 }
 415
 416 early_initcall(exynos_mcpm_init);