arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 #include <asm/cp15.h>
  22 #include <asm/cputype.h>
  23 #include <asm/irq_regs.h>
  24 #include <asm/vfp.h>
  25 #include "../vfp/vfpinstr.h"
  26
  27 #include <linux/of.h>
  28 #include <linux/perf/arm_pmu.h>
  29 #include <linux/platform_device.h>
  30
  31 /*
  32  * Common ARMv7 event types
  33  *
  34  * Note: An implementation may not be able to count all of these events
  35  * but the encodings are considered to be `reserved' in the case that
  36  * they are not available.
  37  */
  38 #define ARMV7_PERFCTR_PMNC_SW_INCR                      0x00
  39 #define ARMV7_PERFCTR_L1_ICACHE_REFILL                  0x01
  40 #define ARMV7_PERFCTR_ITLB_REFILL                       0x02
  41 #define ARMV7_PERFCTR_L1_DCACHE_REFILL                  0x03
  42 #define ARMV7_PERFCTR_L1_DCACHE_ACCESS                  0x04
  43 #define ARMV7_PERFCTR_DTLB_REFILL                       0x05
  44 #define ARMV7_PERFCTR_MEM_READ                          0x06
  45 #define ARMV7_PERFCTR_MEM_WRITE                         0x07
  46 #define ARMV7_PERFCTR_INSTR_EXECUTED                    0x08
  47 #define ARMV7_PERFCTR_EXC_TAKEN                         0x09
  48 #define ARMV7_PERFCTR_EXC_EXECUTED                      0x0A
  49 #define ARMV7_PERFCTR_CID_WRITE                         0x0B
  50
  51 /*
  52  * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  53  * It counts:
  54  *  - all (taken) branch instructions,
  55  *  - instructions that explicitly write the PC,
  56  *  - exception generating instructions.
  57  */
  58 #define ARMV7_PERFCTR_PC_WRITE                          0x0C
  59 #define ARMV7_PERFCTR_PC_IMM_BRANCH                     0x0D
  60 #define ARMV7_PERFCTR_PC_PROC_RETURN                    0x0E
  61 #define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              0x0F
  62 #define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                0x10
  63 #define ARMV7_PERFCTR_CLOCK_CYCLES                      0x11
  64 #define ARMV7_PERFCTR_PC_BRANCH_PRED                    0x12
  65
  66 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  67 #define ARMV7_PERFCTR_MEM_ACCESS                        0x13
  68 #define ARMV7_PERFCTR_L1_ICACHE_ACCESS                  0x14
  69 #define ARMV7_PERFCTR_L1_DCACHE_WB                      0x15
  70 #define ARMV7_PERFCTR_L2_CACHE_ACCESS                   0x16
  71 #define ARMV7_PERFCTR_L2_CACHE_REFILL                   0x17
  72 #define ARMV7_PERFCTR_L2_CACHE_WB                       0x18
  73 #define ARMV7_PERFCTR_BUS_ACCESS                        0x19
  74 #define ARMV7_PERFCTR_MEM_ERROR                         0x1A
  75 #define ARMV7_PERFCTR_INSTR_SPEC                        0x1B
  76 #define ARMV7_PERFCTR_TTBR_WRITE                        0x1C
  77 #define ARMV7_PERFCTR_BUS_CYCLES                        0x1D
  78
  79 #define ARMV7_PERFCTR_CPU_CYCLES                        0xFF
  80
  81 /* ARMv7 Cortex-A8 specific event types */
  82 #define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                0x43
  83 #define ARMV7_A8_PERFCTR_L2_CACHE_REFILL                0x44
  84 #define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               0x50
  85 #define ARMV7_A8_PERFCTR_STALL_ISIDE                    0x56
  86
  87 /* ARMv7 Cortex-A9 specific event types */
  88 #define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              0x68
  89 #define ARMV7_A9_PERFCTR_STALL_ICACHE                   0x60
  90 #define ARMV7_A9_PERFCTR_STALL_DISPATCH                 0x66
  91
  92 /* ARMv7 Cortex-A5 specific event types */
  93 #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              0xc2
  94 #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         0xc3
  95
  96 /* ARMv7 Cortex-A15 specific event types */
  97 #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         0x40
  98 #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        0x41
  99 #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         0x42
 100 #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        0x43
 101
 102 #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           0x4C
 103 #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          0x4D
 104
 105 #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          0x50
 106 #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         0x51
 107 #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          0x52
 108 #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         0x53
 109
 110 #define ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 0x76
 111
 112 /* ARMv7 Cortex-A12 specific event types */
 113 #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ         0x40
 114 #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE        0x41
 115
 116 #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ          0x50
 117 #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE         0x51
 118
 119 #define ARMV7_A12_PERFCTR_PC_WRITE_SPEC                 0x76
 120
 121 #define ARMV7_A12_PERFCTR_PF_TLB_REFILL                 0xe7
 122
 123 /* ARMv7 Krait specific event types */
 124 #define KRAIT_PMRESR0_GROUP0                            0xcc
 125 #define KRAIT_PMRESR1_GROUP0                            0xd0
 126 #define KRAIT_PMRESR2_GROUP0                            0xd4
 127 #define KRAIT_VPMRESR0_GROUP0                           0xd8
 128
 129 #define KRAIT_PERFCTR_L1_ICACHE_ACCESS                  0x10011
 130 #define KRAIT_PERFCTR_L1_ICACHE_MISS                    0x10010
 131
 132 #define KRAIT_PERFCTR_L1_ITLB_ACCESS                    0x12222
 133 #define KRAIT_PERFCTR_L1_DTLB_ACCESS                    0x12210
 134
 135 /* ARMv7 Scorpion specific event types */
 136 #define SCORPION_LPM0_GROUP0                            0x4c
 137 #define SCORPION_LPM1_GROUP0                            0x50
 138 #define SCORPION_LPM2_GROUP0                            0x54
 139 #define SCORPION_L2LPM_GROUP0                           0x58
 140 #define SCORPION_VLPM_GROUP0                            0x5c
 141
 142 #define SCORPION_ICACHE_ACCESS                          0x10053
 143 #define SCORPION_ICACHE_MISS                            0x10052
 144
 145 #define SCORPION_DTLB_ACCESS                            0x12013
 146 #define SCORPION_DTLB_MISS                              0x12012
 147
 148 #define SCORPION_ITLB_MISS                              0x12021
 149
 150 /*
 151  * Cortex-A8 HW events mapping
 152  *
 153  * The hardware events that we support. We do support cache operations but
 154  * we have harvard caches and no way to combine instruction and data
 155  * accesses/misses in hardware.
 156  */
 157 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 158         PERF_MAP_ALL_UNSUPPORTED,
 159         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 160         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 161         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 162         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 163         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 164         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 165         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 166 };
 167
 168 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 169                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 170                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 171         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 172
 173         /*
 174          * The performance counters don't differentiate between read and write
 175          * accesses/misses so this isn't strictly correct, but it's the best we
 176          * can do. Writes and reads get combined.
 177          */
 178         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 179         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 180         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 181         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 182
 183         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 184         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 185
 186         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 187         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 188         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 189         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 190
 191         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 192         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 193
 194         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 195         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 196
 197         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 198         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 199         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 200         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 201 };
 202
 203 /*
 204  * Cortex-A9 HW events mapping
 205  */
 206 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 207         PERF_MAP_ALL_UNSUPPORTED,
 208         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 209         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 210         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 211         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 212         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 213         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 214         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 215         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 216 };
 217
 218 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 219                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 220                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 221         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 222
 223         /*
 224          * The performance counters don't differentiate between read and write
 225          * accesses/misses so this isn't strictly correct, but it's the best we
 226          * can do. Writes and reads get combined.
 227          */
 228         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 229         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 230         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 231         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 232
 233         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 234
 235         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 236         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 237
 238         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 239         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 240
 241         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 242         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 243         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 244         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 245 };
 246
 247 /*
 248  * Cortex-A5 HW events mapping
 249  */
 250 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 251         PERF_MAP_ALL_UNSUPPORTED,
 252         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 253         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 254         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 255         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 256         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 257         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 258 };
 259
 260 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 261                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 262                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 263         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 264
 265         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 266         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 267         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 268         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 269         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 270         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 271
 272         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 273         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 274         /*
 275          * The prefetch counters don't differentiate between the I side and the
 276          * D side.
 277          */
 278         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 279         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 280
 281         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 282         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 283
 284         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 285         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 286
 287         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 288         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 289         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 290         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 291 };
 292
 293 /*
 294  * Cortex-A15 HW events mapping
 295  */
 296 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 297         PERF_MAP_ALL_UNSUPPORTED,
 298         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 299         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 300         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 301         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 302         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 303         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 304         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 305 };
 306
 307 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 308                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 309                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 310         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 311
 312         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 313         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 314         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 315         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 316
 317         /*
 318          * Not all performance counters differentiate between read and write
 319          * accesses/misses so we're not always strictly correct, but it's the
 320          * best we can do. Writes and reads get combined in these cases.
 321          */
 322         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 323         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 324
 325         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 326         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 327         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 328         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 329
 330         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 331         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 332
 333         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 334         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 335
 336         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 337         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 338         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 339         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 340 };
 341
 342 /*
 343  * Cortex-A7 HW events mapping
 344  */
 345 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 346         PERF_MAP_ALL_UNSUPPORTED,
 347         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 348         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 349         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 350         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 351         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 352         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 353         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 354 };
 355
 356 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 357                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 358                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 359         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 360
 361         /*
 362          * The performance counters don't differentiate between read and write
 363          * accesses/misses so this isn't strictly correct, but it's the best we
 364          * can do. Writes and reads get combined.
 365          */
 366         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 367         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 368         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 369         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 370
 371         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 372         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 373
 374         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 375         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 376         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 377         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 378
 379         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 380         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 381
 382         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 383         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 384
 385         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 386         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 387         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 388         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 389 };
 390
 391 /*
 392  * Cortex-A12 HW events mapping
 393  */
 394 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
 395         PERF_MAP_ALL_UNSUPPORTED,
 396         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 397         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 398         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 399         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 400         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
 401         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 402         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 403 };
 404
 405 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 406                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 407                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 408         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 409
 410         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
 411         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 412         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 413         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 414
 415         /*
 416          * Not all performance counters differentiate between read and write
 417          * accesses/misses so we're not always strictly correct, but it's the
 418          * best we can do. Writes and reads get combined in these cases.
 419          */
 420         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 421         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 422
 423         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
 424         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 425         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
 426         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 427
 428         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 429         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 430         [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]       = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
 431
 432         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 433         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 434
 435         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 436         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 437         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 438         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 439 };
 440
 441 /*
 442  * Krait HW events mapping
 443  */
 444 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
 445         PERF_MAP_ALL_UNSUPPORTED,
 446         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 447         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 448         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 449         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 450         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 451 };
 452
 453 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
 454         PERF_MAP_ALL_UNSUPPORTED,
 455         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 456         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 457         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 458         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 459 };
 460
 461 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 462                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 463                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 464         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 465
 466         /*
 467          * The performance counters don't differentiate between read and write
 468          * accesses/misses so this isn't strictly correct, but it's the best we
 469          * can do. Writes and reads get combined.
 470          */
 471         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 472         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 473         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 474         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 475
 476         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
 477         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = KRAIT_PERFCTR_L1_ICACHE_MISS,
 478
 479         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 480         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 481
 482         [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 483         [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 484
 485         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 486         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 487         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 488         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 489 };
 490
 491 /*
 492  * Scorpion HW events mapping
 493  */
 494 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
 495         PERF_MAP_ALL_UNSUPPORTED,
 496         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 497         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 498         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 499         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 500         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 501 };
 502
 503 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 504                                             [PERF_COUNT_HW_CACHE_OP_MAX]
 505                                             [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 506         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 507         /*
 508          * The performance counters don't differentiate between read and write
 509          * accesses/misses so this isn't strictly correct, but it's the best we
 510          * can do. Writes and reads get combined.
 511          */
 512         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 513         [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 514         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 515         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 516         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
 517         [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
 518         /*
 519          * Only ITLB misses and DTLB refills are supported.  If users want the
 520          * DTLB refills misses a raw counter must be used.
 521          */
 522         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 523         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 524         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 525         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 526         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 527         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 528         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 529         [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 530         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 531         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 532 };
 533
 534 PMU_FORMAT_ATTR(event, "config:0-7");
 535
 536 static struct attribute *armv7_pmu_format_attrs[] = {
 537         &format_attr_event.attr,
 538         NULL,
 539 };
 540
 541 static struct attribute_group armv7_pmu_format_attr_group = {
 542         .name = "format",
 543         .attrs = armv7_pmu_format_attrs,
 544 };
 545
 546 #define ARMV7_EVENT_ATTR_RESOLVE(m) #m
 547 #define ARMV7_EVENT_ATTR(name, config) \
 548         PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \
 549                               "event=" ARMV7_EVENT_ATTR_RESOLVE(config))
 550
 551 ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);
 552 ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);
 553 ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);
 554 ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);
 555 ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);
 556 ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);
 557 ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);
 558 ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);
 559 ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);
 560 ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);
 561 ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);
 562 ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);
 563 ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);
 564 ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);
 565 ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);
 566 ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);
 567 ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);
 568 ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);
 569 ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);
 570
 571 static struct attribute *armv7_pmuv1_event_attrs[] = {
 572         &armv7_event_attr_sw_incr.attr.attr,
 573         &armv7_event_attr_l1i_cache_refill.attr.attr,
 574         &armv7_event_attr_l1i_tlb_refill.attr.attr,
 575         &armv7_event_attr_l1d_cache_refill.attr.attr,
 576         &armv7_event_attr_l1d_cache.attr.attr,
 577         &armv7_event_attr_l1d_tlb_refill.attr.attr,
 578         &armv7_event_attr_ld_retired.attr.attr,
 579         &armv7_event_attr_st_retired.attr.attr,
 580         &armv7_event_attr_inst_retired.attr.attr,
 581         &armv7_event_attr_exc_taken.attr.attr,
 582         &armv7_event_attr_exc_return.attr.attr,
 583         &armv7_event_attr_cid_write_retired.attr.attr,
 584         &armv7_event_attr_pc_write_retired.attr.attr,
 585         &armv7_event_attr_br_immed_retired.attr.attr,
 586         &armv7_event_attr_br_return_retired.attr.attr,
 587         &armv7_event_attr_unaligned_ldst_retired.attr.attr,
 588         &armv7_event_attr_br_mis_pred.attr.attr,
 589         &armv7_event_attr_cpu_cycles.attr.attr,
 590         &armv7_event_attr_br_pred.attr.attr,
 591         NULL,
 592 };
 593
 594 static struct attribute_group armv7_pmuv1_events_attr_group = {
 595         .name = "events",
 596         .attrs = armv7_pmuv1_event_attrs,
 597 };
 598
 599 static const struct attribute_group *armv7_pmuv1_attr_groups[] = {
 600         &armv7_pmuv1_events_attr_group,
 601         &armv7_pmu_format_attr_group,
 602         NULL,
 603 };
 604
 605 ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);
 606 ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);
 607 ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);
 608 ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);
 609 ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);
 610 ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);
 611 ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);
 612 ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);
 613 ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);
 614 ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);
 615 ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);
 616
 617 static struct attribute *armv7_pmuv2_event_attrs[] = {
 618         &armv7_event_attr_sw_incr.attr.attr,
 619         &armv7_event_attr_l1i_cache_refill.attr.attr,
 620         &armv7_event_attr_l1i_tlb_refill.attr.attr,
 621         &armv7_event_attr_l1d_cache_refill.attr.attr,
 622         &armv7_event_attr_l1d_cache.attr.attr,
 623         &armv7_event_attr_l1d_tlb_refill.attr.attr,
 624         &armv7_event_attr_ld_retired.attr.attr,
 625         &armv7_event_attr_st_retired.attr.attr,
 626         &armv7_event_attr_inst_retired.attr.attr,
 627         &armv7_event_attr_exc_taken.attr.attr,
 628         &armv7_event_attr_exc_return.attr.attr,
 629         &armv7_event_attr_cid_write_retired.attr.attr,
 630         &armv7_event_attr_pc_write_retired.attr.attr,
 631         &armv7_event_attr_br_immed_retired.attr.attr,
 632         &armv7_event_attr_br_return_retired.attr.attr,
 633         &armv7_event_attr_unaligned_ldst_retired.attr.attr,
 634         &armv7_event_attr_br_mis_pred.attr.attr,
 635         &armv7_event_attr_cpu_cycles.attr.attr,
 636         &armv7_event_attr_br_pred.attr.attr,
 637         &armv7_event_attr_mem_access.attr.attr,
 638         &armv7_event_attr_l1i_cache.attr.attr,
 639         &armv7_event_attr_l1d_cache_wb.attr.attr,
 640         &armv7_event_attr_l2d_cache.attr.attr,
 641         &armv7_event_attr_l2d_cache_refill.attr.attr,
 642         &armv7_event_attr_l2d_cache_wb.attr.attr,
 643         &armv7_event_attr_bus_access.attr.attr,
 644         &armv7_event_attr_memory_error.attr.attr,
 645         &armv7_event_attr_inst_spec.attr.attr,
 646         &armv7_event_attr_ttbr_write_retired.attr.attr,
 647         &armv7_event_attr_bus_cycles.attr.attr,
 648         NULL,
 649 };
 650
 651 static struct attribute_group armv7_pmuv2_events_attr_group = {
 652         .name = "events",
 653         .attrs = armv7_pmuv2_event_attrs,
 654 };
 655
 656 static const struct attribute_group *armv7_pmuv2_attr_groups[] = {
 657         &armv7_pmuv2_events_attr_group,
 658         &armv7_pmu_format_attr_group,
 659         NULL,
 660 };
 661
 662 /*
 663  * Perf Events' indices
 664  */
 665 #define ARMV7_IDX_CYCLE_COUNTER 0
 666 #define ARMV7_IDX_COUNTER0      1
 667 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
 668         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 669
 670 #define ARMV7_MAX_COUNTERS      32
 671 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 672
 673 /*
 674  * ARMv7 low level PMNC access
 675  */
 676
 677 /*
 678  * Perf Event to low level counters mapping
 679  */
 680 #define ARMV7_IDX_TO_COUNTER(x) \
 681         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 682
 683 /*
 684  * Per-CPU PMNC: config reg
 685  */
 686 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 687 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 688 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 689 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 690 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 691 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 692 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 693 #define ARMV7_PMNC_N_MASK       0x1f
 694 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 695
 696 /*
 697  * FLAG: counters overflow flag status reg
 698  */
 699 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 700 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 701
 702 /*
 703  * PMXEVTYPER: Event selection reg
 704  */
 705 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
 706 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 707
 708 /*
 709  * Event filters for PMUv2
 710  */
 711 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 712 #define ARMV7_EXCLUDE_USER      (1 << 30)
 713 #define ARMV7_INCLUDE_HYP       (1 << 27)
 714
 715 /*
 716  * Secure debug enable reg
 717  */
 718 #define ARMV7_SDER_SUNIDEN      BIT(1) /* Permit non-invasive debug */
 719
 720 static inline u32 armv7_pmnc_read(void)
 721 {
 722         u32 val;
 723         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 724         return val;
 725 }
 726
 727 static inline void armv7_pmnc_write(u32 val)
 728 {
 729         val &= ARMV7_PMNC_MASK;
 730         isb();
 731         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 732 }
 733
 734 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 735 {
 736         return pmnc & ARMV7_OVERFLOWED_MASK;
 737 }
 738
 739 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 740 {
 741         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
 742                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
 743 }
 744
 745 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 746 {
 747         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
 748 }
 749
 750 static inline void armv7_pmnc_select_counter(int idx)
 751 {
 752         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 753         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 754         isb();
 755 }
 756
 757 static inline u32 armv7pmu_read_counter(struct perf_event *event)
 758 {
 759         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 760         struct hw_perf_event *hwc = &event->hw;
 761         int idx = hwc->idx;
 762         u32 value = 0;
 763
 764         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 765                 pr_err("CPU%u reading wrong counter %d\n",
 766                         smp_processor_id(), idx);
 767         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 768                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 769         } else {
 770                 armv7_pmnc_select_counter(idx);
 771                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 772         }
 773
 774         return value;
 775 }
 776
 777 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
 778 {
 779         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 780         struct hw_perf_event *hwc = &event->hw;
 781         int idx = hwc->idx;
 782
 783         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 784                 pr_err("CPU%u writing wrong counter %d\n",
 785                         smp_processor_id(), idx);
 786         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 787                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 788         } else {
 789                 armv7_pmnc_select_counter(idx);
 790                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 791         }
 792 }
 793
 794 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 795 {
 796         armv7_pmnc_select_counter(idx);
 797         val &= ARMV7_EVTYPE_MASK;
 798         asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 799 }
 800
 801 static inline void armv7_pmnc_enable_counter(int idx)
 802 {
 803         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 804         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 805 }
 806
 807 static inline void armv7_pmnc_disable_counter(int idx)
 808 {
 809         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 810         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 811 }
 812
 813 static inline void armv7_pmnc_enable_intens(int idx)
 814 {
 815         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 816         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 817 }
 818
 819 static inline void armv7_pmnc_disable_intens(int idx)
 820 {
 821         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 822         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 823         isb();
 824         /* Clear the overflow flag in case an interrupt is pending. */
 825         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 826         isb();
 827 }
 828
 829 static inline u32 armv7_pmnc_getreset_flags(void)
 830 {
 831         u32 val;
 832
 833         /* Read */
 834         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 835
 836         /* Write to clear flags */
 837         val &= ARMV7_FLAG_MASK;
 838         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 839
 840         return val;
 841 }
 842
 843 #ifdef DEBUG
 844 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
 845 {
 846         u32 val;
 847         unsigned int cnt;
 848
 849         pr_info("PMNC registers dump:\n");
 850
 851         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 852         pr_info("PMNC  =0x%08x\n", val);
 853
 854         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 855         pr_info("CNTENS=0x%08x\n", val);
 856
 857         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 858         pr_info("INTENS=0x%08x\n", val);
 859
 860         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 861         pr_info("FLAGS =0x%08x\n", val);
 862
 863         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 864         pr_info("SELECT=0x%08x\n", val);
 865
 866         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 867         pr_info("CCNT  =0x%08x\n", val);
 868
 869         for (cnt = ARMV7_IDX_COUNTER0;
 870                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
 871                 armv7_pmnc_select_counter(cnt);
 872                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 873                 pr_info("CNT[%d] count =0x%08x\n",
 874                         ARMV7_IDX_TO_COUNTER(cnt), val);
 875                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 876                 pr_info("CNT[%d] evtsel=0x%08x\n",
 877                         ARMV7_IDX_TO_COUNTER(cnt), val);
 878         }
 879 }
 880 #endif
 881
 882 static void armv7pmu_enable_event(struct perf_event *event)
 883 {
 884         unsigned long flags;
 885         struct hw_perf_event *hwc = &event->hw;
 886         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 887         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 888         int idx = hwc->idx;
 889
 890         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 891                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 892                         smp_processor_id(), idx);
 893                 return;
 894         }
 895
 896         /*
 897          * Enable counter and interrupt, and set the counter to count
 898          * the event that we're interested in.
 899          */
 900         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 901
 902         /*
 903          * Disable counter
 904          */
 905         armv7_pmnc_disable_counter(idx);
 906
 907         /*
 908          * Set event (if destined for PMNx counters)
 909          * We only need to set the event for the cycle counter if we
 910          * have the ability to perform event filtering.
 911          */
 912         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 913                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 914
 915         /*
 916          * Enable interrupt for this counter
 917          */
 918         armv7_pmnc_enable_intens(idx);
 919
 920         /*
 921          * Enable counter
 922          */
 923         armv7_pmnc_enable_counter(idx);
 924
 925         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 926 }
 927
 928 static void armv7pmu_disable_event(struct perf_event *event)
 929 {
 930         unsigned long flags;
 931         struct hw_perf_event *hwc = &event->hw;
 932         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 933         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 934         int idx = hwc->idx;
 935
 936         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 937                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 938                         smp_processor_id(), idx);
 939                 return;
 940         }
 941
 942         /*
 943          * Disable counter and interrupt
 944          */
 945         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 946
 947         /*
 948          * Disable counter
 949          */
 950         armv7_pmnc_disable_counter(idx);
 951
 952         /*
 953          * Disable interrupt for this counter
 954          */
 955         armv7_pmnc_disable_intens(idx);
 956
 957         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 958 }
 959
 960 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 961 {
 962         u32 pmnc;
 963         struct perf_sample_data data;
 964         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 965         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 966         struct pt_regs *regs;
 967         int idx;
 968
 969         /*
 970          * Get and reset the IRQ flags
 971          */
 972         pmnc = armv7_pmnc_getreset_flags();
 973
 974         /*
 975          * Did an overflow occur?
 976          */
 977         if (!armv7_pmnc_has_overflowed(pmnc))
 978                 return IRQ_NONE;
 979
 980         /*
 981          * Handle the counter(s) overflow(s)
 982          */
 983         regs = get_irq_regs();
 984
 985         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 986                 struct perf_event *event = cpuc->events[idx];
 987                 struct hw_perf_event *hwc;
 988
 989                 /* Ignore if we don't have an event. */
 990                 if (!event)
 991                         continue;
 992
 993                 /*
 994                  * We have a single interrupt for all counters. Check that
 995                  * each counter has overflowed before we process it.
 996                  */
 997                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 998                         continue;
 999
1000                 hwc = &event->hw;
1001                 armpmu_event_update(event);
1002                 perf_sample_data_init(&data, 0, hwc->last_period);
1003                 if (!armpmu_event_set_period(event))
1004                         continue;
1005
1006                 if (perf_event_overflow(event, &data, regs))
1007                         cpu_pmu->disable(event);
1008         }
1009
1010         /*
1011          * Handle the pending perf events.
1012          *
1013          * Note: this call *must* be run with interrupts disabled. For
1014          * platforms that can have the PMU interrupts raised as an NMI, this
1015          * will not work.
1016          */
1017         irq_work_run();
1018
1019         return IRQ_HANDLED;
1020 }
1021
1022 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
1023 {
1024         unsigned long flags;
1025         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1026
1027         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1028         /* Enable all counters */
1029         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1030         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1031 }
1032
1033 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
1034 {
1035         unsigned long flags;
1036         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1037
1038         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1039         /* Disable all counters */
1040         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1041         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1042 }
1043
1044 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1045                                   struct perf_event *event)
1046 {
1047         int idx;
1048         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1049         struct hw_perf_event *hwc = &event->hw;
1050         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
1051
1052         /* Always place a cycle counter into the cycle counter. */
1053         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1054                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1055                         return -EAGAIN;
1056
1057                 return ARMV7_IDX_CYCLE_COUNTER;
1058         }
1059
1060         /*
1061          * For anything other than a cycle counter, try and use
1062          * the events counters
1063          */
1064         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1065                 if (!test_and_set_bit(idx, cpuc->used_mask))
1066                         return idx;
1067         }
1068
1069         /* The counters are all in use. */
1070         return -EAGAIN;
1071 }
1072
1073 /*
1074  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1075  */
1076 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1077                                      struct perf_event_attr *attr)
1078 {
1079         unsigned long config_base = 0;
1080
1081         if (attr->exclude_idle)
1082                 return -EPERM;
1083         if (attr->exclude_user)
1084                 config_base |= ARMV7_EXCLUDE_USER;
1085         if (attr->exclude_kernel)
1086                 config_base |= ARMV7_EXCLUDE_PL1;
1087         if (!attr->exclude_hv)
1088                 config_base |= ARMV7_INCLUDE_HYP;
1089
1090         /*
1091          * Install the filter into config_base as this is used to
1092          * construct the event type.
1093          */
1094         event->config_base = config_base;
1095
1096         return 0;
1097 }
1098
1099 static void armv7pmu_reset(void *info)
1100 {
1101         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1102         u32 idx, nb_cnt = cpu_pmu->num_events, val;
1103
1104         if (cpu_pmu->secure_access) {
1105                 asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));
1106                 val |= ARMV7_SDER_SUNIDEN;
1107                 asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));
1108         }
1109
1110         /* The counter and interrupt enable registers are unknown at reset. */
1111         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1112                 armv7_pmnc_disable_counter(idx);
1113                 armv7_pmnc_disable_intens(idx);
1114         }
1115
1116         /* Initialize & Reset PMNC: C and P bits */
1117         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1118 }
1119
1120 static int armv7_a8_map_event(struct perf_event *event)
1121 {
1122         return armpmu_map_event(event, &armv7_a8_perf_map,
1123                                 &armv7_a8_perf_cache_map, 0xFF);
1124 }
1125
1126 static int armv7_a9_map_event(struct perf_event *event)
1127 {
1128         return armpmu_map_event(event, &armv7_a9_perf_map,
1129                                 &armv7_a9_perf_cache_map, 0xFF);
1130 }
1131
1132 static int armv7_a5_map_event(struct perf_event *event)
1133 {
1134         return armpmu_map_event(event, &armv7_a5_perf_map,
1135                                 &armv7_a5_perf_cache_map, 0xFF);
1136 }
1137
1138 static int armv7_a15_map_event(struct perf_event *event)
1139 {
1140         return armpmu_map_event(event, &armv7_a15_perf_map,
1141                                 &armv7_a15_perf_cache_map, 0xFF);
1142 }
1143
1144 static int armv7_a7_map_event(struct perf_event *event)
1145 {
1146         return armpmu_map_event(event, &armv7_a7_perf_map,
1147                                 &armv7_a7_perf_cache_map, 0xFF);
1148 }
1149
1150 static int armv7_a12_map_event(struct perf_event *event)
1151 {
1152         return armpmu_map_event(event, &armv7_a12_perf_map,
1153                                 &armv7_a12_perf_cache_map, 0xFF);
1154 }
1155
1156 static int krait_map_event(struct perf_event *event)
1157 {
1158         return armpmu_map_event(event, &krait_perf_map,
1159                                 &krait_perf_cache_map, 0xFFFFF);
1160 }
1161
1162 static int krait_map_event_no_branch(struct perf_event *event)
1163 {
1164         return armpmu_map_event(event, &krait_perf_map_no_branch,
1165                                 &krait_perf_cache_map, 0xFFFFF);
1166 }
1167
1168 static int scorpion_map_event(struct perf_event *event)
1169 {
1170         return armpmu_map_event(event, &scorpion_perf_map,
1171                                 &scorpion_perf_cache_map, 0xFFFFF);
1172 }
1173
1174 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1175 {
1176         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1177         cpu_pmu->enable         = armv7pmu_enable_event;
1178         cpu_pmu->disable        = armv7pmu_disable_event;
1179         cpu_pmu->read_counter   = armv7pmu_read_counter;
1180         cpu_pmu->write_counter  = armv7pmu_write_counter;
1181         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1182         cpu_pmu->start          = armv7pmu_start;
1183         cpu_pmu->stop           = armv7pmu_stop;
1184         cpu_pmu->reset          = armv7pmu_reset;
1185         cpu_pmu->max_period     = (1LLU << 32) - 1;
1186 };
1187
1188 static void armv7_read_num_pmnc_events(void *info)
1189 {
1190         int *nb_cnt = info;
1191
1192         /* Read the nb of CNTx counters supported from PMNC */
1193         *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1194
1195         /* Add the CPU cycles counter */
1196         *nb_cnt += 1;
1197 }
1198
1199 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1200 {
1201         return smp_call_function_any(&arm_pmu->supported_cpus,
1202                                      armv7_read_num_pmnc_events,
1203                                      &arm_pmu->num_events, 1);
1204 }
1205
1206 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1207 {
1208         armv7pmu_init(cpu_pmu);
1209         cpu_pmu->name           = "armv7_cortex_a8";
1210         cpu_pmu->map_event      = armv7_a8_map_event;
1211         cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1212         return armv7_probe_num_events(cpu_pmu);
1213 }
1214
1215 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1216 {
1217         armv7pmu_init(cpu_pmu);
1218         cpu_pmu->name           = "armv7_cortex_a9";
1219         cpu_pmu->map_event      = armv7_a9_map_event;
1220         cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1221         return armv7_probe_num_events(cpu_pmu);
1222 }
1223
1224 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1225 {
1226         armv7pmu_init(cpu_pmu);
1227         cpu_pmu->name           = "armv7_cortex_a5";
1228         cpu_pmu->map_event      = armv7_a5_map_event;
1229         cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1230         return armv7_probe_num_events(cpu_pmu);
1231 }
1232
1233 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1234 {
1235         armv7pmu_init(cpu_pmu);
1236         cpu_pmu->name           = "armv7_cortex_a15";
1237         cpu_pmu->map_event      = armv7_a15_map_event;
1238         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1239         cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1240         return armv7_probe_num_events(cpu_pmu);
1241 }
1242
1243 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1244 {
1245         armv7pmu_init(cpu_pmu);
1246         cpu_pmu->name           = "armv7_cortex_a7";
1247         cpu_pmu->map_event      = armv7_a7_map_event;
1248         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1249         cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1250         return armv7_probe_num_events(cpu_pmu);
1251 }
1252
1253 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1254 {
1255         armv7pmu_init(cpu_pmu);
1256         cpu_pmu->name           = "armv7_cortex_a12";
1257         cpu_pmu->map_event      = armv7_a12_map_event;
1258         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1259         cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1260         return armv7_probe_num_events(cpu_pmu);
1261 }
1262
1263 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1264 {
1265         int ret = armv7_a12_pmu_init(cpu_pmu);
1266         cpu_pmu->name = "armv7_cortex_a17";
1267         cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1268         return ret;
1269 }
1270
1271 /*
1272  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1273  *
1274  *            31   30     24     16     8      0
1275  *            +--------------------------------+
1276  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1277  *            +--------------------------------+
1278  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1279  *            +--------------------------------+
1280  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1281  *            +--------------------------------+
1282  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1283  *            +--------------------------------+
1284  *              EN | G=3  | G=2  | G=1  | G=0
1285  *
1286  *  Event Encoding:
1287  *
1288  *      hwc->config_base = 0xNRCCG
1289  *
1290  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1291  *      R  = region register
1292  *      CC = class of events the group G is choosing from
1293  *      G  = group or particular event
1294  *
1295  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1296  *
1297  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1298  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1299  *  events (interrupts for example). An event code is broken down into
1300  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1301  *  example).
1302  */
1303
1304 #define KRAIT_EVENT             (1 << 16)
1305 #define VENUM_EVENT             (2 << 16)
1306 #define KRAIT_EVENT_MASK        (KRAIT_EVENT | VENUM_EVENT)
1307 #define PMRESRn_EN              BIT(31)
1308
1309 #define EVENT_REGION(event)     (((event) >> 12) & 0xf)         /* R */
1310 #define EVENT_GROUP(event)      ((event) & 0xf)                 /* G */
1311 #define EVENT_CODE(event)       (((event) >> 4) & 0xff)         /* CC */
1312 #define EVENT_VENUM(event)      (!!(event & VENUM_EVENT))       /* N=2 */
1313 #define EVENT_CPU(event)        (!!(event & KRAIT_EVENT))       /* N=1 */
1314
1315 static u32 krait_read_pmresrn(int n)
1316 {
1317         u32 val;
1318
1319         switch (n) {
1320         case 0:
1321                 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1322                 break;
1323         case 1:
1324                 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1325                 break;
1326         case 2:
1327                 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1328                 break;
1329         default:
1330                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1331         }
1332
1333         return val;
1334 }
1335
1336 static void krait_write_pmresrn(int n, u32 val)
1337 {
1338         switch (n) {
1339         case 0:
1340                 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1341                 break;
1342         case 1:
1343                 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1344                 break;
1345         case 2:
1346                 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1347                 break;
1348         default:
1349                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1350         }
1351 }
1352
1353 static u32 venum_read_pmresr(void)
1354 {
1355         u32 val;
1356         asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1357         return val;
1358 }
1359
1360 static void venum_write_pmresr(u32 val)
1361 {
1362         asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1363 }
1364
1365 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1366 {
1367         u32 venum_new_val;
1368         u32 fp_new_val;
1369
1370         BUG_ON(preemptible());
1371         /* CPACR Enable CP10 and CP11 access */
1372         *venum_orig_val = get_copro_access();
1373         venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1374         set_copro_access(venum_new_val);
1375
1376         /* Enable FPEXC */
1377         *fp_orig_val = fmrx(FPEXC);
1378         fp_new_val = *fp_orig_val | FPEXC_EN;
1379         fmxr(FPEXC, fp_new_val);
1380 }
1381
1382 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1383 {
1384         BUG_ON(preemptible());
1385         /* Restore FPEXC */
1386         fmxr(FPEXC, fp_orig_val);
1387         isb();
1388         /* Restore CPACR */
1389         set_copro_access(venum_orig_val);
1390 }
1391
1392 static u32 krait_get_pmresrn_event(unsigned int region)
1393 {
1394         static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1395                                              KRAIT_PMRESR1_GROUP0,
1396                                              KRAIT_PMRESR2_GROUP0 };
1397         return pmresrn_table[region];
1398 }
1399
1400 static void krait_evt_setup(int idx, u32 config_base)
1401 {
1402         u32 val;
1403         u32 mask;
1404         u32 vval, fval;
1405         unsigned int region = EVENT_REGION(config_base);
1406         unsigned int group = EVENT_GROUP(config_base);
1407         unsigned int code = EVENT_CODE(config_base);
1408         unsigned int group_shift;
1409         bool venum_event = EVENT_VENUM(config_base);
1410
1411         group_shift = group * 8;
1412         mask = 0xff << group_shift;
1413
1414         /* Configure evtsel for the region and group */
1415         if (venum_event)
1416                 val = KRAIT_VPMRESR0_GROUP0;
1417         else
1418                 val = krait_get_pmresrn_event(region);
1419         val += group;
1420         /* Mix in mode-exclusion bits */
1421         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1422         armv7_pmnc_write_evtsel(idx, val);
1423
1424         if (venum_event) {
1425                 venum_pre_pmresr(&vval, &fval);
1426                 val = venum_read_pmresr();
1427                 val &= ~mask;
1428                 val |= code << group_shift;
1429                 val |= PMRESRn_EN;
1430                 venum_write_pmresr(val);
1431                 venum_post_pmresr(vval, fval);
1432         } else {
1433                 val = krait_read_pmresrn(region);
1434                 val &= ~mask;
1435                 val |= code << group_shift;
1436                 val |= PMRESRn_EN;
1437                 krait_write_pmresrn(region, val);
1438         }
1439 }
1440
1441 static u32 clear_pmresrn_group(u32 val, int group)
1442 {
1443         u32 mask;
1444         int group_shift;
1445
1446         group_shift = group * 8;
1447         mask = 0xff << group_shift;
1448         val &= ~mask;
1449
1450         /* Don't clear enable bit if entire region isn't disabled */
1451         if (val & ~PMRESRn_EN)
1452                 return val |= PMRESRn_EN;
1453
1454         return 0;
1455 }
1456
1457 static void krait_clearpmu(u32 config_base)
1458 {
1459         u32 val;
1460         u32 vval, fval;
1461         unsigned int region = EVENT_REGION(config_base);
1462         unsigned int group = EVENT_GROUP(config_base);
1463         bool venum_event = EVENT_VENUM(config_base);
1464
1465         if (venum_event) {
1466                 venum_pre_pmresr(&vval, &fval);
1467                 val = venum_read_pmresr();
1468                 val = clear_pmresrn_group(val, group);
1469                 venum_write_pmresr(val);
1470                 venum_post_pmresr(vval, fval);
1471         } else {
1472                 val = krait_read_pmresrn(region);
1473                 val = clear_pmresrn_group(val, group);
1474                 krait_write_pmresrn(region, val);
1475         }
1476 }
1477
1478 static void krait_pmu_disable_event(struct perf_event *event)
1479 {
1480         unsigned long flags;
1481         struct hw_perf_event *hwc = &event->hw;
1482         int idx = hwc->idx;
1483         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1484         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1485
1486         /* Disable counter and interrupt */
1487         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1488
1489         /* Disable counter */
1490         armv7_pmnc_disable_counter(idx);
1491
1492         /*
1493          * Clear pmresr code (if destined for PMNx counters)
1494          */
1495         if (hwc->config_base & KRAIT_EVENT_MASK)
1496                 krait_clearpmu(hwc->config_base);
1497
1498         /* Disable interrupt for this counter */
1499         armv7_pmnc_disable_intens(idx);
1500
1501         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1502 }
1503
1504 static void krait_pmu_enable_event(struct perf_event *event)
1505 {
1506         unsigned long flags;
1507         struct hw_perf_event *hwc = &event->hw;
1508         int idx = hwc->idx;
1509         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1510         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1511
1512         /*
1513          * Enable counter and interrupt, and set the counter to count
1514          * the event that we're interested in.
1515          */
1516         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1517
1518         /* Disable counter */
1519         armv7_pmnc_disable_counter(idx);
1520
1521         /*
1522          * Set event (if destined for PMNx counters)
1523          * We set the event for the cycle counter because we
1524          * have the ability to perform event filtering.
1525          */
1526         if (hwc->config_base & KRAIT_EVENT_MASK)
1527                 krait_evt_setup(idx, hwc->config_base);
1528         else
1529                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1530
1531         /* Enable interrupt for this counter */
1532         armv7_pmnc_enable_intens(idx);
1533
1534         /* Enable counter */
1535         armv7_pmnc_enable_counter(idx);
1536
1537         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1538 }
1539
1540 static void krait_pmu_reset(void *info)
1541 {
1542         u32 vval, fval;
1543         struct arm_pmu *cpu_pmu = info;
1544         u32 idx, nb_cnt = cpu_pmu->num_events;
1545
1546         armv7pmu_reset(info);
1547
1548         /* Clear all pmresrs */
1549         krait_write_pmresrn(0, 0);
1550         krait_write_pmresrn(1, 0);
1551         krait_write_pmresrn(2, 0);
1552
1553         venum_pre_pmresr(&vval, &fval);
1554         venum_write_pmresr(0);
1555         venum_post_pmresr(vval, fval);
1556
1557         /* Reset PMxEVNCTCR to sane default */
1558         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1559                 armv7_pmnc_select_counter(idx);
1560                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1561         }
1562
1563 }
1564
1565 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1566                               unsigned int group)
1567 {
1568         int bit;
1569         struct hw_perf_event *hwc = &event->hw;
1570         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1571
1572         if (hwc->config_base & VENUM_EVENT)
1573                 bit = KRAIT_VPMRESR0_GROUP0;
1574         else
1575                 bit = krait_get_pmresrn_event(region);
1576         bit -= krait_get_pmresrn_event(0);
1577         bit += group;
1578         /*
1579          * Lower bits are reserved for use by the counters (see
1580          * armv7pmu_get_event_idx() for more info)
1581          */
1582         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1583
1584         return bit;
1585 }
1586
1587 /*
1588  * We check for column exclusion constraints here.
1589  * Two events cant use the same group within a pmresr register.
1590  */
1591 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1592                                    struct perf_event *event)
1593 {
1594         int idx;
1595         int bit = -1;
1596         struct hw_perf_event *hwc = &event->hw;
1597         unsigned int region = EVENT_REGION(hwc->config_base);
1598         unsigned int code = EVENT_CODE(hwc->config_base);
1599         unsigned int group = EVENT_GROUP(hwc->config_base);
1600         bool venum_event = EVENT_VENUM(hwc->config_base);
1601         bool krait_event = EVENT_CPU(hwc->config_base);
1602
1603         if (venum_event || krait_event) {
1604                 /* Ignore invalid events */
1605                 if (group > 3 || region > 2)
1606                         return -EINVAL;
1607                 if (venum_event && (code & 0xe0))
1608                         return -EINVAL;
1609
1610                 bit = krait_event_to_bit(event, region, group);
1611                 if (test_and_set_bit(bit, cpuc->used_mask))
1612                         return -EAGAIN;
1613         }
1614
1615         idx = armv7pmu_get_event_idx(cpuc, event);
1616         if (idx < 0 && bit >= 0)
1617                 clear_bit(bit, cpuc->used_mask);
1618
1619         return idx;
1620 }
1621
1622 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1623                                       struct perf_event *event)
1624 {
1625         int bit;
1626         struct hw_perf_event *hwc = &event->hw;
1627         unsigned int region = EVENT_REGION(hwc->config_base);
1628         unsigned int group = EVENT_GROUP(hwc->config_base);
1629         bool venum_event = EVENT_VENUM(hwc->config_base);
1630         bool krait_event = EVENT_CPU(hwc->config_base);
1631
1632         if (venum_event || krait_event) {
1633                 bit = krait_event_to_bit(event, region, group);
1634                 clear_bit(bit, cpuc->used_mask);
1635         }
1636 }
1637
1638 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1639 {
1640         armv7pmu_init(cpu_pmu);
1641         cpu_pmu->name           = "armv7_krait";
1642         /* Some early versions of Krait don't support PC write events */
1643         if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1644                                   "qcom,no-pc-write"))
1645                 cpu_pmu->map_event = krait_map_event_no_branch;
1646         else
1647                 cpu_pmu->map_event = krait_map_event;
1648         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1649         cpu_pmu->reset          = krait_pmu_reset;
1650         cpu_pmu->enable         = krait_pmu_enable_event;
1651         cpu_pmu->disable        = krait_pmu_disable_event;
1652         cpu_pmu->get_event_idx  = krait_pmu_get_event_idx;
1653         cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1654         return armv7_probe_num_events(cpu_pmu);
1655 }
1656
1657 /*
1658  * Scorpion Local Performance Monitor Register (LPMn)
1659  *
1660  *            31   30     24     16     8      0
1661  *            +--------------------------------+
1662  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1663  *            +--------------------------------+
1664  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1665  *            +--------------------------------+
1666  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1667  *            +--------------------------------+
1668  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1669  *            +--------------------------------+
1670  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1671  *            +--------------------------------+
1672  *              EN | G=3  | G=2  | G=1  | G=0
1673  *
1674  *
1675  *  Event Encoding:
1676  *
1677  *      hwc->config_base = 0xNRCCG
1678  *
1679  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1680  *      R  = region register
1681  *      CC = class of events the group G is choosing from
1682  *      G  = group or particular event
1683  *
1684  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1685  *
1686  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1687  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1688  *  events (interrupts for example). An event code is broken down into
1689  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1690  *  example).
1691  */
1692
1693 static u32 scorpion_read_pmresrn(int n)
1694 {
1695         u32 val;
1696
1697         switch (n) {
1698         case 0:
1699                 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1700                 break;
1701         case 1:
1702                 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1703                 break;
1704         case 2:
1705                 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1706                 break;
1707         case 3:
1708                 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1709                 break;
1710         default:
1711                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1712         }
1713
1714         return val;
1715 }
1716
1717 static void scorpion_write_pmresrn(int n, u32 val)
1718 {
1719         switch (n) {
1720         case 0:
1721                 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1722                 break;
1723         case 1:
1724                 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1725                 break;
1726         case 2:
1727                 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1728                 break;
1729         case 3:
1730                 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1731                 break;
1732         default:
1733                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1734         }
1735 }
1736
1737 static u32 scorpion_get_pmresrn_event(unsigned int region)
1738 {
1739         static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1740                                              SCORPION_LPM1_GROUP0,
1741                                              SCORPION_LPM2_GROUP0,
1742                                              SCORPION_L2LPM_GROUP0 };
1743         return pmresrn_table[region];
1744 }
1745
1746 static void scorpion_evt_setup(int idx, u32 config_base)
1747 {
1748         u32 val;
1749         u32 mask;
1750         u32 vval, fval;
1751         unsigned int region = EVENT_REGION(config_base);
1752         unsigned int group = EVENT_GROUP(config_base);
1753         unsigned int code = EVENT_CODE(config_base);
1754         unsigned int group_shift;
1755         bool venum_event = EVENT_VENUM(config_base);
1756
1757         group_shift = group * 8;
1758         mask = 0xff << group_shift;
1759
1760         /* Configure evtsel for the region and group */
1761         if (venum_event)
1762                 val = SCORPION_VLPM_GROUP0;
1763         else
1764                 val = scorpion_get_pmresrn_event(region);
1765         val += group;
1766         /* Mix in mode-exclusion bits */
1767         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1768         armv7_pmnc_write_evtsel(idx, val);
1769
1770         asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1771
1772         if (venum_event) {
1773                 venum_pre_pmresr(&vval, &fval);
1774                 val = venum_read_pmresr();
1775                 val &= ~mask;
1776                 val |= code << group_shift;
1777                 val |= PMRESRn_EN;
1778                 venum_write_pmresr(val);
1779                 venum_post_pmresr(vval, fval);
1780         } else {
1781                 val = scorpion_read_pmresrn(region);
1782                 val &= ~mask;
1783                 val |= code << group_shift;
1784                 val |= PMRESRn_EN;
1785                 scorpion_write_pmresrn(region, val);
1786         }
1787 }
1788
1789 static void scorpion_clearpmu(u32 config_base)
1790 {
1791         u32 val;
1792         u32 vval, fval;
1793         unsigned int region = EVENT_REGION(config_base);
1794         unsigned int group = EVENT_GROUP(config_base);
1795         bool venum_event = EVENT_VENUM(config_base);
1796
1797         if (venum_event) {
1798                 venum_pre_pmresr(&vval, &fval);
1799                 val = venum_read_pmresr();
1800                 val = clear_pmresrn_group(val, group);
1801                 venum_write_pmresr(val);
1802                 venum_post_pmresr(vval, fval);
1803         } else {
1804                 val = scorpion_read_pmresrn(region);
1805                 val = clear_pmresrn_group(val, group);
1806                 scorpion_write_pmresrn(region, val);
1807         }
1808 }
1809
1810 static void scorpion_pmu_disable_event(struct perf_event *event)
1811 {
1812         unsigned long flags;
1813         struct hw_perf_event *hwc = &event->hw;
1814         int idx = hwc->idx;
1815         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1816         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1817
1818         /* Disable counter and interrupt */
1819         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1820
1821         /* Disable counter */
1822         armv7_pmnc_disable_counter(idx);
1823
1824         /*
1825          * Clear pmresr code (if destined for PMNx counters)
1826          */
1827         if (hwc->config_base & KRAIT_EVENT_MASK)
1828                 scorpion_clearpmu(hwc->config_base);
1829
1830         /* Disable interrupt for this counter */
1831         armv7_pmnc_disable_intens(idx);
1832
1833         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1834 }
1835
1836 static void scorpion_pmu_enable_event(struct perf_event *event)
1837 {
1838         unsigned long flags;
1839         struct hw_perf_event *hwc = &event->hw;
1840         int idx = hwc->idx;
1841         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1842         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1843
1844         /*
1845          * Enable counter and interrupt, and set the counter to count
1846          * the event that we're interested in.
1847          */
1848         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1849
1850         /* Disable counter */
1851         armv7_pmnc_disable_counter(idx);
1852
1853         /*
1854          * Set event (if destined for PMNx counters)
1855          * We don't set the event for the cycle counter because we
1856          * don't have the ability to perform event filtering.
1857          */
1858         if (hwc->config_base & KRAIT_EVENT_MASK)
1859                 scorpion_evt_setup(idx, hwc->config_base);
1860         else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1861                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1862
1863         /* Enable interrupt for this counter */
1864         armv7_pmnc_enable_intens(idx);
1865
1866         /* Enable counter */
1867         armv7_pmnc_enable_counter(idx);
1868
1869         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1870 }
1871
1872 static void scorpion_pmu_reset(void *info)
1873 {
1874         u32 vval, fval;
1875         struct arm_pmu *cpu_pmu = info;
1876         u32 idx, nb_cnt = cpu_pmu->num_events;
1877
1878         armv7pmu_reset(info);
1879
1880         /* Clear all pmresrs */
1881         scorpion_write_pmresrn(0, 0);
1882         scorpion_write_pmresrn(1, 0);
1883         scorpion_write_pmresrn(2, 0);
1884         scorpion_write_pmresrn(3, 0);
1885
1886         venum_pre_pmresr(&vval, &fval);
1887         venum_write_pmresr(0);
1888         venum_post_pmresr(vval, fval);
1889
1890         /* Reset PMxEVNCTCR to sane default */
1891         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1892                 armv7_pmnc_select_counter(idx);
1893                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1894         }
1895 }
1896
1897 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1898                               unsigned int group)
1899 {
1900         int bit;
1901         struct hw_perf_event *hwc = &event->hw;
1902         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1903
1904         if (hwc->config_base & VENUM_EVENT)
1905                 bit = SCORPION_VLPM_GROUP0;
1906         else
1907                 bit = scorpion_get_pmresrn_event(region);
1908         bit -= scorpion_get_pmresrn_event(0);
1909         bit += group;
1910         /*
1911          * Lower bits are reserved for use by the counters (see
1912          * armv7pmu_get_event_idx() for more info)
1913          */
1914         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1915
1916         return bit;
1917 }
1918
1919 /*
1920  * We check for column exclusion constraints here.
1921  * Two events cant use the same group within a pmresr register.
1922  */
1923 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1924                                    struct perf_event *event)
1925 {
1926         int idx;
1927         int bit = -1;
1928         struct hw_perf_event *hwc = &event->hw;
1929         unsigned int region = EVENT_REGION(hwc->config_base);
1930         unsigned int group = EVENT_GROUP(hwc->config_base);
1931         bool venum_event = EVENT_VENUM(hwc->config_base);
1932         bool scorpion_event = EVENT_CPU(hwc->config_base);
1933
1934         if (venum_event || scorpion_event) {
1935                 /* Ignore invalid events */
1936                 if (group > 3 || region > 3)
1937                         return -EINVAL;
1938
1939                 bit = scorpion_event_to_bit(event, region, group);
1940                 if (test_and_set_bit(bit, cpuc->used_mask))
1941                         return -EAGAIN;
1942         }
1943
1944         idx = armv7pmu_get_event_idx(cpuc, event);
1945         if (idx < 0 && bit >= 0)
1946                 clear_bit(bit, cpuc->used_mask);
1947
1948         return idx;
1949 }
1950
1951 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1952                                       struct perf_event *event)
1953 {
1954         int bit;
1955         struct hw_perf_event *hwc = &event->hw;
1956         unsigned int region = EVENT_REGION(hwc->config_base);
1957         unsigned int group = EVENT_GROUP(hwc->config_base);
1958         bool venum_event = EVENT_VENUM(hwc->config_base);
1959         bool scorpion_event = EVENT_CPU(hwc->config_base);
1960
1961         if (venum_event || scorpion_event) {
1962                 bit = scorpion_event_to_bit(event, region, group);
1963                 clear_bit(bit, cpuc->used_mask);
1964         }
1965 }
1966
1967 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1968 {
1969         armv7pmu_init(cpu_pmu);
1970         cpu_pmu->name           = "armv7_scorpion";
1971         cpu_pmu->map_event      = scorpion_map_event;
1972         cpu_pmu->reset          = scorpion_pmu_reset;
1973         cpu_pmu->enable         = scorpion_pmu_enable_event;
1974         cpu_pmu->disable        = scorpion_pmu_disable_event;
1975         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1976         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1977         return armv7_probe_num_events(cpu_pmu);
1978 }
1979
1980 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1981 {
1982         armv7pmu_init(cpu_pmu);
1983         cpu_pmu->name           = "armv7_scorpion_mp";
1984         cpu_pmu->map_event      = scorpion_map_event;
1985         cpu_pmu->reset          = scorpion_pmu_reset;
1986         cpu_pmu->enable         = scorpion_pmu_enable_event;
1987         cpu_pmu->disable        = scorpion_pmu_disable_event;
1988         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1989         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1990         return armv7_probe_num_events(cpu_pmu);
1991 }
1992
1993 static const struct of_device_id armv7_pmu_of_device_ids[] = {
1994         {.compatible = "arm,cortex-a17-pmu",    .data = armv7_a17_pmu_init},
1995         {.compatible = "arm,cortex-a15-pmu",    .data = armv7_a15_pmu_init},
1996         {.compatible = "arm,cortex-a12-pmu",    .data = armv7_a12_pmu_init},
1997         {.compatible = "arm,cortex-a9-pmu",     .data = armv7_a9_pmu_init},
1998         {.compatible = "arm,cortex-a8-pmu",     .data = armv7_a8_pmu_init},
1999         {.compatible = "arm,cortex-a7-pmu",     .data = armv7_a7_pmu_init},
2000         {.compatible = "arm,cortex-a5-pmu",     .data = armv7_a5_pmu_init},
2001         {.compatible = "qcom,krait-pmu",        .data = krait_pmu_init},
2002         {.compatible = "qcom,scorpion-pmu",     .data = scorpion_pmu_init},
2003         {.compatible = "qcom,scorpion-mp-pmu",  .data = scorpion_mp_pmu_init},
2004         {},
2005 };
2006
2007 static const struct pmu_probe_info armv7_pmu_probe_table[] = {
2008         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
2009         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
2010         { /* sentinel value */ }
2011 };
2012
2013
2014 static int armv7_pmu_device_probe(struct platform_device *pdev)
2015 {
2016         return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
2017                                     armv7_pmu_probe_table);
2018 }
2019
2020 static struct platform_driver armv7_pmu_driver = {
2021         .driver         = {
2022                 .name   = "armv7-pmu",
2023                 .of_match_table = armv7_pmu_of_device_ids,
2024         },
2025         .probe          = armv7_pmu_device_probe,
2026 };
2027
2028 static int __init register_armv7_pmu_driver(void)
2029 {
2030         return platform_driver_register(&armv7_pmu_driver);
2031 }
2032 device_initcall(register_armv7_pmu_driver);
2033 #endif  /* CONFIG_CPU_V7 */