arch/arm64/kernel/perf_event.c

   1 /*
   2  * PMU support
   3  *
   4  * Copyright (C) 2012 ARM Limited
   5  * Author: Will Deacon <will.deacon@arm.com>
   6  *
   7  * This code is based heavily on the ARMv7 perf event code.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 #include <asm/irq_regs.h>
  23 #include <asm/perf_event.h>
  24 #include <asm/virt.h>
  25
  26 #include <linux/of.h>
  27 #include <linux/perf/arm_pmu.h>
  28 #include <linux/platform_device.h>
  29
  30 /*
  31  * ARMv8 PMUv3 Performance Events handling code.
  32  * Common event types.
  33  */
  34
  35 /* Required events. */
  36 #define ARMV8_PMUV3_PERFCTR_PMNC_SW_INCR                        0x00
  37 #define ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL                    0x03
  38 #define ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS                    0x04
  39 #define ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED                  0x10
  40 #define ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES                        0x11
  41 #define ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED                      0x12
  42
  43 /* At least one of the following is required. */
  44 #define ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED                      0x08
  45 #define ARMV8_PMUV3_PERFCTR_OP_SPEC                             0x1B
  46
  47 /* Common architectural events. */
  48 #define ARMV8_PMUV3_PERFCTR_MEM_READ                            0x06
  49 #define ARMV8_PMUV3_PERFCTR_MEM_WRITE                           0x07
  50 #define ARMV8_PMUV3_PERFCTR_EXC_TAKEN                           0x09
  51 #define ARMV8_PMUV3_PERFCTR_EXC_EXECUTED                        0x0A
  52 #define ARMV8_PMUV3_PERFCTR_CID_WRITE                           0x0B
  53 #define ARMV8_PMUV3_PERFCTR_PC_WRITE                            0x0C
  54 #define ARMV8_PMUV3_PERFCTR_PC_IMM_BRANCH                       0x0D
  55 #define ARMV8_PMUV3_PERFCTR_PC_PROC_RETURN                      0x0E
  56 #define ARMV8_PMUV3_PERFCTR_MEM_UNALIGNED_ACCESS                0x0F
  57 #define ARMV8_PMUV3_PERFCTR_TTBR_WRITE                          0x1C
  58 #define ARMV8_PMUV3_PERFCTR_CHAIN                               0x1E
  59 #define ARMV8_PMUV3_PERFCTR_BR_RETIRED                          0x21
  60
  61 /* Common microarchitectural events. */
  62 #define ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL                    0x01
  63 #define ARMV8_PMUV3_PERFCTR_ITLB_REFILL                         0x02
  64 #define ARMV8_PMUV3_PERFCTR_DTLB_REFILL                         0x05
  65 #define ARMV8_PMUV3_PERFCTR_MEM_ACCESS                          0x13
  66 #define ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS                    0x14
  67 #define ARMV8_PMUV3_PERFCTR_L1_DCACHE_WB                        0x15
  68 #define ARMV8_PMUV3_PERFCTR_L2_CACHE_ACCESS                     0x16
  69 #define ARMV8_PMUV3_PERFCTR_L2_CACHE_REFILL                     0x17
  70 #define ARMV8_PMUV3_PERFCTR_L2_CACHE_WB                         0x18
  71 #define ARMV8_PMUV3_PERFCTR_BUS_ACCESS                          0x19
  72 #define ARMV8_PMUV3_PERFCTR_MEM_ERROR                           0x1A
  73 #define ARMV8_PMUV3_PERFCTR_BUS_CYCLES                          0x1D
  74 #define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE                  0x1F
  75 #define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE                  0x20
  76 #define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED                 0x22
  77 #define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND                      0x23
  78 #define ARMV8_PMUV3_PERFCTR_STALL_BACKEND                       0x24
  79 #define ARMV8_PMUV3_PERFCTR_L1D_TLB                             0x25
  80 #define ARMV8_PMUV3_PERFCTR_L1I_TLB                             0x26
  81 #define ARMV8_PMUV3_PERFCTR_L2I_CACHE                           0x27
  82 #define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL                    0x28
  83 #define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE                  0x29
  84 #define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL                    0x2A
  85 #define ARMV8_PMUV3_PERFCTR_L3D_CACHE                           0x2B
  86 #define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB                        0x2C
  87 #define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL                      0x2D
  88 #define ARMV8_PMUV3_PERFCTR_L21_TLB_REFILL                      0x2E
  89 #define ARMV8_PMUV3_PERFCTR_L2D_TLB                             0x2F
  90 #define ARMV8_PMUV3_PERFCTR_L21_TLB                             0x30
  91
  92 /* ARMv8 implementation defined event types. */
  93 #define ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_LD                0x40
  94 #define ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_ST                0x41
  95 #define ARMV8_IMPDEF_PERFCTR_L1_DCACHE_REFILL_LD                0x42
  96 #define ARMV8_IMPDEF_PERFCTR_L1_DCACHE_REFILL_ST                0x43
  97 #define ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_LD                     0x4C
  98 #define ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_ST                     0x4D
  99 #define ARMV8_IMPDEF_PERFCTR_DTLB_ACCESS_LD                     0x4E
 100 #define ARMV8_IMPDEF_PERFCTR_DTLB_ACCESS_ST                     0x4F
 101
 102 /* ARMv8 Cortex-A53 specific event types. */
 103 #define ARMV8_A53_PERFCTR_PREFETCH_LINEFILL                     0xC2
 104
 105 /* ARMv8 Cavium ThunderX specific event types. */
 106 #define ARMV8_THUNDER_PERFCTR_L1_DCACHE_MISS_ST                 0xE9
 107 #define ARMV8_THUNDER_PERFCTR_L1_DCACHE_PREF_ACCESS             0xEA
 108 #define ARMV8_THUNDER_PERFCTR_L1_DCACHE_PREF_MISS               0xEB
 109 #define ARMV8_THUNDER_PERFCTR_L1_ICACHE_PREF_ACCESS             0xEC
 110 #define ARMV8_THUNDER_PERFCTR_L1_ICACHE_PREF_MISS               0xED
 111
 112 /* PMUv3 HW events mapping. */
 113 static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
 114         PERF_MAP_ALL_UNSUPPORTED,
 115         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 116         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 117         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 118         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 119         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 120 };
 121
 122 /* ARM Cortex-A53 HW events mapping. */
 123 static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = {
 124         PERF_MAP_ALL_UNSUPPORTED,
 125         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 126         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 127         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 128         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 129         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV8_PMUV3_PERFCTR_PC_WRITE,
 130         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 131         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
 132 };
 133
 134 /* ARM Cortex-A57 and Cortex-A72 events mapping. */
 135 static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = {
 136         PERF_MAP_ALL_UNSUPPORTED,
 137         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 138         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 139         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 140         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 141         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 142         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
 143 };
 144
 145 static const unsigned armv8_thunder_perf_map[PERF_COUNT_HW_MAX] = {
 146         PERF_MAP_ALL_UNSUPPORTED,
 147         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 148         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 149         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 150         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 151         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV8_PMUV3_PERFCTR_PC_WRITE,
 152         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 153         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
 154         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
 155 };
 156
 157 static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 158                                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 159                                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 160         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 161
 162         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 163         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 164         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 165         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 166
 167         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 168         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 169         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 170         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 171 };
 172
 173 static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 174                                               [PERF_COUNT_HW_CACHE_OP_MAX]
 175                                               [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 176         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 177
 178         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 179         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 180         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 181         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 182         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREFETCH_LINEFILL,
 183
 184         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS,
 185         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL,
 186
 187         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_ITLB_REFILL,
 188
 189         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 190         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 191         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 192         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 193 };
 194
 195 static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 196                                               [PERF_COUNT_HW_CACHE_OP_MAX]
 197                                               [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 198         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 199
 200         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_LD,
 201         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_REFILL_LD,
 202         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_ST,
 203         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_REFILL_ST,
 204
 205         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS,
 206         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL,
 207
 208         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_LD,
 209         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_ST,
 210
 211         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_ITLB_REFILL,
 212
 213         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 214         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 215         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 216         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 217 };
 218
 219 static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 220                                                    [PERF_COUNT_HW_CACHE_OP_MAX]
 221                                                    [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 222         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 223
 224         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_LD,
 225         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_REFILL_LD,
 226         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1_DCACHE_ACCESS_ST,
 227         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_THUNDER_PERFCTR_L1_DCACHE_MISS_ST,
 228         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1_DCACHE_PREF_ACCESS,
 229         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1_DCACHE_PREF_MISS,
 230
 231         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS,
 232         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL,
 233         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1_ICACHE_PREF_ACCESS,
 234         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1_ICACHE_PREF_MISS,
 235
 236         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_DTLB_ACCESS_LD,
 237         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_LD,
 238         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_DTLB_ACCESS_ST,
 239         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV8_IMPDEF_PERFCTR_DTLB_REFILL_ST,
 240
 241         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_ITLB_REFILL,
 242
 243         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 244         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 245         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 246         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 247 };
 248
 249 #define ARMV8_EVENT_ATTR_RESOLVE(m) #m
 250 #define ARMV8_EVENT_ATTR(name, config) \
 251         PMU_EVENT_ATTR_STRING(name, armv8_event_attr_##name, \
 252                               "event=" ARMV8_EVENT_ATTR_RESOLVE(config))
 253
 254 ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_PMNC_SW_INCR);
 255 ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL);
 256 ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_ITLB_REFILL);
 257 ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL);
 258 ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS);
 259 ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_DTLB_REFILL);
 260 ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_MEM_READ);
 261 ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_MEM_WRITE);
 262 ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED);
 263 ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN);
 264 ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_EXECUTED);
 265 ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE);
 266 ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE);
 267 ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_PC_IMM_BRANCH);
 268 ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_PC_PROC_RETURN);
 269 ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_MEM_UNALIGNED_ACCESS);
 270 ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED);
 271 ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES);
 272 ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED);
 273 ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS);
 274 ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS);
 275 ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1_DCACHE_WB);
 276 ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2_CACHE_ACCESS);
 277 ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2_CACHE_REFILL);
 278 ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2_CACHE_WB);
 279 ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS);
 280 ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEM_ERROR);
 281 ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_OP_SPEC);
 282 ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE);
 283 ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES);
 284 ARMV8_EVENT_ATTR(chain, ARMV8_PMUV3_PERFCTR_CHAIN);
 285 ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE);
 286 ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE);
 287 ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED);
 288 ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED);
 289 ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND);
 290 ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND);
 291 ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB);
 292 ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB);
 293 ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE);
 294 ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL);
 295 ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE);
 296 ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL);
 297 ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE);
 298 ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB);
 299 ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL);
 300 ARMV8_EVENT_ATTR(l21_tlb_refill, ARMV8_PMUV3_PERFCTR_L21_TLB_REFILL);
 301 ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB);
 302 ARMV8_EVENT_ATTR(l21_tlb, ARMV8_PMUV3_PERFCTR_L21_TLB);
 303
 304 static struct attribute *armv8_pmuv3_event_attrs[] = {
 305         &armv8_event_attr_sw_incr.attr.attr,
 306         &armv8_event_attr_l1i_cache_refill.attr.attr,
 307         &armv8_event_attr_l1i_tlb_refill.attr.attr,
 308         &armv8_event_attr_l1d_cache_refill.attr.attr,
 309         &armv8_event_attr_l1d_cache.attr.attr,
 310         &armv8_event_attr_l1d_tlb_refill.attr.attr,
 311         &armv8_event_attr_ld_retired.attr.attr,
 312         &armv8_event_attr_st_retired.attr.attr,
 313         &armv8_event_attr_inst_retired.attr.attr,
 314         &armv8_event_attr_exc_taken.attr.attr,
 315         &armv8_event_attr_exc_return.attr.attr,
 316         &armv8_event_attr_cid_write_retired.attr.attr,
 317         &armv8_event_attr_pc_write_retired.attr.attr,
 318         &armv8_event_attr_br_immed_retired.attr.attr,
 319         &armv8_event_attr_br_return_retired.attr.attr,
 320         &armv8_event_attr_unaligned_ldst_retired.attr.attr,
 321         &armv8_event_attr_br_mis_pred.attr.attr,
 322         &armv8_event_attr_cpu_cycles.attr.attr,
 323         &armv8_event_attr_br_pred.attr.attr,
 324         &armv8_event_attr_mem_access.attr.attr,
 325         &armv8_event_attr_l1i_cache.attr.attr,
 326         &armv8_event_attr_l1d_cache_wb.attr.attr,
 327         &armv8_event_attr_l2d_cache.attr.attr,
 328         &armv8_event_attr_l2d_cache_refill.attr.attr,
 329         &armv8_event_attr_l2d_cache_wb.attr.attr,
 330         &armv8_event_attr_bus_access.attr.attr,
 331         &armv8_event_attr_memory_error.attr.attr,
 332         &armv8_event_attr_inst_spec.attr.attr,
 333         &armv8_event_attr_ttbr_write_retired.attr.attr,
 334         &armv8_event_attr_bus_cycles.attr.attr,
 335         &armv8_event_attr_chain.attr.attr,
 336         &armv8_event_attr_l1d_cache_allocate.attr.attr,
 337         &armv8_event_attr_l2d_cache_allocate.attr.attr,
 338         &armv8_event_attr_br_retired.attr.attr,
 339         &armv8_event_attr_br_mis_pred_retired.attr.attr,
 340         &armv8_event_attr_stall_frontend.attr.attr,
 341         &armv8_event_attr_stall_backend.attr.attr,
 342         &armv8_event_attr_l1d_tlb.attr.attr,
 343         &armv8_event_attr_l1i_tlb.attr.attr,
 344         &armv8_event_attr_l2i_cache.attr.attr,
 345         &armv8_event_attr_l2i_cache_refill.attr.attr,
 346         &armv8_event_attr_l3d_cache_allocate.attr.attr,
 347         &armv8_event_attr_l3d_cache_refill.attr.attr,
 348         &armv8_event_attr_l3d_cache.attr.attr,
 349         &armv8_event_attr_l3d_cache_wb.attr.attr,
 350         &armv8_event_attr_l2d_tlb_refill.attr.attr,
 351         &armv8_event_attr_l21_tlb_refill.attr.attr,
 352         &armv8_event_attr_l2d_tlb.attr.attr,
 353         &armv8_event_attr_l21_tlb.attr.attr,
 354         NULL,
 355 };
 356
 357 static struct attribute_group armv8_pmuv3_events_attr_group = {
 358         .name = "events",
 359         .attrs = armv8_pmuv3_event_attrs,
 360 };
 361
 362 PMU_FORMAT_ATTR(event, "config:0-9");
 363
 364 static struct attribute *armv8_pmuv3_format_attrs[] = {
 365         &format_attr_event.attr,
 366         NULL,
 367 };
 368
 369 static struct attribute_group armv8_pmuv3_format_attr_group = {
 370         .name = "format",
 371         .attrs = armv8_pmuv3_format_attrs,
 372 };
 373
 374 static const struct attribute_group *armv8_pmuv3_attr_groups[] = {
 375         &armv8_pmuv3_events_attr_group,
 376         &armv8_pmuv3_format_attr_group,
 377         NULL,
 378 };
 379
 380 /*
 381  * Perf Events' indices
 382  */
 383 #define ARMV8_IDX_CYCLE_COUNTER 0
 384 #define ARMV8_IDX_COUNTER0      1
 385 #define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
 386         (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 387
 388 /*
 389  * ARMv8 low level PMU access
 390  */
 391
 392 /*
 393  * Perf Event to low level counters mapping
 394  */
 395 #define ARMV8_IDX_TO_COUNTER(x) \
 396         (((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
 397
 398 static inline u32 armv8pmu_pmcr_read(void)
 399 {
 400         u32 val;
 401         asm volatile("mrs %0, pmcr_el0" : "=r" (val));
 402         return val;
 403 }
 404
 405 static inline void armv8pmu_pmcr_write(u32 val)
 406 {
 407         val &= ARMV8_PMU_PMCR_MASK;
 408         isb();
 409         asm volatile("msr pmcr_el0, %0" :: "r" (val));
 410 }
 411
 412 static inline int armv8pmu_has_overflowed(u32 pmovsr)
 413 {
 414         return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
 415 }
 416
 417 static inline int armv8pmu_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 418 {
 419         return idx >= ARMV8_IDX_CYCLE_COUNTER &&
 420                 idx <= ARMV8_IDX_COUNTER_LAST(cpu_pmu);
 421 }
 422
 423 static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
 424 {
 425         return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
 426 }
 427
 428 static inline int armv8pmu_select_counter(int idx)
 429 {
 430         u32 counter = ARMV8_IDX_TO_COUNTER(idx);
 431         asm volatile("msr pmselr_el0, %0" :: "r" (counter));
 432         isb();
 433
 434         return idx;
 435 }
 436
 437 static inline u32 armv8pmu_read_counter(struct perf_event *event)
 438 {
 439         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 440         struct hw_perf_event *hwc = &event->hw;
 441         int idx = hwc->idx;
 442         u32 value = 0;
 443
 444         if (!armv8pmu_counter_valid(cpu_pmu, idx))
 445                 pr_err("CPU%u reading wrong counter %d\n",
 446                         smp_processor_id(), idx);
 447         else if (idx == ARMV8_IDX_CYCLE_COUNTER)
 448                 asm volatile("mrs %0, pmccntr_el0" : "=r" (value));
 449         else if (armv8pmu_select_counter(idx) == idx)
 450                 asm volatile("mrs %0, pmxevcntr_el0" : "=r" (value));
 451
 452         return value;
 453 }
 454
 455 static inline void armv8pmu_write_counter(struct perf_event *event, u32 value)
 456 {
 457         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 458         struct hw_perf_event *hwc = &event->hw;
 459         int idx = hwc->idx;
 460
 461         if (!armv8pmu_counter_valid(cpu_pmu, idx))
 462                 pr_err("CPU%u writing wrong counter %d\n",
 463                         smp_processor_id(), idx);
 464         else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
 465                 /*
 466                  * Set the upper 32bits as this is a 64bit counter but we only
 467                  * count using the lower 32bits and we want an interrupt when
 468                  * it overflows.
 469                  */
 470                 u64 value64 = 0xffffffff00000000ULL | value;
 471
 472                 asm volatile("msr pmccntr_el0, %0" :: "r" (value64));
 473         } else if (armv8pmu_select_counter(idx) == idx)
 474                 asm volatile("msr pmxevcntr_el0, %0" :: "r" (value));
 475 }
 476
 477 static inline void armv8pmu_write_evtype(int idx, u32 val)
 478 {
 479         if (armv8pmu_select_counter(idx) == idx) {
 480                 val &= ARMV8_PMU_EVTYPE_MASK;
 481                 asm volatile("msr pmxevtyper_el0, %0" :: "r" (val));
 482         }
 483 }
 484
 485 static inline int armv8pmu_enable_counter(int idx)
 486 {
 487         u32 counter = ARMV8_IDX_TO_COUNTER(idx);
 488         asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter)));
 489         return idx;
 490 }
 491
 492 static inline int armv8pmu_disable_counter(int idx)
 493 {
 494         u32 counter = ARMV8_IDX_TO_COUNTER(idx);
 495         asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter)));
 496         return idx;
 497 }
 498
 499 static inline int armv8pmu_enable_intens(int idx)
 500 {
 501         u32 counter = ARMV8_IDX_TO_COUNTER(idx);
 502         asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter)));
 503         return idx;
 504 }
 505
 506 static inline int armv8pmu_disable_intens(int idx)
 507 {
 508         u32 counter = ARMV8_IDX_TO_COUNTER(idx);
 509         asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter)));
 510         isb();
 511         /* Clear the overflow flag in case an interrupt is pending. */
 512         asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter)));
 513         isb();
 514
 515         return idx;
 516 }
 517
 518 static inline u32 armv8pmu_getreset_flags(void)
 519 {
 520         u32 value;
 521
 522         /* Read */
 523         asm volatile("mrs %0, pmovsclr_el0" : "=r" (value));
 524
 525         /* Write to clear flags */
 526         value &= ARMV8_PMU_OVSR_MASK;
 527         asm volatile("msr pmovsclr_el0, %0" :: "r" (value));
 528
 529         return value;
 530 }
 531
 532 static void armv8pmu_enable_event(struct perf_event *event)
 533 {
 534         unsigned long flags;
 535         struct hw_perf_event *hwc = &event->hw;
 536         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 537         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 538         int idx = hwc->idx;
 539
 540         /*
 541          * Enable counter and interrupt, and set the counter to count
 542          * the event that we're interested in.
 543          */
 544         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 545
 546         /*
 547          * Disable counter
 548          */
 549         armv8pmu_disable_counter(idx);
 550
 551         /*
 552          * Set event (if destined for PMNx counters).
 553          */
 554         armv8pmu_write_evtype(idx, hwc->config_base);
 555
 556         /*
 557          * Enable interrupt for this counter
 558          */
 559         armv8pmu_enable_intens(idx);
 560
 561         /*
 562          * Enable counter
 563          */
 564         armv8pmu_enable_counter(idx);
 565
 566         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 567 }
 568
 569 static void armv8pmu_disable_event(struct perf_event *event)
 570 {
 571         unsigned long flags;
 572         struct hw_perf_event *hwc = &event->hw;
 573         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 574         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 575         int idx = hwc->idx;
 576
 577         /*
 578          * Disable counter and interrupt
 579          */
 580         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 581
 582         /*
 583          * Disable counter
 584          */
 585         armv8pmu_disable_counter(idx);
 586
 587         /*
 588          * Disable interrupt for this counter
 589          */
 590         armv8pmu_disable_intens(idx);
 591
 592         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 593 }
 594
 595 static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
 596 {
 597         u32 pmovsr;
 598         struct perf_sample_data data;
 599         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 600         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 601         struct pt_regs *regs;
 602         int idx;
 603
 604         /*
 605          * Get and reset the IRQ flags
 606          */
 607         pmovsr = armv8pmu_getreset_flags();
 608
 609         /*
 610          * Did an overflow occur?
 611          */
 612         if (!armv8pmu_has_overflowed(pmovsr))
 613                 return IRQ_NONE;
 614
 615         /*
 616          * Handle the counter(s) overflow(s)
 617          */
 618         regs = get_irq_regs();
 619
 620         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 621                 struct perf_event *event = cpuc->events[idx];
 622                 struct hw_perf_event *hwc;
 623
 624                 /* Ignore if we don't have an event. */
 625                 if (!event)
 626                         continue;
 627
 628                 /*
 629                  * We have a single interrupt for all counters. Check that
 630                  * each counter has overflowed before we process it.
 631                  */
 632                 if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
 633                         continue;
 634
 635                 hwc = &event->hw;
 636                 armpmu_event_update(event);
 637                 perf_sample_data_init(&data, 0, hwc->last_period);
 638                 if (!armpmu_event_set_period(event))
 639                         continue;
 640
 641                 if (perf_event_overflow(event, &data, regs))
 642                         cpu_pmu->disable(event);
 643         }
 644
 645         /*
 646          * Handle the pending perf events.
 647          *
 648          * Note: this call *must* be run with interrupts disabled. For
 649          * platforms that can have the PMU interrupts raised as an NMI, this
 650          * will not work.
 651          */
 652         irq_work_run();
 653
 654         return IRQ_HANDLED;
 655 }
 656
 657 static void armv8pmu_start(struct arm_pmu *cpu_pmu)
 658 {
 659         unsigned long flags;
 660         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 661
 662         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 663         /* Enable all counters */
 664         armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
 665         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 666 }
 667
 668 static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
 669 {
 670         unsigned long flags;
 671         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 672
 673         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 674         /* Disable all counters */
 675         armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
 676         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 677 }
 678
 679 static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
 680                                   struct perf_event *event)
 681 {
 682         int idx;
 683         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 684         struct hw_perf_event *hwc = &event->hw;
 685         unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
 686
 687         /* Always place a cycle counter into the cycle counter. */
 688         if (evtype == ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES) {
 689                 if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
 690                         return -EAGAIN;
 691
 692                 return ARMV8_IDX_CYCLE_COUNTER;
 693         }
 694
 695         /*
 696          * For anything other than a cycle counter, try and use
 697          * the events counters
 698          */
 699         for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
 700                 if (!test_and_set_bit(idx, cpuc->used_mask))
 701                         return idx;
 702         }
 703
 704         /* The counters are all in use. */
 705         return -EAGAIN;
 706 }
 707
 708 /*
 709  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
 710  */
 711 static int armv8pmu_set_event_filter(struct hw_perf_event *event,
 712                                      struct perf_event_attr *attr)
 713 {
 714         unsigned long config_base = 0;
 715
 716         if (attr->exclude_idle)
 717                 return -EPERM;
 718         if (is_kernel_in_hyp_mode() &&
 719             attr->exclude_kernel != attr->exclude_hv)
 720                 return -EINVAL;
 721         if (attr->exclude_user)
 722                 config_base |= ARMV8_PMU_EXCLUDE_EL0;
 723         if (!is_kernel_in_hyp_mode() && attr->exclude_kernel)
 724                 config_base |= ARMV8_PMU_EXCLUDE_EL1;
 725         if (!attr->exclude_hv)
 726                 config_base |= ARMV8_PMU_INCLUDE_EL2;
 727
 728         /*
 729          * Install the filter into config_base as this is used to
 730          * construct the event type.
 731          */
 732         event->config_base = config_base;
 733
 734         return 0;
 735 }
 736
 737 static void armv8pmu_reset(void *info)
 738 {
 739         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
 740         u32 idx, nb_cnt = cpu_pmu->num_events;
 741
 742         /* The counter and interrupt enable registers are unknown at reset. */
 743         for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
 744                 armv8pmu_disable_counter(idx);
 745                 armv8pmu_disable_intens(idx);
 746         }
 747
 748         /*
 749          * Initialize & Reset PMNC. Request overflow interrupt for
 750          * 64 bit cycle counter but cheat in armv8pmu_write_counter().
 751          */
 752         armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C |
 753                             ARMV8_PMU_PMCR_LC);
 754 }
 755
 756 static int armv8_pmuv3_map_event(struct perf_event *event)
 757 {
 758         return armpmu_map_event(event, &armv8_pmuv3_perf_map,
 759                                 &armv8_pmuv3_perf_cache_map,
 760                                 ARMV8_PMU_EVTYPE_EVENT);
 761 }
 762
 763 static int armv8_a53_map_event(struct perf_event *event)
 764 {
 765         return armpmu_map_event(event, &armv8_a53_perf_map,
 766                                 &armv8_a53_perf_cache_map,
 767                                 ARMV8_PMU_EVTYPE_EVENT);
 768 }
 769
 770 static int armv8_a57_map_event(struct perf_event *event)
 771 {
 772         return armpmu_map_event(event, &armv8_a57_perf_map,
 773                                 &armv8_a57_perf_cache_map,
 774                                 ARMV8_PMU_EVTYPE_EVENT);
 775 }
 776
 777 static int armv8_thunder_map_event(struct perf_event *event)
 778 {
 779         return armpmu_map_event(event, &armv8_thunder_perf_map,
 780                                 &armv8_thunder_perf_cache_map,
 781                                 ARMV8_PMU_EVTYPE_EVENT);
 782 }
 783
 784 static void armv8pmu_read_num_pmnc_events(void *info)
 785 {
 786         int *nb_cnt = info;
 787
 788         /* Read the nb of CNTx counters supported from PMNC */
 789         *nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
 790
 791         /* Add the CPU cycles counter */
 792         *nb_cnt += 1;
 793 }
 794
 795 static int armv8pmu_probe_num_events(struct arm_pmu *arm_pmu)
 796 {
 797         return smp_call_function_any(&arm_pmu->supported_cpus,
 798                                     armv8pmu_read_num_pmnc_events,
 799                                     &arm_pmu->num_events, 1);
 800 }
 801
 802 static void armv8_pmu_init(struct arm_pmu *cpu_pmu)
 803 {
 804         cpu_pmu->handle_irq             = armv8pmu_handle_irq,
 805         cpu_pmu->enable                 = armv8pmu_enable_event,
 806         cpu_pmu->disable                = armv8pmu_disable_event,
 807         cpu_pmu->read_counter           = armv8pmu_read_counter,
 808         cpu_pmu->write_counter          = armv8pmu_write_counter,
 809         cpu_pmu->get_event_idx          = armv8pmu_get_event_idx,
 810         cpu_pmu->start                  = armv8pmu_start,
 811         cpu_pmu->stop                   = armv8pmu_stop,
 812         cpu_pmu->reset                  = armv8pmu_reset,
 813         cpu_pmu->max_period             = (1LLU << 32) - 1,
 814         cpu_pmu->set_event_filter       = armv8pmu_set_event_filter;
 815 }
 816
 817 static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
 818 {
 819         armv8_pmu_init(cpu_pmu);
 820         cpu_pmu->name                   = "armv8_pmuv3";
 821         cpu_pmu->map_event              = armv8_pmuv3_map_event;
 822         return armv8pmu_probe_num_events(cpu_pmu);
 823 }
 824
 825 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
 826 {
 827         armv8_pmu_init(cpu_pmu);
 828         cpu_pmu->name                   = "armv8_cortex_a53";
 829         cpu_pmu->map_event              = armv8_a53_map_event;
 830         cpu_pmu->pmu.attr_groups        = armv8_pmuv3_attr_groups;
 831         return armv8pmu_probe_num_events(cpu_pmu);
 832 }
 833
 834 static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
 835 {
 836         armv8_pmu_init(cpu_pmu);
 837         cpu_pmu->name                   = "armv8_cortex_a57";
 838         cpu_pmu->map_event              = armv8_a57_map_event;
 839         cpu_pmu->pmu.attr_groups        = armv8_pmuv3_attr_groups;
 840         return armv8pmu_probe_num_events(cpu_pmu);
 841 }
 842
 843 static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
 844 {
 845         armv8_pmu_init(cpu_pmu);
 846         cpu_pmu->name                   = "armv8_cortex_a72";
 847         cpu_pmu->map_event              = armv8_a57_map_event;
 848         cpu_pmu->pmu.attr_groups        = armv8_pmuv3_attr_groups;
 849         return armv8pmu_probe_num_events(cpu_pmu);
 850 }
 851
 852 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
 853 {
 854         armv8_pmu_init(cpu_pmu);
 855         cpu_pmu->name                   = "armv8_cavium_thunder";
 856         cpu_pmu->map_event              = armv8_thunder_map_event;
 857         cpu_pmu->pmu.attr_groups        = armv8_pmuv3_attr_groups;
 858         return armv8pmu_probe_num_events(cpu_pmu);
 859 }
 860
 861 static const struct of_device_id armv8_pmu_of_device_ids[] = {
 862         {.compatible = "arm,armv8-pmuv3",       .data = armv8_pmuv3_init},
 863         {.compatible = "arm,cortex-a53-pmu",    .data = armv8_a53_pmu_init},
 864         {.compatible = "arm,cortex-a57-pmu",    .data = armv8_a57_pmu_init},
 865         {.compatible = "arm,cortex-a72-pmu",    .data = armv8_a72_pmu_init},
 866         {.compatible = "cavium,thunder-pmu",    .data = armv8_thunder_pmu_init},
 867         {},
 868 };
 869
 870 static int armv8_pmu_device_probe(struct platform_device *pdev)
 871 {
 872         return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
 873 }
 874
 875 static struct platform_driver armv8_pmu_driver = {
 876         .driver         = {
 877                 .name   = "armv8-pmu",
 878                 .of_match_table = armv8_pmu_of_device_ids,
 879         },
 880         .probe          = armv8_pmu_device_probe,
 881 };
 882
 883 static int __init register_armv8_pmu_driver(void)
 884 {
 885         return platform_driver_register(&armv8_pmu_driver);
 886 }
 887 device_initcall(register_armv8_pmu_driver);