tools/perf/util/evsel.c

   1 /*
   2  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
   3  *
   4  * Parts came from builtin-{top,stat,record}.c, see those files for further
   5  * copyright notes.
   6  *
   7  * Released under the GPL v2. (and only v2, not any later version)
   8  */
   9
  10 #include <byteswap.h>
  11 #include <linux/bitops.h>
  12 #include <api/fs/tracing_path.h>
  13 #include <traceevent/event-parse.h>
  14 #include <linux/hw_breakpoint.h>
  15 #include <linux/perf_event.h>
  16 #include <linux/err.h>
  17 #include <sys/resource.h>
  18 #include "asm/bug.h"
  19 #include "callchain.h"
  20 #include "cgroup.h"
  21 #include "evsel.h"
  22 #include "evlist.h"
  23 #include "util.h"
  24 #include "cpumap.h"
  25 #include "thread_map.h"
  26 #include "target.h"
  27 #include "perf_regs.h"
  28 #include "debug.h"
  29 #include "trace-event.h"
  30 #include "stat.h"
  31
  32 static struct {
  33         bool sample_id_all;
  34         bool exclude_guest;
  35         bool mmap2;
  36         bool cloexec;
  37         bool clockid;
  38         bool clockid_wrong;
  39         bool lbr_flags;
  40         bool write_backward;
  41 } perf_missing_features;
  42
  43 static clockid_t clockid;
  44
  45 static int perf_evsel__no_extra_init(struct perf_evsel *evsel __maybe_unused)
  46 {
  47         return 0;
  48 }
  49
  50 static void perf_evsel__no_extra_fini(struct perf_evsel *evsel __maybe_unused)
  51 {
  52 }
  53
  54 static struct {
  55         size_t  size;
  56         int     (*init)(struct perf_evsel *evsel);
  57         void    (*fini)(struct perf_evsel *evsel);
  58 } perf_evsel__object = {
  59         .size = sizeof(struct perf_evsel),
  60         .init = perf_evsel__no_extra_init,
  61         .fini = perf_evsel__no_extra_fini,
  62 };
  63
  64 int perf_evsel__object_config(size_t object_size,
  65                               int (*init)(struct perf_evsel *evsel),
  66                               void (*fini)(struct perf_evsel *evsel))
  67 {
  68
  69         if (object_size == 0)
  70                 goto set_methods;
  71
  72         if (perf_evsel__object.size > object_size)
  73                 return -EINVAL;
  74
  75         perf_evsel__object.size = object_size;
  76
  77 set_methods:
  78         if (init != NULL)
  79                 perf_evsel__object.init = init;
  80
  81         if (fini != NULL)
  82                 perf_evsel__object.fini = fini;
  83
  84         return 0;
  85 }
  86
  87 #define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
  88
  89 int __perf_evsel__sample_size(u64 sample_type)
  90 {
  91         u64 mask = sample_type & PERF_SAMPLE_MASK;
  92         int size = 0;
  93         int i;
  94
  95         for (i = 0; i < 64; i++) {
  96                 if (mask & (1ULL << i))
  97                         size++;
  98         }
  99
 100         size *= sizeof(u64);
 101
 102         return size;
 103 }
 104
 105 /**
 106  * __perf_evsel__calc_id_pos - calculate id_pos.
 107  * @sample_type: sample type
 108  *
 109  * This function returns the position of the event id (PERF_SAMPLE_ID or
 110  * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
 111  * sample_event.
 112  */
 113 static int __perf_evsel__calc_id_pos(u64 sample_type)
 114 {
 115         int idx = 0;
 116
 117         if (sample_type & PERF_SAMPLE_IDENTIFIER)
 118                 return 0;
 119
 120         if (!(sample_type & PERF_SAMPLE_ID))
 121                 return -1;
 122
 123         if (sample_type & PERF_SAMPLE_IP)
 124                 idx += 1;
 125
 126         if (sample_type & PERF_SAMPLE_TID)
 127                 idx += 1;
 128
 129         if (sample_type & PERF_SAMPLE_TIME)
 130                 idx += 1;
 131
 132         if (sample_type & PERF_SAMPLE_ADDR)
 133                 idx += 1;
 134
 135         return idx;
 136 }
 137
 138 /**
 139  * __perf_evsel__calc_is_pos - calculate is_pos.
 140  * @sample_type: sample type
 141  *
 142  * This function returns the position (counting backwards) of the event id
 143  * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
 144  * sample_id_all is used there is an id sample appended to non-sample events.
 145  */
 146 static int __perf_evsel__calc_is_pos(u64 sample_type)
 147 {
 148         int idx = 1;
 149
 150         if (sample_type & PERF_SAMPLE_IDENTIFIER)
 151                 return 1;
 152
 153         if (!(sample_type & PERF_SAMPLE_ID))
 154                 return -1;
 155
 156         if (sample_type & PERF_SAMPLE_CPU)
 157                 idx += 1;
 158
 159         if (sample_type & PERF_SAMPLE_STREAM_ID)
 160                 idx += 1;
 161
 162         return idx;
 163 }
 164
 165 void perf_evsel__calc_id_pos(struct perf_evsel *evsel)
 166 {
 167         evsel->id_pos = __perf_evsel__calc_id_pos(evsel->attr.sample_type);
 168         evsel->is_pos = __perf_evsel__calc_is_pos(evsel->attr.sample_type);
 169 }
 170
 171 void __perf_evsel__set_sample_bit(struct perf_evsel *evsel,
 172                                   enum perf_event_sample_format bit)
 173 {
 174         if (!(evsel->attr.sample_type & bit)) {
 175                 evsel->attr.sample_type |= bit;
 176                 evsel->sample_size += sizeof(u64);
 177                 perf_evsel__calc_id_pos(evsel);
 178         }
 179 }
 180
 181 void __perf_evsel__reset_sample_bit(struct perf_evsel *evsel,
 182                                     enum perf_event_sample_format bit)
 183 {
 184         if (evsel->attr.sample_type & bit) {
 185                 evsel->attr.sample_type &= ~bit;
 186                 evsel->sample_size -= sizeof(u64);
 187                 perf_evsel__calc_id_pos(evsel);
 188         }
 189 }
 190
 191 void perf_evsel__set_sample_id(struct perf_evsel *evsel,
 192                                bool can_sample_identifier)
 193 {
 194         if (can_sample_identifier) {
 195                 perf_evsel__reset_sample_bit(evsel, ID);
 196                 perf_evsel__set_sample_bit(evsel, IDENTIFIER);
 197         } else {
 198                 perf_evsel__set_sample_bit(evsel, ID);
 199         }
 200         evsel->attr.read_format |= PERF_FORMAT_ID;
 201 }
 202
 203 /**
 204  * perf_evsel__is_function_event - Return whether given evsel is a function
 205  * trace event
 206  *
 207  * @evsel - evsel selector to be tested
 208  *
 209  * Return %true if event is function trace event
 210  */
 211 bool perf_evsel__is_function_event(struct perf_evsel *evsel)
 212 {
 213 #define FUNCTION_EVENT "ftrace:function"
 214
 215         return evsel->name &&
 216                !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
 217
 218 #undef FUNCTION_EVENT
 219 }
 220
 221 void perf_evsel__init(struct perf_evsel *evsel,
 222                       struct perf_event_attr *attr, int idx)
 223 {
 224         evsel->idx         = idx;
 225         evsel->tracking    = !idx;
 226         evsel->attr        = *attr;
 227         evsel->leader      = evsel;
 228         evsel->unit        = "";
 229         evsel->scale       = 1.0;
 230         evsel->evlist      = NULL;
 231         evsel->bpf_fd      = -1;
 232         INIT_LIST_HEAD(&evsel->node);
 233         INIT_LIST_HEAD(&evsel->config_terms);
 234         perf_evsel__object.init(evsel);
 235         evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
 236         perf_evsel__calc_id_pos(evsel);
 237         evsel->cmdline_group_boundary = false;
 238 }
 239
 240 struct perf_evsel *perf_evsel__new_idx(struct perf_event_attr *attr, int idx)
 241 {
 242         struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
 243
 244         if (evsel != NULL)
 245                 perf_evsel__init(evsel, attr, idx);
 246
 247         if (perf_evsel__is_bpf_output(evsel)) {
 248                 evsel->attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 249                                             PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 250                 evsel->attr.sample_period = 1;
 251         }
 252
 253         return evsel;
 254 }
 255
 256 /*
 257  * Returns pointer with encoded error via <linux/err.h> interface.
 258  */
 259 struct perf_evsel *perf_evsel__newtp_idx(const char *sys, const char *name, int idx)
 260 {
 261         struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
 262         int err = -ENOMEM;
 263
 264         if (evsel == NULL) {
 265                 goto out_err;
 266         } else {
 267                 struct perf_event_attr attr = {
 268                         .type          = PERF_TYPE_TRACEPOINT,
 269                         .sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 270                                           PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 271                 };
 272
 273                 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
 274                         goto out_free;
 275
 276                 evsel->tp_format = trace_event__tp_format(sys, name);
 277                 if (IS_ERR(evsel->tp_format)) {
 278                         err = PTR_ERR(evsel->tp_format);
 279                         goto out_free;
 280                 }
 281
 282                 event_attr_init(&attr);
 283                 attr.config = evsel->tp_format->id;
 284                 attr.sample_period = 1;
 285                 perf_evsel__init(evsel, &attr, idx);
 286         }
 287
 288         return evsel;
 289
 290 out_free:
 291         zfree(&evsel->name);
 292         free(evsel);
 293 out_err:
 294         return ERR_PTR(err);
 295 }
 296
 297 const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
 298         "cycles",
 299         "instructions",
 300         "cache-references",
 301         "cache-misses",
 302         "branches",
 303         "branch-misses",
 304         "bus-cycles",
 305         "stalled-cycles-frontend",
 306         "stalled-cycles-backend",
 307         "ref-cycles",
 308 };
 309
 310 static const char *__perf_evsel__hw_name(u64 config)
 311 {
 312         if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
 313                 return perf_evsel__hw_names[config];
 314
 315         return "unknown-hardware";
 316 }
 317
 318 static int perf_evsel__add_modifiers(struct perf_evsel *evsel, char *bf, size_t size)
 319 {
 320         int colon = 0, r = 0;
 321         struct perf_event_attr *attr = &evsel->attr;
 322         bool exclude_guest_default = false;
 323
 324 #define MOD_PRINT(context, mod) do {                                    \
 325                 if (!attr->exclude_##context) {                         \
 326                         if (!colon) colon = ++r;                        \
 327                         r += scnprintf(bf + r, size - r, "%c", mod);    \
 328                 } } while(0)
 329
 330         if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
 331                 MOD_PRINT(kernel, 'k');
 332                 MOD_PRINT(user, 'u');
 333                 MOD_PRINT(hv, 'h');
 334                 exclude_guest_default = true;
 335         }
 336
 337         if (attr->precise_ip) {
 338                 if (!colon)
 339                         colon = ++r;
 340                 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
 341                 exclude_guest_default = true;
 342         }
 343
 344         if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
 345                 MOD_PRINT(host, 'H');
 346                 MOD_PRINT(guest, 'G');
 347         }
 348 #undef MOD_PRINT
 349         if (colon)
 350                 bf[colon - 1] = ':';
 351         return r;
 352 }
 353
 354 static int perf_evsel__hw_name(struct perf_evsel *evsel, char *bf, size_t size)
 355 {
 356         int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->attr.config));
 357         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 358 }
 359
 360 const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
 361         "cpu-clock",
 362         "task-clock",
 363         "page-faults",
 364         "context-switches",
 365         "cpu-migrations",
 366         "minor-faults",
 367         "major-faults",
 368         "alignment-faults",
 369         "emulation-faults",
 370         "dummy",
 371 };
 372
 373 static const char *__perf_evsel__sw_name(u64 config)
 374 {
 375         if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
 376                 return perf_evsel__sw_names[config];
 377         return "unknown-software";
 378 }
 379
 380 static int perf_evsel__sw_name(struct perf_evsel *evsel, char *bf, size_t size)
 381 {
 382         int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->attr.config));
 383         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 384 }
 385
 386 static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
 387 {
 388         int r;
 389
 390         r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
 391
 392         if (type & HW_BREAKPOINT_R)
 393                 r += scnprintf(bf + r, size - r, "r");
 394
 395         if (type & HW_BREAKPOINT_W)
 396                 r += scnprintf(bf + r, size - r, "w");
 397
 398         if (type & HW_BREAKPOINT_X)
 399                 r += scnprintf(bf + r, size - r, "x");
 400
 401         return r;
 402 }
 403
 404 static int perf_evsel__bp_name(struct perf_evsel *evsel, char *bf, size_t size)
 405 {
 406         struct perf_event_attr *attr = &evsel->attr;
 407         int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
 408         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 409 }
 410
 411 const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
 412                                 [PERF_EVSEL__MAX_ALIASES] = {
 413  { "L1-dcache", "l1-d",         "l1d",          "L1-data",              },
 414  { "L1-icache", "l1-i",         "l1i",          "L1-instruction",       },
 415  { "LLC",       "L2",                                                   },
 416  { "dTLB",      "d-tlb",        "Data-TLB",                             },
 417  { "iTLB",      "i-tlb",        "Instruction-TLB",                      },
 418  { "branch",    "branches",     "bpu",          "btb",          "bpc",  },
 419  { "node",                                                              },
 420 };
 421
 422 const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
 423                                    [PERF_EVSEL__MAX_ALIASES] = {
 424  { "load",      "loads",        "read",                                 },
 425  { "store",     "stores",       "write",                                },
 426  { "prefetch",  "prefetches",   "speculative-read", "speculative-load", },
 427 };
 428
 429 const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
 430                                        [PERF_EVSEL__MAX_ALIASES] = {
 431  { "refs",      "Reference",    "ops",          "access",               },
 432  { "misses",    "miss",                                                 },
 433 };
 434
 435 #define C(x)            PERF_COUNT_HW_CACHE_##x
 436 #define CACHE_READ      (1 << C(OP_READ))
 437 #define CACHE_WRITE     (1 << C(OP_WRITE))
 438 #define CACHE_PREFETCH  (1 << C(OP_PREFETCH))
 439 #define COP(x)          (1 << x)
 440
 441 /*
 442  * cache operartion stat
 443  * L1I : Read and prefetch only
 444  * ITLB and BPU : Read-only
 445  */
 446 static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
 447  [C(L1D)]       = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 448  [C(L1I)]       = (CACHE_READ | CACHE_PREFETCH),
 449  [C(LL)]        = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 450  [C(DTLB)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 451  [C(ITLB)]      = (CACHE_READ),
 452  [C(BPU)]       = (CACHE_READ),
 453  [C(NODE)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 454 };
 455
 456 bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
 457 {
 458         if (perf_evsel__hw_cache_stat[type] & COP(op))
 459                 return true;    /* valid */
 460         else
 461                 return false;   /* invalid */
 462 }
 463
 464 int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
 465                                             char *bf, size_t size)
 466 {
 467         if (result) {
 468                 return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
 469                                  perf_evsel__hw_cache_op[op][0],
 470                                  perf_evsel__hw_cache_result[result][0]);
 471         }
 472
 473         return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
 474                          perf_evsel__hw_cache_op[op][1]);
 475 }
 476
 477 static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
 478 {
 479         u8 op, result, type = (config >>  0) & 0xff;
 480         const char *err = "unknown-ext-hardware-cache-type";
 481
 482         if (type > PERF_COUNT_HW_CACHE_MAX)
 483                 goto out_err;
 484
 485         op = (config >>  8) & 0xff;
 486         err = "unknown-ext-hardware-cache-op";
 487         if (op > PERF_COUNT_HW_CACHE_OP_MAX)
 488                 goto out_err;
 489
 490         result = (config >> 16) & 0xff;
 491         err = "unknown-ext-hardware-cache-result";
 492         if (result > PERF_COUNT_HW_CACHE_RESULT_MAX)
 493                 goto out_err;
 494
 495         err = "invalid-cache";
 496         if (!perf_evsel__is_cache_op_valid(type, op))
 497                 goto out_err;
 498
 499         return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
 500 out_err:
 501         return scnprintf(bf, size, "%s", err);
 502 }
 503
 504 static int perf_evsel__hw_cache_name(struct perf_evsel *evsel, char *bf, size_t size)
 505 {
 506         int ret = __perf_evsel__hw_cache_name(evsel->attr.config, bf, size);
 507         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 508 }
 509
 510 static int perf_evsel__raw_name(struct perf_evsel *evsel, char *bf, size_t size)
 511 {
 512         int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->attr.config);
 513         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 514 }
 515
 516 const char *perf_evsel__name(struct perf_evsel *evsel)
 517 {
 518         char bf[128];
 519
 520         if (evsel->name)
 521                 return evsel->name;
 522
 523         switch (evsel->attr.type) {
 524         case PERF_TYPE_RAW:
 525                 perf_evsel__raw_name(evsel, bf, sizeof(bf));
 526                 break;
 527
 528         case PERF_TYPE_HARDWARE:
 529                 perf_evsel__hw_name(evsel, bf, sizeof(bf));
 530                 break;
 531
 532         case PERF_TYPE_HW_CACHE:
 533                 perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
 534                 break;
 535
 536         case PERF_TYPE_SOFTWARE:
 537                 perf_evsel__sw_name(evsel, bf, sizeof(bf));
 538                 break;
 539
 540         case PERF_TYPE_TRACEPOINT:
 541                 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
 542                 break;
 543
 544         case PERF_TYPE_BREAKPOINT:
 545                 perf_evsel__bp_name(evsel, bf, sizeof(bf));
 546                 break;
 547
 548         default:
 549                 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
 550                           evsel->attr.type);
 551                 break;
 552         }
 553
 554         evsel->name = strdup(bf);
 555
 556         return evsel->name ?: "unknown";
 557 }
 558
 559 const char *perf_evsel__group_name(struct perf_evsel *evsel)
 560 {
 561         return evsel->group_name ?: "anon group";
 562 }
 563
 564 int perf_evsel__group_desc(struct perf_evsel *evsel, char *buf, size_t size)
 565 {
 566         int ret;
 567         struct perf_evsel *pos;
 568         const char *group_name = perf_evsel__group_name(evsel);
 569
 570         ret = scnprintf(buf, size, "%s", group_name);
 571
 572         ret += scnprintf(buf + ret, size - ret, " { %s",
 573                          perf_evsel__name(evsel));
 574
 575         for_each_group_member(pos, evsel)
 576                 ret += scnprintf(buf + ret, size - ret, ", %s",
 577                                  perf_evsel__name(pos));
 578
 579         ret += scnprintf(buf + ret, size - ret, " }");
 580
 581         return ret;
 582 }
 583
 584 void perf_evsel__config_callchain(struct perf_evsel *evsel,
 585                                   struct record_opts *opts,
 586                                   struct callchain_param *param)
 587 {
 588         bool function = perf_evsel__is_function_event(evsel);
 589         struct perf_event_attr *attr = &evsel->attr;
 590
 591         perf_evsel__set_sample_bit(evsel, CALLCHAIN);
 592
 593         attr->sample_max_stack = param->max_stack;
 594
 595         if (param->record_mode == CALLCHAIN_LBR) {
 596                 if (!opts->branch_stack) {
 597                         if (attr->exclude_user) {
 598                                 pr_warning("LBR callstack option is only available "
 599                                            "to get user callchain information. "
 600                                            "Falling back to framepointers.\n");
 601                         } else {
 602                                 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
 603                                 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
 604                                                         PERF_SAMPLE_BRANCH_CALL_STACK |
 605                                                         PERF_SAMPLE_BRANCH_NO_CYCLES |
 606                                                         PERF_SAMPLE_BRANCH_NO_FLAGS;
 607                         }
 608                 } else
 609                          pr_warning("Cannot use LBR callstack with branch stack. "
 610                                     "Falling back to framepointers.\n");
 611         }
 612
 613         if (param->record_mode == CALLCHAIN_DWARF) {
 614                 if (!function) {
 615                         perf_evsel__set_sample_bit(evsel, REGS_USER);
 616                         perf_evsel__set_sample_bit(evsel, STACK_USER);
 617                         attr->sample_regs_user = PERF_REGS_MASK;
 618                         attr->sample_stack_user = param->dump_size;
 619                         attr->exclude_callchain_user = 1;
 620                 } else {
 621                         pr_info("Cannot use DWARF unwind for function trace event,"
 622                                 " falling back to framepointers.\n");
 623                 }
 624         }
 625
 626         if (function) {
 627                 pr_info("Disabling user space callchains for function trace event.\n");
 628                 attr->exclude_callchain_user = 1;
 629         }
 630 }
 631
 632 static void
 633 perf_evsel__reset_callgraph(struct perf_evsel *evsel,
 634                             struct callchain_param *param)
 635 {
 636         struct perf_event_attr *attr = &evsel->attr;
 637
 638         perf_evsel__reset_sample_bit(evsel, CALLCHAIN);
 639         if (param->record_mode == CALLCHAIN_LBR) {
 640                 perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
 641                 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
 642                                               PERF_SAMPLE_BRANCH_CALL_STACK);
 643         }
 644         if (param->record_mode == CALLCHAIN_DWARF) {
 645                 perf_evsel__reset_sample_bit(evsel, REGS_USER);
 646                 perf_evsel__reset_sample_bit(evsel, STACK_USER);
 647         }
 648 }
 649
 650 static void apply_config_terms(struct perf_evsel *evsel,
 651                                struct record_opts *opts)
 652 {
 653         struct perf_evsel_config_term *term;
 654         struct list_head *config_terms = &evsel->config_terms;
 655         struct perf_event_attr *attr = &evsel->attr;
 656         struct callchain_param param;
 657         u32 dump_size = 0;
 658         int max_stack = 0;
 659         const char *callgraph_buf = NULL;
 660
 661         /* callgraph default */
 662         param.record_mode = callchain_param.record_mode;
 663
 664         list_for_each_entry(term, config_terms, list) {
 665                 switch (term->type) {
 666                 case PERF_EVSEL__CONFIG_TERM_PERIOD:
 667                         attr->sample_period = term->val.period;
 668                         attr->freq = 0;
 669                         break;
 670                 case PERF_EVSEL__CONFIG_TERM_FREQ:
 671                         attr->sample_freq = term->val.freq;
 672                         attr->freq = 1;
 673                         break;
 674                 case PERF_EVSEL__CONFIG_TERM_TIME:
 675                         if (term->val.time)
 676                                 perf_evsel__set_sample_bit(evsel, TIME);
 677                         else
 678                                 perf_evsel__reset_sample_bit(evsel, TIME);
 679                         break;
 680                 case PERF_EVSEL__CONFIG_TERM_CALLGRAPH:
 681                         callgraph_buf = term->val.callgraph;
 682                         break;
 683                 case PERF_EVSEL__CONFIG_TERM_STACK_USER:
 684                         dump_size = term->val.stack_user;
 685                         break;
 686                 case PERF_EVSEL__CONFIG_TERM_MAX_STACK:
 687                         max_stack = term->val.max_stack;
 688                         break;
 689                 case PERF_EVSEL__CONFIG_TERM_INHERIT:
 690                         /*
 691                          * attr->inherit should has already been set by
 692                          * perf_evsel__config. If user explicitly set
 693                          * inherit using config terms, override global
 694                          * opt->no_inherit setting.
 695                          */
 696                         attr->inherit = term->val.inherit ? 1 : 0;
 697                         break;
 698                 case PERF_EVSEL__CONFIG_TERM_OVERWRITE:
 699                         attr->write_backward = term->val.overwrite ? 1 : 0;
 700                         break;
 701                 default:
 702                         break;
 703                 }
 704         }
 705
 706         /* User explicitly set per-event callgraph, clear the old setting and reset. */
 707         if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
 708                 if (max_stack) {
 709                         param.max_stack = max_stack;
 710                         if (callgraph_buf == NULL)
 711                                 callgraph_buf = "fp";
 712                 }
 713
 714                 /* parse callgraph parameters */
 715                 if (callgraph_buf != NULL) {
 716                         if (!strcmp(callgraph_buf, "no")) {
 717                                 param.enabled = false;
 718                                 param.record_mode = CALLCHAIN_NONE;
 719                         } else {
 720                                 param.enabled = true;
 721                                 if (parse_callchain_record(callgraph_buf, &param)) {
 722                                         pr_err("per-event callgraph setting for %s failed. "
 723                                                "Apply callgraph global setting for it\n",
 724                                                evsel->name);
 725                                         return;
 726                                 }
 727                         }
 728                 }
 729                 if (dump_size > 0) {
 730                         dump_size = round_up(dump_size, sizeof(u64));
 731                         param.dump_size = dump_size;
 732                 }
 733
 734                 /* If global callgraph set, clear it */
 735                 if (callchain_param.enabled)
 736                         perf_evsel__reset_callgraph(evsel, &callchain_param);
 737
 738                 /* set perf-event callgraph */
 739                 if (param.enabled)
 740                         perf_evsel__config_callchain(evsel, opts, &param);
 741         }
 742 }
 743
 744 /*
 745  * The enable_on_exec/disabled value strategy:
 746  *
 747  *  1) For any type of traced program:
 748  *    - all independent events and group leaders are disabled
 749  *    - all group members are enabled
 750  *
 751  *     Group members are ruled by group leaders. They need to
 752  *     be enabled, because the group scheduling relies on that.
 753  *
 754  *  2) For traced programs executed by perf:
 755  *     - all independent events and group leaders have
 756  *       enable_on_exec set
 757  *     - we don't specifically enable or disable any event during
 758  *       the record command
 759  *
 760  *     Independent events and group leaders are initially disabled
 761  *     and get enabled by exec. Group members are ruled by group
 762  *     leaders as stated in 1).
 763  *
 764  *  3) For traced programs attached by perf (pid/tid):
 765  *     - we specifically enable or disable all events during
 766  *       the record command
 767  *
 768  *     When attaching events to already running traced we
 769  *     enable/disable events specifically, as there's no
 770  *     initial traced exec call.
 771  */
 772 void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts,
 773                         struct callchain_param *callchain)
 774 {
 775         struct perf_evsel *leader = evsel->leader;
 776         struct perf_event_attr *attr = &evsel->attr;
 777         int track = evsel->tracking;
 778         bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
 779
 780         attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
 781         attr->inherit       = !opts->no_inherit;
 782         attr->write_backward = opts->overwrite ? 1 : 0;
 783
 784         perf_evsel__set_sample_bit(evsel, IP);
 785         perf_evsel__set_sample_bit(evsel, TID);
 786
 787         if (evsel->sample_read) {
 788                 perf_evsel__set_sample_bit(evsel, READ);
 789
 790                 /*
 791                  * We need ID even in case of single event, because
 792                  * PERF_SAMPLE_READ process ID specific data.
 793                  */
 794                 perf_evsel__set_sample_id(evsel, false);
 795
 796                 /*
 797                  * Apply group format only if we belong to group
 798                  * with more than one members.
 799                  */
 800                 if (leader->nr_members > 1) {
 801                         attr->read_format |= PERF_FORMAT_GROUP;
 802                         attr->inherit = 0;
 803                 }
 804         }
 805
 806         /*
 807          * We default some events to have a default interval. But keep
 808          * it a weak assumption overridable by the user.
 809          */
 810         if (!attr->sample_period || (opts->user_freq != UINT_MAX ||
 811                                      opts->user_interval != ULLONG_MAX)) {
 812                 if (opts->freq) {
 813                         perf_evsel__set_sample_bit(evsel, PERIOD);
 814                         attr->freq              = 1;
 815                         attr->sample_freq       = opts->freq;
 816                 } else {
 817                         attr->sample_period = opts->default_interval;
 818                 }
 819         }
 820
 821         /*
 822          * Disable sampling for all group members other
 823          * than leader in case leader 'leads' the sampling.
 824          */
 825         if ((leader != evsel) && leader->sample_read) {
 826                 attr->sample_freq   = 0;
 827                 attr->sample_period = 0;
 828         }
 829
 830         if (opts->no_samples)
 831                 attr->sample_freq = 0;
 832
 833         if (opts->inherit_stat)
 834                 attr->inherit_stat = 1;
 835
 836         if (opts->sample_address) {
 837                 perf_evsel__set_sample_bit(evsel, ADDR);
 838                 attr->mmap_data = track;
 839         }
 840
 841         /*
 842          * We don't allow user space callchains for  function trace
 843          * event, due to issues with page faults while tracing page
 844          * fault handler and its overall trickiness nature.
 845          */
 846         if (perf_evsel__is_function_event(evsel))
 847                 evsel->attr.exclude_callchain_user = 1;
 848
 849         if (callchain && callchain->enabled && !evsel->no_aux_samples)
 850                 perf_evsel__config_callchain(evsel, opts, callchain);
 851
 852         if (opts->sample_intr_regs) {
 853                 attr->sample_regs_intr = opts->sample_intr_regs;
 854                 perf_evsel__set_sample_bit(evsel, REGS_INTR);
 855         }
 856
 857         if (target__has_cpu(&opts->target))
 858                 perf_evsel__set_sample_bit(evsel, CPU);
 859
 860         if (opts->period)
 861                 perf_evsel__set_sample_bit(evsel, PERIOD);
 862
 863         /*
 864          * When the user explicitly disabled time don't force it here.
 865          */
 866         if (opts->sample_time &&
 867             (!perf_missing_features.sample_id_all &&
 868             (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
 869              opts->sample_time_set)))
 870                 perf_evsel__set_sample_bit(evsel, TIME);
 871
 872         if (opts->raw_samples && !evsel->no_aux_samples) {
 873                 perf_evsel__set_sample_bit(evsel, TIME);
 874                 perf_evsel__set_sample_bit(evsel, RAW);
 875                 perf_evsel__set_sample_bit(evsel, CPU);
 876         }
 877
 878         if (opts->sample_address)
 879                 perf_evsel__set_sample_bit(evsel, DATA_SRC);
 880
 881         if (opts->no_buffering) {
 882                 attr->watermark = 0;
 883                 attr->wakeup_events = 1;
 884         }
 885         if (opts->branch_stack && !evsel->no_aux_samples) {
 886                 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
 887                 attr->branch_sample_type = opts->branch_stack;
 888         }
 889
 890         if (opts->sample_weight)
 891                 perf_evsel__set_sample_bit(evsel, WEIGHT);
 892
 893         attr->task  = track;
 894         attr->mmap  = track;
 895         attr->mmap2 = track && !perf_missing_features.mmap2;
 896         attr->comm  = track;
 897
 898         if (opts->record_switch_events)
 899                 attr->context_switch = track;
 900
 901         if (opts->sample_transaction)
 902                 perf_evsel__set_sample_bit(evsel, TRANSACTION);
 903
 904         if (opts->running_time) {
 905                 evsel->attr.read_format |=
 906                         PERF_FORMAT_TOTAL_TIME_ENABLED |
 907                         PERF_FORMAT_TOTAL_TIME_RUNNING;
 908         }
 909
 910         /*
 911          * XXX see the function comment above
 912          *
 913          * Disabling only independent events or group leaders,
 914          * keeping group members enabled.
 915          */
 916         if (perf_evsel__is_group_leader(evsel))
 917                 attr->disabled = 1;
 918
 919         /*
 920          * Setting enable_on_exec for independent events and
 921          * group leaders for traced executed by perf.
 922          */
 923         if (target__none(&opts->target) && perf_evsel__is_group_leader(evsel) &&
 924                 !opts->initial_delay)
 925                 attr->enable_on_exec = 1;
 926
 927         if (evsel->immediate) {
 928                 attr->disabled = 0;
 929                 attr->enable_on_exec = 0;
 930         }
 931
 932         clockid = opts->clockid;
 933         if (opts->use_clockid) {
 934                 attr->use_clockid = 1;
 935                 attr->clockid = opts->clockid;
 936         }
 937
 938         if (evsel->precise_max)
 939                 perf_event_attr__set_max_precise_ip(attr);
 940
 941         if (opts->all_user) {
 942                 attr->exclude_kernel = 1;
 943                 attr->exclude_user   = 0;
 944         }
 945
 946         if (opts->all_kernel) {
 947                 attr->exclude_kernel = 0;
 948                 attr->exclude_user   = 1;
 949         }
 950
 951         /*
 952          * Apply event specific term settings,
 953          * it overloads any global configuration.
 954          */
 955         apply_config_terms(evsel, opts);
 956 }
 957
 958 static int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 959 {
 960         int cpu, thread;
 961
 962         if (evsel->system_wide)
 963                 nthreads = 1;
 964
 965         evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
 966
 967         if (evsel->fd) {
 968                 for (cpu = 0; cpu < ncpus; cpu++) {
 969                         for (thread = 0; thread < nthreads; thread++) {
 970                                 FD(evsel, cpu, thread) = -1;
 971                         }
 972                 }
 973         }
 974
 975         return evsel->fd != NULL ? 0 : -ENOMEM;
 976 }
 977
 978 static int perf_evsel__run_ioctl(struct perf_evsel *evsel, int ncpus, int nthreads,
 979                           int ioc,  void *arg)
 980 {
 981         int cpu, thread;
 982
 983         if (evsel->system_wide)
 984                 nthreads = 1;
 985
 986         for (cpu = 0; cpu < ncpus; cpu++) {
 987                 for (thread = 0; thread < nthreads; thread++) {
 988                         int fd = FD(evsel, cpu, thread),
 989                             err = ioctl(fd, ioc, arg);
 990
 991                         if (err)
 992                                 return err;
 993                 }
 994         }
 995
 996         return 0;
 997 }
 998
 999 int perf_evsel__apply_filter(struct perf_evsel *evsel, int ncpus, int nthreads,
1000                              const char *filter)
1001 {
1002         return perf_evsel__run_ioctl(evsel, ncpus, nthreads,
1003                                      PERF_EVENT_IOC_SET_FILTER,
1004                                      (void *)filter);
1005 }
1006
1007 int perf_evsel__set_filter(struct perf_evsel *evsel, const char *filter)
1008 {
1009         char *new_filter = strdup(filter);
1010
1011         if (new_filter != NULL) {
1012                 free(evsel->filter);
1013                 evsel->filter = new_filter;
1014                 return 0;
1015         }
1016
1017         return -1;
1018 }
1019
1020 int perf_evsel__append_filter(struct perf_evsel *evsel,
1021                               const char *op, const char *filter)
1022 {
1023         char *new_filter;
1024
1025         if (evsel->filter == NULL)
1026                 return perf_evsel__set_filter(evsel, filter);
1027
1028         if (asprintf(&new_filter,"(%s) %s (%s)", evsel->filter, op, filter) > 0) {
1029                 free(evsel->filter);
1030                 evsel->filter = new_filter;
1031                 return 0;
1032         }
1033
1034         return -1;
1035 }
1036
1037 int perf_evsel__enable(struct perf_evsel *evsel)
1038 {
1039         int nthreads = thread_map__nr(evsel->threads);
1040         int ncpus = cpu_map__nr(evsel->cpus);
1041
1042         return perf_evsel__run_ioctl(evsel, ncpus, nthreads,
1043                                      PERF_EVENT_IOC_ENABLE,
1044                                      0);
1045 }
1046
1047 int perf_evsel__disable(struct perf_evsel *evsel)
1048 {
1049         int nthreads = thread_map__nr(evsel->threads);
1050         int ncpus = cpu_map__nr(evsel->cpus);
1051
1052         return perf_evsel__run_ioctl(evsel, ncpus, nthreads,
1053                                      PERF_EVENT_IOC_DISABLE,
1054                                      0);
1055 }
1056
1057 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
1058 {
1059         if (ncpus == 0 || nthreads == 0)
1060                 return 0;
1061
1062         if (evsel->system_wide)
1063                 nthreads = 1;
1064
1065         evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
1066         if (evsel->sample_id == NULL)
1067                 return -ENOMEM;
1068
1069         evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
1070         if (evsel->id == NULL) {
1071                 xyarray__delete(evsel->sample_id);
1072                 evsel->sample_id = NULL;
1073                 return -ENOMEM;
1074         }
1075
1076         return 0;
1077 }
1078
1079 static void perf_evsel__free_fd(struct perf_evsel *evsel)
1080 {
1081         xyarray__delete(evsel->fd);
1082         evsel->fd = NULL;
1083 }
1084
1085 static void perf_evsel__free_id(struct perf_evsel *evsel)
1086 {
1087         xyarray__delete(evsel->sample_id);
1088         evsel->sample_id = NULL;
1089         zfree(&evsel->id);
1090 }
1091
1092 static void perf_evsel__free_config_terms(struct perf_evsel *evsel)
1093 {
1094         struct perf_evsel_config_term *term, *h;
1095
1096         list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1097                 list_del(&term->list);
1098                 free(term);
1099         }
1100 }
1101
1102 void perf_evsel__close_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
1103 {
1104         int cpu, thread;
1105
1106         if (evsel->system_wide)
1107                 nthreads = 1;
1108
1109         for (cpu = 0; cpu < ncpus; cpu++)
1110                 for (thread = 0; thread < nthreads; ++thread) {
1111                         close(FD(evsel, cpu, thread));
1112                         FD(evsel, cpu, thread) = -1;
1113                 }
1114 }
1115
1116 void perf_evsel__exit(struct perf_evsel *evsel)
1117 {
1118         assert(list_empty(&evsel->node));
1119         assert(evsel->evlist == NULL);
1120         perf_evsel__free_fd(evsel);
1121         perf_evsel__free_id(evsel);
1122         perf_evsel__free_config_terms(evsel);
1123         close_cgroup(evsel->cgrp);
1124         cpu_map__put(evsel->cpus);
1125         cpu_map__put(evsel->own_cpus);
1126         thread_map__put(evsel->threads);
1127         zfree(&evsel->group_name);
1128         zfree(&evsel->name);
1129         perf_evsel__object.fini(evsel);
1130 }
1131
1132 void perf_evsel__delete(struct perf_evsel *evsel)
1133 {
1134         perf_evsel__exit(evsel);
1135         free(evsel);
1136 }
1137
1138 void perf_evsel__compute_deltas(struct perf_evsel *evsel, int cpu, int thread,
1139                                 struct perf_counts_values *count)
1140 {
1141         struct perf_counts_values tmp;
1142
1143         if (!evsel->prev_raw_counts)
1144                 return;
1145
1146         if (cpu == -1) {
1147                 tmp = evsel->prev_raw_counts->aggr;
1148                 evsel->prev_raw_counts->aggr = *count;
1149         } else {
1150                 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1151                 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1152         }
1153
1154         count->val = count->val - tmp.val;
1155         count->ena = count->ena - tmp.ena;
1156         count->run = count->run - tmp.run;
1157 }
1158
1159 void perf_counts_values__scale(struct perf_counts_values *count,
1160                                bool scale, s8 *pscaled)
1161 {
1162         s8 scaled = 0;
1163
1164         if (scale) {
1165                 if (count->run == 0) {
1166                         scaled = -1;
1167                         count->val = 0;
1168                 } else if (count->run < count->ena) {
1169                         scaled = 1;
1170                         count->val = (u64)((double) count->val * count->ena / count->run + 0.5);
1171                 }
1172         } else
1173                 count->ena = count->run = 0;
1174
1175         if (pscaled)
1176                 *pscaled = scaled;
1177 }
1178
1179 int perf_evsel__read(struct perf_evsel *evsel, int cpu, int thread,
1180                      struct perf_counts_values *count)
1181 {
1182         memset(count, 0, sizeof(*count));
1183
1184         if (FD(evsel, cpu, thread) < 0)
1185                 return -EINVAL;
1186
1187         if (readn(FD(evsel, cpu, thread), count, sizeof(*count)) < 0)
1188                 return -errno;
1189
1190         return 0;
1191 }
1192
1193 int __perf_evsel__read_on_cpu(struct perf_evsel *evsel,
1194                               int cpu, int thread, bool scale)
1195 {
1196         struct perf_counts_values count;
1197         size_t nv = scale ? 3 : 1;
1198
1199         if (FD(evsel, cpu, thread) < 0)
1200                 return -EINVAL;
1201
1202         if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1203                 return -ENOMEM;
1204
1205         if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) < 0)
1206                 return -errno;
1207
1208         perf_evsel__compute_deltas(evsel, cpu, thread, &count);
1209         perf_counts_values__scale(&count, scale, NULL);
1210         *perf_counts(evsel->counts, cpu, thread) = count;
1211         return 0;
1212 }
1213
1214 static int get_group_fd(struct perf_evsel *evsel, int cpu, int thread)
1215 {
1216         struct perf_evsel *leader = evsel->leader;
1217         int fd;
1218
1219         if (perf_evsel__is_group_leader(evsel))
1220                 return -1;
1221
1222         /*
1223          * Leader must be already processed/open,
1224          * if not it's a bug.
1225          */
1226         BUG_ON(!leader->fd);
1227
1228         fd = FD(leader, cpu, thread);
1229         BUG_ON(fd == -1);
1230
1231         return fd;
1232 }
1233
1234 struct bit_names {
1235         int bit;
1236         const char *name;
1237 };
1238
1239 static void __p_bits(char *buf, size_t size, u64 value, struct bit_names *bits)
1240 {
1241         bool first_bit = true;
1242         int i = 0;
1243
1244         do {
1245                 if (value & bits[i].bit) {
1246                         buf += scnprintf(buf, size, "%s%s", first_bit ? "" : "|", bits[i].name);
1247                         first_bit = false;
1248                 }
1249         } while (bits[++i].name != NULL);
1250 }
1251
1252 static void __p_sample_type(char *buf, size_t size, u64 value)
1253 {
1254 #define bit_name(n) { PERF_SAMPLE_##n, #n }
1255         struct bit_names bits[] = {
1256                 bit_name(IP), bit_name(TID), bit_name(TIME), bit_name(ADDR),
1257                 bit_name(READ), bit_name(CALLCHAIN), bit_name(ID), bit_name(CPU),
1258                 bit_name(PERIOD), bit_name(STREAM_ID), bit_name(RAW),
1259                 bit_name(BRANCH_STACK), bit_name(REGS_USER), bit_name(STACK_USER),
1260                 bit_name(IDENTIFIER), bit_name(REGS_INTR), bit_name(DATA_SRC),
1261                 bit_name(WEIGHT),
1262                 { .name = NULL, }
1263         };
1264 #undef bit_name
1265         __p_bits(buf, size, value, bits);
1266 }
1267
1268 static void __p_branch_sample_type(char *buf, size_t size, u64 value)
1269 {
1270 #define bit_name(n) { PERF_SAMPLE_BRANCH_##n, #n }
1271         struct bit_names bits[] = {
1272                 bit_name(USER), bit_name(KERNEL), bit_name(HV), bit_name(ANY),
1273                 bit_name(ANY_CALL), bit_name(ANY_RETURN), bit_name(IND_CALL),
1274                 bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
1275                 bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
1276                 bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
1277                 { .name = NULL, }
1278         };
1279 #undef bit_name
1280         __p_bits(buf, size, value, bits);
1281 }
1282
1283 static void __p_read_format(char *buf, size_t size, u64 value)
1284 {
1285 #define bit_name(n) { PERF_FORMAT_##n, #n }
1286         struct bit_names bits[] = {
1287                 bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
1288                 bit_name(ID), bit_name(GROUP),
1289                 { .name = NULL, }
1290         };
1291 #undef bit_name
1292         __p_bits(buf, size, value, bits);
1293 }
1294
1295 #define BUF_SIZE                1024
1296
1297 #define p_hex(val)              snprintf(buf, BUF_SIZE, "%#"PRIx64, (uint64_t)(val))
1298 #define p_unsigned(val)         snprintf(buf, BUF_SIZE, "%"PRIu64, (uint64_t)(val))
1299 #define p_signed(val)           snprintf(buf, BUF_SIZE, "%"PRId64, (int64_t)(val))
1300 #define p_sample_type(val)      __p_sample_type(buf, BUF_SIZE, val)
1301 #define p_branch_sample_type(val) __p_branch_sample_type(buf, BUF_SIZE, val)
1302 #define p_read_format(val)      __p_read_format(buf, BUF_SIZE, val)
1303
1304 #define PRINT_ATTRn(_n, _f, _p)                         \
1305 do {                                                    \
1306         if (attr->_f) {                                 \
1307                 _p(attr->_f);                           \
1308                 ret += attr__fprintf(fp, _n, buf, priv);\
1309         }                                               \
1310 } while (0)
1311
1312 #define PRINT_ATTRf(_f, _p)     PRINT_ATTRn(#_f, _f, _p)
1313
1314 int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr,
1315                              attr__fprintf_f attr__fprintf, void *priv)
1316 {
1317         char buf[BUF_SIZE];
1318         int ret = 0;
1319
1320         PRINT_ATTRf(type, p_unsigned);
1321         PRINT_ATTRf(size, p_unsigned);
1322         PRINT_ATTRf(config, p_hex);
1323         PRINT_ATTRn("{ sample_period, sample_freq }", sample_period, p_unsigned);
1324         PRINT_ATTRf(sample_type, p_sample_type);
1325         PRINT_ATTRf(read_format, p_read_format);
1326
1327         PRINT_ATTRf(disabled, p_unsigned);
1328         PRINT_ATTRf(inherit, p_unsigned);
1329         PRINT_ATTRf(pinned, p_unsigned);
1330         PRINT_ATTRf(exclusive, p_unsigned);
1331         PRINT_ATTRf(exclude_user, p_unsigned);
1332         PRINT_ATTRf(exclude_kernel, p_unsigned);
1333         PRINT_ATTRf(exclude_hv, p_unsigned);
1334         PRINT_ATTRf(exclude_idle, p_unsigned);
1335         PRINT_ATTRf(mmap, p_unsigned);
1336         PRINT_ATTRf(comm, p_unsigned);
1337         PRINT_ATTRf(freq, p_unsigned);
1338         PRINT_ATTRf(inherit_stat, p_unsigned);
1339         PRINT_ATTRf(enable_on_exec, p_unsigned);
1340         PRINT_ATTRf(task, p_unsigned);
1341         PRINT_ATTRf(watermark, p_unsigned);
1342         PRINT_ATTRf(precise_ip, p_unsigned);
1343         PRINT_ATTRf(mmap_data, p_unsigned);
1344         PRINT_ATTRf(sample_id_all, p_unsigned);
1345         PRINT_ATTRf(exclude_host, p_unsigned);
1346         PRINT_ATTRf(exclude_guest, p_unsigned);
1347         PRINT_ATTRf(exclude_callchain_kernel, p_unsigned);
1348         PRINT_ATTRf(exclude_callchain_user, p_unsigned);
1349         PRINT_ATTRf(mmap2, p_unsigned);
1350         PRINT_ATTRf(comm_exec, p_unsigned);
1351         PRINT_ATTRf(use_clockid, p_unsigned);
1352         PRINT_ATTRf(context_switch, p_unsigned);
1353         PRINT_ATTRf(write_backward, p_unsigned);
1354
1355         PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned);
1356         PRINT_ATTRf(bp_type, p_unsigned);
1357         PRINT_ATTRn("{ bp_addr, config1 }", bp_addr, p_hex);
1358         PRINT_ATTRn("{ bp_len, config2 }", bp_len, p_hex);
1359         PRINT_ATTRf(branch_sample_type, p_branch_sample_type);
1360         PRINT_ATTRf(sample_regs_user, p_hex);
1361         PRINT_ATTRf(sample_stack_user, p_unsigned);
1362         PRINT_ATTRf(clockid, p_signed);
1363         PRINT_ATTRf(sample_regs_intr, p_hex);
1364         PRINT_ATTRf(aux_watermark, p_unsigned);
1365         PRINT_ATTRf(sample_max_stack, p_unsigned);
1366
1367         return ret;
1368 }
1369
1370 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1371                                 void *priv __attribute__((unused)))
1372 {
1373         return fprintf(fp, "  %-32s %s\n", name, val);
1374 }
1375
1376 static int __perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
1377                               struct thread_map *threads)
1378 {
1379         int cpu, thread, nthreads;
1380         unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1381         int pid = -1, err;
1382         enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1383
1384         if (perf_missing_features.write_backward && evsel->attr.write_backward)
1385                 return -EINVAL;
1386
1387         if (evsel->system_wide)
1388                 nthreads = 1;
1389         else
1390                 nthreads = threads->nr;
1391
1392         if (evsel->fd == NULL &&
1393             perf_evsel__alloc_fd(evsel, cpus->nr, nthreads) < 0)
1394                 return -ENOMEM;
1395
1396         if (evsel->cgrp) {
1397                 flags |= PERF_FLAG_PID_CGROUP;
1398                 pid = evsel->cgrp->fd;
1399         }
1400
1401 fallback_missing_features:
1402         if (perf_missing_features.clockid_wrong)
1403                 evsel->attr.clockid = CLOCK_MONOTONIC; /* should always work */
1404         if (perf_missing_features.clockid) {
1405                 evsel->attr.use_clockid = 0;
1406                 evsel->attr.clockid = 0;
1407         }
1408         if (perf_missing_features.cloexec)
1409                 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1410         if (perf_missing_features.mmap2)
1411                 evsel->attr.mmap2 = 0;
1412         if (perf_missing_features.exclude_guest)
1413                 evsel->attr.exclude_guest = evsel->attr.exclude_host = 0;
1414         if (perf_missing_features.lbr_flags)
1415                 evsel->attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1416                                      PERF_SAMPLE_BRANCH_NO_CYCLES);
1417 retry_sample_id:
1418         if (perf_missing_features.sample_id_all)
1419                 evsel->attr.sample_id_all = 0;
1420
1421         if (verbose >= 2) {
1422                 fprintf(stderr, "%.60s\n", graph_dotted_line);
1423                 fprintf(stderr, "perf_event_attr:\n");
1424                 perf_event_attr__fprintf(stderr, &evsel->attr, __open_attr__fprintf, NULL);
1425                 fprintf(stderr, "%.60s\n", graph_dotted_line);
1426         }
1427
1428         for (cpu = 0; cpu < cpus->nr; cpu++) {
1429
1430                 for (thread = 0; thread < nthreads; thread++) {
1431                         int group_fd;
1432
1433                         if (!evsel->cgrp && !evsel->system_wide)
1434                                 pid = thread_map__pid(threads, thread);
1435
1436                         group_fd = get_group_fd(evsel, cpu, thread);
1437 retry_open:
1438                         pr_debug2("sys_perf_event_open: pid %d  cpu %d  group_fd %d  flags %#lx\n",
1439                                   pid, cpus->map[cpu], group_fd, flags);
1440
1441                         FD(evsel, cpu, thread) = sys_perf_event_open(&evsel->attr,
1442                                                                      pid,
1443                                                                      cpus->map[cpu],
1444                                                                      group_fd, flags);
1445                         if (FD(evsel, cpu, thread) < 0) {
1446                                 err = -errno;
1447                                 pr_debug2("sys_perf_event_open failed, error %d\n",
1448                                           err);
1449                                 goto try_fallback;
1450                         }
1451
1452                         if (evsel->bpf_fd >= 0) {
1453                                 int evt_fd = FD(evsel, cpu, thread);
1454                                 int bpf_fd = evsel->bpf_fd;
1455
1456                                 err = ioctl(evt_fd,
1457                                             PERF_EVENT_IOC_SET_BPF,
1458                                             bpf_fd);
1459                                 if (err && errno != EEXIST) {
1460                                         pr_err("failed to attach bpf fd %d: %s\n",
1461                                                bpf_fd, strerror(errno));
1462                                         err = -EINVAL;
1463                                         goto out_close;
1464                                 }
1465                         }
1466
1467                         set_rlimit = NO_CHANGE;
1468
1469                         /*
1470                          * If we succeeded but had to kill clockid, fail and
1471                          * have perf_evsel__open_strerror() print us a nice
1472                          * error.
1473                          */
1474                         if (perf_missing_features.clockid ||
1475                             perf_missing_features.clockid_wrong) {
1476                                 err = -EINVAL;
1477                                 goto out_close;
1478                         }
1479                 }
1480         }
1481
1482         return 0;
1483
1484 try_fallback:
1485         /*
1486          * perf stat needs between 5 and 22 fds per CPU. When we run out
1487          * of them try to increase the limits.
1488          */
1489         if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1490                 struct rlimit l;
1491                 int old_errno = errno;
1492
1493                 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1494                         if (set_rlimit == NO_CHANGE)
1495                                 l.rlim_cur = l.rlim_max;
1496                         else {
1497                                 l.rlim_cur = l.rlim_max + 1000;
1498                                 l.rlim_max = l.rlim_cur;
1499                         }
1500                         if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1501                                 set_rlimit++;
1502                                 errno = old_errno;
1503                                 goto retry_open;
1504                         }
1505                 }
1506                 errno = old_errno;
1507         }
1508
1509         if (err != -EINVAL || cpu > 0 || thread > 0)
1510                 goto out_close;
1511
1512         /*
1513          * Must probe features in the order they were added to the
1514          * perf_event_attr interface.
1515          */
1516         if (!perf_missing_features.write_backward && evsel->attr.write_backward) {
1517                 perf_missing_features.write_backward = true;
1518                 goto out_close;
1519         } else if (!perf_missing_features.clockid_wrong && evsel->attr.use_clockid) {
1520                 perf_missing_features.clockid_wrong = true;
1521                 goto fallback_missing_features;
1522         } else if (!perf_missing_features.clockid && evsel->attr.use_clockid) {
1523                 perf_missing_features.clockid = true;
1524                 goto fallback_missing_features;
1525         } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1526                 perf_missing_features.cloexec = true;
1527                 goto fallback_missing_features;
1528         } else if (!perf_missing_features.mmap2 && evsel->attr.mmap2) {
1529                 perf_missing_features.mmap2 = true;
1530                 goto fallback_missing_features;
1531         } else if (!perf_missing_features.exclude_guest &&
1532                    (evsel->attr.exclude_guest || evsel->attr.exclude_host)) {
1533                 perf_missing_features.exclude_guest = true;
1534                 goto fallback_missing_features;
1535         } else if (!perf_missing_features.sample_id_all) {
1536                 perf_missing_features.sample_id_all = true;
1537                 goto retry_sample_id;
1538         } else if (!perf_missing_features.lbr_flags &&
1539                         (evsel->attr.branch_sample_type &
1540                          (PERF_SAMPLE_BRANCH_NO_CYCLES |
1541                           PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1542                 perf_missing_features.lbr_flags = true;
1543                 goto fallback_missing_features;
1544         }
1545 out_close:
1546         do {
1547                 while (--thread >= 0) {
1548                         close(FD(evsel, cpu, thread));
1549                         FD(evsel, cpu, thread) = -1;
1550                 }
1551                 thread = nthreads;
1552         } while (--cpu >= 0);
1553         return err;
1554 }
1555
1556 void perf_evsel__close(struct perf_evsel *evsel, int ncpus, int nthreads)
1557 {
1558         if (evsel->fd == NULL)
1559                 return;
1560
1561         perf_evsel__close_fd(evsel, ncpus, nthreads);
1562         perf_evsel__free_fd(evsel);
1563 }
1564
1565 static struct {
1566         struct cpu_map map;
1567         int cpus[1];
1568 } empty_cpu_map = {
1569         .map.nr = 1,
1570         .cpus   = { -1, },
1571 };
1572
1573 static struct {
1574         struct thread_map map;
1575         int threads[1];
1576 } empty_thread_map = {
1577         .map.nr  = 1,
1578         .threads = { -1, },
1579 };
1580
1581 int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
1582                      struct thread_map *threads)
1583 {
1584         if (cpus == NULL) {
1585                 /* Work around old compiler warnings about strict aliasing */
1586                 cpus = &empty_cpu_map.map;
1587         }
1588
1589         if (threads == NULL)
1590                 threads = &empty_thread_map.map;
1591
1592         return __perf_evsel__open(evsel, cpus, threads);
1593 }
1594
1595 int perf_evsel__open_per_cpu(struct perf_evsel *evsel,
1596                              struct cpu_map *cpus)
1597 {
1598         return __perf_evsel__open(evsel, cpus, &empty_thread_map.map);
1599 }
1600
1601 int perf_evsel__open_per_thread(struct perf_evsel *evsel,
1602                                 struct thread_map *threads)
1603 {
1604         return __perf_evsel__open(evsel, &empty_cpu_map.map, threads);
1605 }
1606
1607 static int perf_evsel__parse_id_sample(const struct perf_evsel *evsel,
1608                                        const union perf_event *event,
1609                                        struct perf_sample *sample)
1610 {
1611         u64 type = evsel->attr.sample_type;
1612         const u64 *array = event->sample.array;
1613         bool swapped = evsel->needs_swap;
1614         union u64_swap u;
1615
1616         array += ((event->header.size -
1617                    sizeof(event->header)) / sizeof(u64)) - 1;
1618
1619         if (type & PERF_SAMPLE_IDENTIFIER) {
1620                 sample->id = *array;
1621                 array--;
1622         }
1623
1624         if (type & PERF_SAMPLE_CPU) {
1625                 u.val64 = *array;
1626                 if (swapped) {
1627                         /* undo swap of u64, then swap on individual u32s */
1628                         u.val64 = bswap_64(u.val64);
1629                         u.val32[0] = bswap_32(u.val32[0]);
1630                 }
1631
1632                 sample->cpu = u.val32[0];
1633                 array--;
1634         }
1635
1636         if (type & PERF_SAMPLE_STREAM_ID) {
1637                 sample->stream_id = *array;
1638                 array--;
1639         }
1640
1641         if (type & PERF_SAMPLE_ID) {
1642                 sample->id = *array;
1643                 array--;
1644         }
1645
1646         if (type & PERF_SAMPLE_TIME) {
1647                 sample->time = *array;
1648                 array--;
1649         }
1650
1651         if (type & PERF_SAMPLE_TID) {
1652                 u.val64 = *array;
1653                 if (swapped) {
1654                         /* undo swap of u64, then swap on individual u32s */
1655                         u.val64 = bswap_64(u.val64);
1656                         u.val32[0] = bswap_32(u.val32[0]);
1657                         u.val32[1] = bswap_32(u.val32[1]);
1658                 }
1659
1660                 sample->pid = u.val32[0];
1661                 sample->tid = u.val32[1];
1662                 array--;
1663         }
1664
1665         return 0;
1666 }
1667
1668 static inline bool overflow(const void *endp, u16 max_size, const void *offset,
1669                             u64 size)
1670 {
1671         return size > max_size || offset + size > endp;
1672 }
1673
1674 #define OVERFLOW_CHECK(offset, size, max_size)                          \
1675         do {                                                            \
1676                 if (overflow(endp, (max_size), (offset), (size)))       \
1677                         return -EFAULT;                                 \
1678         } while (0)
1679
1680 #define OVERFLOW_CHECK_u64(offset) \
1681         OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
1682
1683 int perf_evsel__parse_sample(struct perf_evsel *evsel, union perf_event *event,
1684                              struct perf_sample *data)
1685 {
1686         u64 type = evsel->attr.sample_type;
1687         bool swapped = evsel->needs_swap;
1688         const u64 *array;
1689         u16 max_size = event->header.size;
1690         const void *endp = (void *)event + max_size;
1691         u64 sz;
1692
1693         /*
1694          * used for cross-endian analysis. See git commit 65014ab3
1695          * for why this goofiness is needed.
1696          */
1697         union u64_swap u;
1698
1699         memset(data, 0, sizeof(*data));
1700         data->cpu = data->pid = data->tid = -1;
1701         data->stream_id = data->id = data->time = -1ULL;
1702         data->period = evsel->attr.sample_period;
1703         data->weight = 0;
1704         data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
1705
1706         if (event->header.type != PERF_RECORD_SAMPLE) {
1707                 if (!evsel->attr.sample_id_all)
1708                         return 0;
1709                 return perf_evsel__parse_id_sample(evsel, event, data);
1710         }
1711
1712         array = event->sample.array;
1713
1714         /*
1715          * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
1716          * up to PERF_SAMPLE_PERIOD.  After that overflow() must be used to
1717          * check the format does not go past the end of the event.
1718          */
1719         if (evsel->sample_size + sizeof(event->header) > event->header.size)
1720                 return -EFAULT;
1721
1722         data->id = -1ULL;
1723         if (type & PERF_SAMPLE_IDENTIFIER) {
1724                 data->id = *array;
1725                 array++;
1726         }
1727
1728         if (type & PERF_SAMPLE_IP) {
1729                 data->ip = *array;
1730                 array++;
1731         }
1732
1733         if (type & PERF_SAMPLE_TID) {
1734                 u.val64 = *array;
1735                 if (swapped) {
1736                         /* undo swap of u64, then swap on individual u32s */
1737                         u.val64 = bswap_64(u.val64);
1738                         u.val32[0] = bswap_32(u.val32[0]);
1739                         u.val32[1] = bswap_32(u.val32[1]);
1740                 }
1741
1742                 data->pid = u.val32[0];
1743                 data->tid = u.val32[1];
1744                 array++;
1745         }
1746
1747         if (type & PERF_SAMPLE_TIME) {
1748                 data->time = *array;
1749                 array++;
1750         }
1751
1752         data->addr = 0;
1753         if (type & PERF_SAMPLE_ADDR) {
1754                 data->addr = *array;
1755                 array++;
1756         }
1757
1758         if (type & PERF_SAMPLE_ID) {
1759                 data->id = *array;
1760                 array++;
1761         }
1762
1763         if (type & PERF_SAMPLE_STREAM_ID) {
1764                 data->stream_id = *array;
1765                 array++;
1766         }
1767
1768         if (type & PERF_SAMPLE_CPU) {
1769
1770                 u.val64 = *array;
1771                 if (swapped) {
1772                         /* undo swap of u64, then swap on individual u32s */
1773                         u.val64 = bswap_64(u.val64);
1774                         u.val32[0] = bswap_32(u.val32[0]);
1775                 }
1776
1777                 data->cpu = u.val32[0];
1778                 array++;
1779         }
1780
1781         if (type & PERF_SAMPLE_PERIOD) {
1782                 data->period = *array;
1783                 array++;
1784         }
1785
1786         if (type & PERF_SAMPLE_READ) {
1787                 u64 read_format = evsel->attr.read_format;
1788
1789                 OVERFLOW_CHECK_u64(array);
1790                 if (read_format & PERF_FORMAT_GROUP)
1791                         data->read.group.nr = *array;
1792                 else
1793                         data->read.one.value = *array;
1794
1795                 array++;
1796
1797                 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1798                         OVERFLOW_CHECK_u64(array);
1799                         data->read.time_enabled = *array;
1800                         array++;
1801                 }
1802
1803                 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1804                         OVERFLOW_CHECK_u64(array);
1805                         data->read.time_running = *array;
1806                         array++;
1807                 }
1808
1809                 /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
1810                 if (read_format & PERF_FORMAT_GROUP) {
1811                         const u64 max_group_nr = UINT64_MAX /
1812                                         sizeof(struct sample_read_value);
1813
1814                         if (data->read.group.nr > max_group_nr)
1815                                 return -EFAULT;
1816                         sz = data->read.group.nr *
1817                              sizeof(struct sample_read_value);
1818                         OVERFLOW_CHECK(array, sz, max_size);
1819                         data->read.group.values =
1820                                         (struct sample_read_value *)array;
1821                         array = (void *)array + sz;
1822                 } else {
1823                         OVERFLOW_CHECK_u64(array);
1824                         data->read.one.id = *array;
1825                         array++;
1826                 }
1827         }
1828
1829         if (type & PERF_SAMPLE_CALLCHAIN) {
1830                 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
1831
1832                 OVERFLOW_CHECK_u64(array);
1833                 data->callchain = (struct ip_callchain *)array++;
1834                 if (data->callchain->nr > max_callchain_nr)
1835                         return -EFAULT;
1836                 sz = data->callchain->nr * sizeof(u64);
1837                 OVERFLOW_CHECK(array, sz, max_size);
1838                 array = (void *)array + sz;
1839         }
1840
1841         if (type & PERF_SAMPLE_RAW) {
1842                 OVERFLOW_CHECK_u64(array);
1843                 u.val64 = *array;
1844                 if (WARN_ONCE(swapped,
1845                               "Endianness of raw data not corrected!\n")) {
1846                         /* undo swap of u64, then swap on individual u32s */
1847                         u.val64 = bswap_64(u.val64);
1848                         u.val32[0] = bswap_32(u.val32[0]);
1849                         u.val32[1] = bswap_32(u.val32[1]);
1850                 }
1851                 data->raw_size = u.val32[0];
1852                 array = (void *)array + sizeof(u32);
1853
1854                 OVERFLOW_CHECK(array, data->raw_size, max_size);
1855                 data->raw_data = (void *)array;
1856                 array = (void *)array + data->raw_size;
1857         }
1858
1859         if (type & PERF_SAMPLE_BRANCH_STACK) {
1860                 const u64 max_branch_nr = UINT64_MAX /
1861                                           sizeof(struct branch_entry);
1862
1863                 OVERFLOW_CHECK_u64(array);
1864                 data->branch_stack = (struct branch_stack *)array++;
1865
1866                 if (data->branch_stack->nr > max_branch_nr)
1867                         return -EFAULT;
1868                 sz = data->branch_stack->nr * sizeof(struct branch_entry);
1869                 OVERFLOW_CHECK(array, sz, max_size);
1870                 array = (void *)array + sz;
1871         }
1872
1873         if (type & PERF_SAMPLE_REGS_USER) {
1874                 OVERFLOW_CHECK_u64(array);
1875                 data->user_regs.abi = *array;
1876                 array++;
1877
1878                 if (data->user_regs.abi) {
1879                         u64 mask = evsel->attr.sample_regs_user;
1880
1881                         sz = hweight_long(mask) * sizeof(u64);
1882                         OVERFLOW_CHECK(array, sz, max_size);
1883                         data->user_regs.mask = mask;
1884                         data->user_regs.regs = (u64 *)array;
1885                         array = (void *)array + sz;
1886                 }
1887         }
1888
1889         if (type & PERF_SAMPLE_STACK_USER) {
1890                 OVERFLOW_CHECK_u64(array);
1891                 sz = *array++;
1892
1893                 data->user_stack.offset = ((char *)(array - 1)
1894                                           - (char *) event);
1895
1896                 if (!sz) {
1897                         data->user_stack.size = 0;
1898                 } else {
1899                         OVERFLOW_CHECK(array, sz, max_size);
1900                         data->user_stack.data = (char *)array;
1901                         array = (void *)array + sz;
1902                         OVERFLOW_CHECK_u64(array);
1903                         data->user_stack.size = *array++;
1904                         if (WARN_ONCE(data->user_stack.size > sz,
1905                                       "user stack dump failure\n"))
1906                                 return -EFAULT;
1907                 }
1908         }
1909
1910         data->weight = 0;
1911         if (type & PERF_SAMPLE_WEIGHT) {
1912                 OVERFLOW_CHECK_u64(array);
1913                 data->weight = *array;
1914                 array++;
1915         }
1916
1917         data->data_src = PERF_MEM_DATA_SRC_NONE;
1918         if (type & PERF_SAMPLE_DATA_SRC) {
1919                 OVERFLOW_CHECK_u64(array);
1920                 data->data_src = *array;
1921                 array++;
1922         }
1923
1924         data->transaction = 0;
1925         if (type & PERF_SAMPLE_TRANSACTION) {
1926                 OVERFLOW_CHECK_u64(array);
1927                 data->transaction = *array;
1928                 array++;
1929         }
1930
1931         data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
1932         if (type & PERF_SAMPLE_REGS_INTR) {
1933                 OVERFLOW_CHECK_u64(array);
1934                 data->intr_regs.abi = *array;
1935                 array++;
1936
1937                 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
1938                         u64 mask = evsel->attr.sample_regs_intr;
1939
1940                         sz = hweight_long(mask) * sizeof(u64);
1941                         OVERFLOW_CHECK(array, sz, max_size);
1942                         data->intr_regs.mask = mask;
1943                         data->intr_regs.regs = (u64 *)array;
1944                         array = (void *)array + sz;
1945                 }
1946         }
1947
1948         return 0;
1949 }
1950
1951 size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type,
1952                                      u64 read_format)
1953 {
1954         size_t sz, result = sizeof(struct sample_event);
1955
1956         if (type & PERF_SAMPLE_IDENTIFIER)
1957                 result += sizeof(u64);
1958
1959         if (type & PERF_SAMPLE_IP)
1960                 result += sizeof(u64);
1961
1962         if (type & PERF_SAMPLE_TID)
1963                 result += sizeof(u64);
1964
1965         if (type & PERF_SAMPLE_TIME)
1966                 result += sizeof(u64);
1967
1968         if (type & PERF_SAMPLE_ADDR)
1969                 result += sizeof(u64);
1970
1971         if (type & PERF_SAMPLE_ID)
1972                 result += sizeof(u64);
1973
1974         if (type & PERF_SAMPLE_STREAM_ID)
1975                 result += sizeof(u64);
1976
1977         if (type & PERF_SAMPLE_CPU)
1978                 result += sizeof(u64);
1979
1980         if (type & PERF_SAMPLE_PERIOD)
1981                 result += sizeof(u64);
1982
1983         if (type & PERF_SAMPLE_READ) {
1984                 result += sizeof(u64);
1985                 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1986                         result += sizeof(u64);
1987                 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1988                         result += sizeof(u64);
1989                 /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
1990                 if (read_format & PERF_FORMAT_GROUP) {
1991                         sz = sample->read.group.nr *
1992                              sizeof(struct sample_read_value);
1993                         result += sz;
1994                 } else {
1995                         result += sizeof(u64);
1996                 }
1997         }
1998
1999         if (type & PERF_SAMPLE_CALLCHAIN) {
2000                 sz = (sample->callchain->nr + 1) * sizeof(u64);
2001                 result += sz;
2002         }
2003
2004         if (type & PERF_SAMPLE_RAW) {
2005                 result += sizeof(u32);
2006                 result += sample->raw_size;
2007         }
2008
2009         if (type & PERF_SAMPLE_BRANCH_STACK) {
2010                 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2011                 sz += sizeof(u64);
2012                 result += sz;
2013         }
2014
2015         if (type & PERF_SAMPLE_REGS_USER) {
2016                 if (sample->user_regs.abi) {
2017                         result += sizeof(u64);
2018                         sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
2019                         result += sz;
2020                 } else {
2021                         result += sizeof(u64);
2022                 }
2023         }
2024
2025         if (type & PERF_SAMPLE_STACK_USER) {
2026                 sz = sample->user_stack.size;
2027                 result += sizeof(u64);
2028                 if (sz) {
2029                         result += sz;
2030                         result += sizeof(u64);
2031                 }
2032         }
2033
2034         if (type & PERF_SAMPLE_WEIGHT)
2035                 result += sizeof(u64);
2036
2037         if (type & PERF_SAMPLE_DATA_SRC)
2038                 result += sizeof(u64);
2039
2040         if (type & PERF_SAMPLE_TRANSACTION)
2041                 result += sizeof(u64);
2042
2043         if (type & PERF_SAMPLE_REGS_INTR) {
2044                 if (sample->intr_regs.abi) {
2045                         result += sizeof(u64);
2046                         sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
2047                         result += sz;
2048                 } else {
2049                         result += sizeof(u64);
2050                 }
2051         }
2052
2053         return result;
2054 }
2055
2056 int perf_event__synthesize_sample(union perf_event *event, u64 type,
2057                                   u64 read_format,
2058                                   const struct perf_sample *sample,
2059                                   bool swapped)
2060 {
2061         u64 *array;
2062         size_t sz;
2063         /*
2064          * used for cross-endian analysis. See git commit 65014ab3
2065          * for why this goofiness is needed.
2066          */
2067         union u64_swap u;
2068
2069         array = event->sample.array;
2070
2071         if (type & PERF_SAMPLE_IDENTIFIER) {
2072                 *array = sample->id;
2073                 array++;
2074         }
2075
2076         if (type & PERF_SAMPLE_IP) {
2077                 *array = sample->ip;
2078                 array++;
2079         }
2080
2081         if (type & PERF_SAMPLE_TID) {
2082                 u.val32[0] = sample->pid;
2083                 u.val32[1] = sample->tid;
2084                 if (swapped) {
2085                         /*
2086                          * Inverse of what is done in perf_evsel__parse_sample
2087                          */
2088                         u.val32[0] = bswap_32(u.val32[0]);
2089                         u.val32[1] = bswap_32(u.val32[1]);
2090                         u.val64 = bswap_64(u.val64);
2091                 }
2092
2093                 *array = u.val64;
2094                 array++;
2095         }
2096
2097         if (type & PERF_SAMPLE_TIME) {
2098                 *array = sample->time;
2099                 array++;
2100         }
2101
2102         if (type & PERF_SAMPLE_ADDR) {
2103                 *array = sample->addr;
2104                 array++;
2105         }
2106
2107         if (type & PERF_SAMPLE_ID) {
2108                 *array = sample->id;
2109                 array++;
2110         }
2111
2112         if (type & PERF_SAMPLE_STREAM_ID) {
2113                 *array = sample->stream_id;
2114                 array++;
2115         }
2116
2117         if (type & PERF_SAMPLE_CPU) {
2118                 u.val32[0] = sample->cpu;
2119                 if (swapped) {
2120                         /*
2121                          * Inverse of what is done in perf_evsel__parse_sample
2122                          */
2123                         u.val32[0] = bswap_32(u.val32[0]);
2124                         u.val64 = bswap_64(u.val64);
2125                 }
2126                 *array = u.val64;
2127                 array++;
2128         }
2129
2130         if (type & PERF_SAMPLE_PERIOD) {
2131                 *array = sample->period;
2132                 array++;
2133         }
2134
2135         if (type & PERF_SAMPLE_READ) {
2136                 if (read_format & PERF_FORMAT_GROUP)
2137                         *array = sample->read.group.nr;
2138                 else
2139                         *array = sample->read.one.value;
2140                 array++;
2141
2142                 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2143                         *array = sample->read.time_enabled;
2144                         array++;
2145                 }
2146
2147                 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2148                         *array = sample->read.time_running;
2149                         array++;
2150                 }
2151
2152                 /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2153                 if (read_format & PERF_FORMAT_GROUP) {
2154                         sz = sample->read.group.nr *
2155                              sizeof(struct sample_read_value);
2156                         memcpy(array, sample->read.group.values, sz);
2157                         array = (void *)array + sz;
2158                 } else {
2159                         *array = sample->read.one.id;
2160                         array++;
2161                 }
2162         }
2163
2164         if (type & PERF_SAMPLE_CALLCHAIN) {
2165                 sz = (sample->callchain->nr + 1) * sizeof(u64);
2166                 memcpy(array, sample->callchain, sz);
2167                 array = (void *)array + sz;
2168         }
2169
2170         if (type & PERF_SAMPLE_RAW) {
2171                 u.val32[0] = sample->raw_size;
2172                 if (WARN_ONCE(swapped,
2173                               "Endianness of raw data not corrected!\n")) {
2174                         /*
2175                          * Inverse of what is done in perf_evsel__parse_sample
2176                          */
2177                         u.val32[0] = bswap_32(u.val32[0]);
2178                         u.val32[1] = bswap_32(u.val32[1]);
2179                         u.val64 = bswap_64(u.val64);
2180                 }
2181                 *array = u.val64;
2182                 array = (void *)array + sizeof(u32);
2183
2184                 memcpy(array, sample->raw_data, sample->raw_size);
2185                 array = (void *)array + sample->raw_size;
2186         }
2187
2188         if (type & PERF_SAMPLE_BRANCH_STACK) {
2189                 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2190                 sz += sizeof(u64);
2191                 memcpy(array, sample->branch_stack, sz);
2192                 array = (void *)array + sz;
2193         }
2194
2195         if (type & PERF_SAMPLE_REGS_USER) {
2196                 if (sample->user_regs.abi) {
2197                         *array++ = sample->user_regs.abi;
2198                         sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
2199                         memcpy(array, sample->user_regs.regs, sz);
2200                         array = (void *)array + sz;
2201                 } else {
2202                         *array++ = 0;
2203                 }
2204         }
2205
2206         if (type & PERF_SAMPLE_STACK_USER) {
2207                 sz = sample->user_stack.size;
2208                 *array++ = sz;
2209                 if (sz) {
2210                         memcpy(array, sample->user_stack.data, sz);
2211                         array = (void *)array + sz;
2212                         *array++ = sz;
2213                 }
2214         }
2215
2216         if (type & PERF_SAMPLE_WEIGHT) {
2217                 *array = sample->weight;
2218                 array++;
2219         }
2220
2221         if (type & PERF_SAMPLE_DATA_SRC) {
2222                 *array = sample->data_src;
2223                 array++;
2224         }
2225
2226         if (type & PERF_SAMPLE_TRANSACTION) {
2227                 *array = sample->transaction;
2228                 array++;
2229         }
2230
2231         if (type & PERF_SAMPLE_REGS_INTR) {
2232                 if (sample->intr_regs.abi) {
2233                         *array++ = sample->intr_regs.abi;
2234                         sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
2235                         memcpy(array, sample->intr_regs.regs, sz);
2236                         array = (void *)array + sz;
2237                 } else {
2238                         *array++ = 0;
2239                 }
2240         }
2241
2242         return 0;
2243 }
2244
2245 struct format_field *perf_evsel__field(struct perf_evsel *evsel, const char *name)
2246 {
2247         return pevent_find_field(evsel->tp_format, name);
2248 }
2249
2250 void *perf_evsel__rawptr(struct perf_evsel *evsel, struct perf_sample *sample,
2251                          const char *name)
2252 {
2253         struct format_field *field = perf_evsel__field(evsel, name);
2254         int offset;
2255
2256         if (!field)
2257                 return NULL;
2258
2259         offset = field->offset;
2260
2261         if (field->flags & FIELD_IS_DYNAMIC) {
2262                 offset = *(int *)(sample->raw_data + field->offset);
2263                 offset &= 0xffff;
2264         }
2265
2266         return sample->raw_data + offset;
2267 }
2268
2269 u64 format_field__intval(struct format_field *field, struct perf_sample *sample,
2270                          bool needs_swap)
2271 {
2272         u64 value;
2273         void *ptr = sample->raw_data + field->offset;
2274
2275         switch (field->size) {
2276         case 1:
2277                 return *(u8 *)ptr;
2278         case 2:
2279                 value = *(u16 *)ptr;
2280                 break;
2281         case 4:
2282                 value = *(u32 *)ptr;
2283                 break;
2284         case 8:
2285                 memcpy(&value, ptr, sizeof(u64));
2286                 break;
2287         default:
2288                 return 0;
2289         }
2290
2291         if (!needs_swap)
2292                 return value;
2293
2294         switch (field->size) {
2295         case 2:
2296                 return bswap_16(value);
2297         case 4:
2298                 return bswap_32(value);
2299         case 8:
2300                 return bswap_64(value);
2301         default:
2302                 return 0;
2303         }
2304
2305         return 0;
2306 }
2307
2308 u64 perf_evsel__intval(struct perf_evsel *evsel, struct perf_sample *sample,
2309                        const char *name)
2310 {
2311         struct format_field *field = perf_evsel__field(evsel, name);
2312
2313         if (!field)
2314                 return 0;
2315
2316         return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2317 }
2318
2319 bool perf_evsel__fallback(struct perf_evsel *evsel, int err,
2320                           char *msg, size_t msgsize)
2321 {
2322         int paranoid;
2323
2324         if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2325             evsel->attr.type   == PERF_TYPE_HARDWARE &&
2326             evsel->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2327                 /*
2328                  * If it's cycles then fall back to hrtimer based
2329                  * cpu-clock-tick sw counter, which is always available even if
2330                  * no PMU support.
2331                  *
2332                  * PPC returns ENXIO until 2.6.37 (behavior changed with commit
2333                  * b0a873e).
2334                  */
2335                 scnprintf(msg, msgsize, "%s",
2336 "The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2337
2338                 evsel->attr.type   = PERF_TYPE_SOFTWARE;
2339                 evsel->attr.config = PERF_COUNT_SW_CPU_CLOCK;
2340
2341                 zfree(&evsel->name);
2342                 return true;
2343         } else if (err == EACCES && !evsel->attr.exclude_kernel &&
2344                    (paranoid = perf_event_paranoid()) > 1) {
2345                 const char *name = perf_evsel__name(evsel);
2346                 char *new_name;
2347
2348                 if (asprintf(&new_name, "%s%su", name, strchr(name, ':') ? "" : ":") < 0)
2349                         return false;
2350
2351                 if (evsel->name)
2352                         free(evsel->name);
2353                 evsel->name = new_name;
2354                 scnprintf(msg, msgsize,
2355 "kernel.perf_event_paranoid=%d, trying to fall back to excluding kernel samples", paranoid);
2356                 evsel->attr.exclude_kernel = 1;
2357
2358                 return true;
2359         }
2360
2361         return false;
2362 }
2363
2364 int perf_evsel__open_strerror(struct perf_evsel *evsel, struct target *target,
2365                               int err, char *msg, size_t size)
2366 {
2367         char sbuf[STRERR_BUFSIZE];
2368
2369         switch (err) {
2370         case EPERM:
2371         case EACCES:
2372                 return scnprintf(msg, size,
2373                  "You may not have permission to collect %sstats.\n\n"
2374                  "Consider tweaking /proc/sys/kernel/perf_event_paranoid,\n"
2375                  "which controls use of the performance events system by\n"
2376                  "unprivileged users (without CAP_SYS_ADMIN).\n\n"
2377                  "The current value is %d:\n\n"
2378                  "  -1: Allow use of (almost) all events by all users\n"
2379                  ">= 0: Disallow raw tracepoint access by users without CAP_IOC_LOCK\n"
2380                  ">= 1: Disallow CPU event access by users without CAP_SYS_ADMIN\n"
2381                  ">= 2: Disallow kernel profiling by users without CAP_SYS_ADMIN",
2382                                  target->system_wide ? "system-wide " : "",
2383                                  perf_event_paranoid());
2384         case ENOENT:
2385                 return scnprintf(msg, size, "The %s event is not supported.",
2386                                  perf_evsel__name(evsel));
2387         case EMFILE:
2388                 return scnprintf(msg, size, "%s",
2389                          "Too many events are opened.\n"
2390                          "Probably the maximum number of open file descriptors has been reached.\n"
2391                          "Hint: Try again after reducing the number of events.\n"
2392                          "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2393         case ENOMEM:
2394                 if ((evsel->attr.sample_type & PERF_SAMPLE_CALLCHAIN) != 0 &&
2395                     access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2396                         return scnprintf(msg, size,
2397                                          "Not enough memory to setup event with callchain.\n"
2398                                          "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2399                                          "Hint: Current value: %d", sysctl_perf_event_max_stack);
2400                 break;
2401         case ENODEV:
2402                 if (target->cpu_list)
2403                         return scnprintf(msg, size, "%s",
2404          "No such device - did you specify an out-of-range profile CPU?");
2405                 break;
2406         case EOPNOTSUPP:
2407                 if (evsel->attr.sample_period != 0)
2408                         return scnprintf(msg, size, "%s",
2409         "PMU Hardware doesn't support sampling/overflow-interrupts.");
2410                 if (evsel->attr.precise_ip)
2411                         return scnprintf(msg, size, "%s",
2412         "\'precise\' request may not be supported. Try removing 'p' modifier.");
2413 #if defined(__i386__) || defined(__x86_64__)
2414                 if (evsel->attr.type == PERF_TYPE_HARDWARE)
2415                         return scnprintf(msg, size, "%s",
2416         "No hardware sampling interrupt available.\n"
2417         "No APIC? If so then you can boot the kernel with the \"lapic\" boot parameter to force-enable it.");
2418 #endif
2419                 break;
2420         case EBUSY:
2421                 if (find_process("oprofiled"))
2422                         return scnprintf(msg, size,
2423         "The PMU counters are busy/taken by another profiler.\n"
2424         "We found oprofile daemon running, please stop it and try again.");
2425                 break;
2426         case EINVAL:
2427                 if (evsel->attr.write_backward && perf_missing_features.write_backward)
2428                         return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2429                 if (perf_missing_features.clockid)
2430                         return scnprintf(msg, size, "clockid feature not supported.");
2431                 if (perf_missing_features.clockid_wrong)
2432                         return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2433                 break;
2434         default:
2435                 break;
2436         }
2437
2438         return scnprintf(msg, size,
2439         "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2440         "/bin/dmesg may provide additional information.\n"
2441         "No CONFIG_PERF_EVENTS=y kernel support configured?",
2442                          err, str_error_r(err, sbuf, sizeof(sbuf)),
2443                          perf_evsel__name(evsel));
2444 }
2445
2446 char *perf_evsel__env_arch(struct perf_evsel *evsel)
2447 {
2448         if (evsel && evsel->evlist && evsel->evlist->env)
2449                 return evsel->evlist->env->arch;
2450         return NULL;
2451 }