tools/perf/util/jitdump.c

   1 #include <sys/sysmacros.h>
   2 #include <sys/types.h>
   3 #include <stdio.h>
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <inttypes.h>
   9 #include <byteswap.h>
  10 #include <sys/stat.h>
  11 #include <sys/mman.h>
  12
  13 #include "util.h"
  14 #include "event.h"
  15 #include "debug.h"
  16 #include "evlist.h"
  17 #include "symbol.h"
  18 #include "strlist.h"
  19 #include <elf.h>
  20
  21 #include "tsc.h"
  22 #include "session.h"
  23 #include "jit.h"
  24 #include "jitdump.h"
  25 #include "genelf.h"
  26 #include "../builtin.h"
  27
  28 struct jit_buf_desc {
  29         struct perf_data_file *output;
  30         struct perf_session *session;
  31         struct machine *machine;
  32         union jr_entry   *entry;
  33         void             *buf;
  34         uint64_t         sample_type;
  35         size_t           bufsize;
  36         FILE             *in;
  37         bool             needs_bswap; /* handles cross-endianess */
  38         bool             use_arch_timestamp;
  39         void             *debug_data;
  40         size_t           nr_debug_entries;
  41         uint32_t         code_load_count;
  42         u64              bytes_written;
  43         struct rb_root   code_root;
  44         char             dir[PATH_MAX];
  45 };
  46
  47 struct debug_line_info {
  48         unsigned long vma;
  49         unsigned int lineno;
  50         /* The filename format is unspecified, absolute path, relative etc. */
  51         char const filename[0];
  52 };
  53
  54 struct jit_tool {
  55         struct perf_tool tool;
  56         struct perf_data_file   output;
  57         struct perf_data_file   input;
  58         u64 bytes_written;
  59 };
  60
  61 #define hmax(a, b) ((a) > (b) ? (a) : (b))
  62 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
  63
  64 static int
  65 jit_emit_elf(char *filename,
  66              const char *sym,
  67              uint64_t code_addr,
  68              const void *code,
  69              int csize,
  70              void *debug,
  71              int nr_debug_entries)
  72 {
  73         int ret, fd;
  74
  75         if (verbose > 0)
  76                 fprintf(stderr, "write ELF image %s\n", filename);
  77
  78         fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644);
  79         if (fd == -1) {
  80                 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno));
  81                 return -1;
  82         }
  83
  84         ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries);
  85
  86         close(fd);
  87
  88         if (ret)
  89                 unlink(filename);
  90
  91         return ret;
  92 }
  93
  94 static void
  95 jit_close(struct jit_buf_desc *jd)
  96 {
  97         if (!(jd && jd->in))
  98                 return;
  99         funlockfile(jd->in);
 100         fclose(jd->in);
 101         jd->in = NULL;
 102 }
 103
 104 static int
 105 jit_validate_events(struct perf_session *session)
 106 {
 107         struct perf_evsel *evsel;
 108
 109         /*
 110          * check that all events use CLOCK_MONOTONIC
 111          */
 112         evlist__for_each_entry(session->evlist, evsel) {
 113                 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC)
 114                         return -1;
 115         }
 116         return 0;
 117 }
 118
 119 static int
 120 jit_open(struct jit_buf_desc *jd, const char *name)
 121 {
 122         struct jitheader header;
 123         struct jr_prefix *prefix;
 124         ssize_t bs, bsz = 0;
 125         void *n, *buf = NULL;
 126         int ret, retval = -1;
 127
 128         jd->in = fopen(name, "r");
 129         if (!jd->in)
 130                 return -1;
 131
 132         bsz = hmax(sizeof(header), sizeof(*prefix));
 133
 134         buf = malloc(bsz);
 135         if (!buf)
 136                 goto error;
 137
 138         /*
 139          * protect from writer modifying the file while we are reading it
 140          */
 141         flockfile(jd->in);
 142
 143         ret = fread(buf, sizeof(header), 1, jd->in);
 144         if (ret != 1)
 145                 goto error;
 146
 147         memcpy(&header, buf, sizeof(header));
 148
 149         if (header.magic != JITHEADER_MAGIC) {
 150                 if (header.magic != JITHEADER_MAGIC_SW)
 151                         goto error;
 152                 jd->needs_bswap = true;
 153         }
 154
 155         if (jd->needs_bswap) {
 156                 header.version    = bswap_32(header.version);
 157                 header.total_size = bswap_32(header.total_size);
 158                 header.pid        = bswap_32(header.pid);
 159                 header.elf_mach   = bswap_32(header.elf_mach);
 160                 header.timestamp  = bswap_64(header.timestamp);
 161                 header.flags      = bswap_64(header.flags);
 162         }
 163
 164         jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP;
 165
 166         if (verbose > 2)
 167                 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
 168                         header.version,
 169                         header.total_size,
 170                         (unsigned long long)header.timestamp,
 171                         header.pid,
 172                         header.elf_mach,
 173                         jd->use_arch_timestamp);
 174
 175         if (header.flags & JITDUMP_FLAGS_RESERVED) {
 176                 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
 177                        (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED);
 178                 goto error;
 179         }
 180
 181         if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) {
 182                 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
 183                 goto error;
 184         }
 185
 186         /*
 187          * validate event is using the correct clockid
 188          */
 189         if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) {
 190                 pr_err("error, jitted code must be sampled with perf record -k 1\n");
 191                 goto error;
 192         }
 193
 194         bs = header.total_size - sizeof(header);
 195
 196         if (bs > bsz) {
 197                 n = realloc(buf, bs);
 198                 if (!n)
 199                         goto error;
 200                 bsz = bs;
 201                 buf = n;
 202                 /* read extra we do not know about */
 203                 ret = fread(buf, bs - bsz, 1, jd->in);
 204                 if (ret != 1)
 205                         goto error;
 206         }
 207         /*
 208          * keep dirname for generating files and mmap records
 209          */
 210         strcpy(jd->dir, name);
 211         dirname(jd->dir);
 212
 213         return 0;
 214 error:
 215         funlockfile(jd->in);
 216         fclose(jd->in);
 217         return retval;
 218 }
 219
 220 static union jr_entry *
 221 jit_get_next_entry(struct jit_buf_desc *jd)
 222 {
 223         struct jr_prefix *prefix;
 224         union jr_entry *jr;
 225         void *addr;
 226         size_t bs, size;
 227         int id, ret;
 228
 229         if (!(jd && jd->in))
 230                 return NULL;
 231
 232         if (jd->buf == NULL) {
 233                 size_t sz = getpagesize();
 234                 if (sz < sizeof(*prefix))
 235                         sz = sizeof(*prefix);
 236
 237                 jd->buf = malloc(sz);
 238                 if (jd->buf == NULL)
 239                         return NULL;
 240
 241                 jd->bufsize = sz;
 242         }
 243
 244         prefix = jd->buf;
 245
 246         /*
 247          * file is still locked at this point
 248          */
 249         ret = fread(prefix, sizeof(*prefix), 1, jd->in);
 250         if (ret  != 1)
 251                 return NULL;
 252
 253         if (jd->needs_bswap) {
 254                 prefix->id         = bswap_32(prefix->id);
 255                 prefix->total_size = bswap_32(prefix->total_size);
 256                 prefix->timestamp  = bswap_64(prefix->timestamp);
 257         }
 258         id   = prefix->id;
 259         size = prefix->total_size;
 260
 261         bs = (size_t)size;
 262         if (bs < sizeof(*prefix))
 263                 return NULL;
 264
 265         if (id >= JIT_CODE_MAX) {
 266                 pr_warning("next_entry: unknown prefix %d, skipping\n", id);
 267                 return NULL;
 268         }
 269         if (bs > jd->bufsize) {
 270                 void *n;
 271                 n = realloc(jd->buf, bs);
 272                 if (!n)
 273                         return NULL;
 274                 jd->buf = n;
 275                 jd->bufsize = bs;
 276         }
 277
 278         addr = ((void *)jd->buf) + sizeof(*prefix);
 279
 280         ret = fread(addr, bs - sizeof(*prefix), 1, jd->in);
 281         if (ret != 1)
 282                 return NULL;
 283
 284         jr = (union jr_entry *)jd->buf;
 285
 286         switch(id) {
 287         case JIT_CODE_DEBUG_INFO:
 288                 if (jd->needs_bswap) {
 289                         uint64_t n;
 290                         jr->info.code_addr = bswap_64(jr->info.code_addr);
 291                         jr->info.nr_entry  = bswap_64(jr->info.nr_entry);
 292                         for (n = 0 ; n < jr->info.nr_entry; n++) {
 293                                 jr->info.entries[n].addr    = bswap_64(jr->info.entries[n].addr);
 294                                 jr->info.entries[n].lineno  = bswap_32(jr->info.entries[n].lineno);
 295                                 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim);
 296                         }
 297                 }
 298                 break;
 299         case JIT_CODE_CLOSE:
 300                 break;
 301         case JIT_CODE_LOAD:
 302                 if (jd->needs_bswap) {
 303                         jr->load.pid       = bswap_32(jr->load.pid);
 304                         jr->load.tid       = bswap_32(jr->load.tid);
 305                         jr->load.vma       = bswap_64(jr->load.vma);
 306                         jr->load.code_addr = bswap_64(jr->load.code_addr);
 307                         jr->load.code_size = bswap_64(jr->load.code_size);
 308                         jr->load.code_index= bswap_64(jr->load.code_index);
 309                 }
 310                 jd->code_load_count++;
 311                 break;
 312         case JIT_CODE_MOVE:
 313                 if (jd->needs_bswap) {
 314                         jr->move.pid           = bswap_32(jr->move.pid);
 315                         jr->move.tid           = bswap_32(jr->move.tid);
 316                         jr->move.vma           = bswap_64(jr->move.vma);
 317                         jr->move.old_code_addr = bswap_64(jr->move.old_code_addr);
 318                         jr->move.new_code_addr = bswap_64(jr->move.new_code_addr);
 319                         jr->move.code_size     = bswap_64(jr->move.code_size);
 320                         jr->move.code_index    = bswap_64(jr->move.code_index);
 321                 }
 322                 break;
 323         case JIT_CODE_MAX:
 324         default:
 325                 return NULL;
 326         }
 327         return jr;
 328 }
 329
 330 static int
 331 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event)
 332 {
 333         ssize_t size;
 334
 335         size = perf_data_file__write(jd->output, event, event->header.size);
 336         if (size < 0)
 337                 return -1;
 338
 339         jd->bytes_written += size;
 340         return 0;
 341 }
 342
 343 static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp)
 344 {
 345         struct perf_tsc_conversion tc;
 346
 347         if (!jd->use_arch_timestamp)
 348                 return timestamp;
 349
 350         tc.time_shift = jd->session->time_conv.time_shift;
 351         tc.time_mult  = jd->session->time_conv.time_mult;
 352         tc.time_zero  = jd->session->time_conv.time_zero;
 353
 354         if (!tc.time_mult)
 355                 return 0;
 356
 357         return tsc_to_perf_time(timestamp, &tc);
 358 }
 359
 360 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr)
 361 {
 362         struct perf_sample sample;
 363         union perf_event *event;
 364         struct perf_tool *tool = jd->session->tool;
 365         uint64_t code, addr;
 366         uintptr_t uaddr;
 367         char *filename;
 368         struct stat st;
 369         size_t size;
 370         u16 idr_size;
 371         const char *sym;
 372         uint32_t count;
 373         int ret, csize;
 374         pid_t pid, tid;
 375         struct {
 376                 u32 pid, tid;
 377                 u64 time;
 378         } *id;
 379
 380         pid   = jr->load.pid;
 381         tid   = jr->load.tid;
 382         csize = jr->load.code_size;
 383         addr  = jr->load.code_addr;
 384         sym   = (void *)((unsigned long)jr + sizeof(jr->load));
 385         code  = (unsigned long)jr + jr->load.p.total_size - csize;
 386         count = jr->load.code_index;
 387         idr_size = jd->machine->id_hdr_size;
 388
 389         event = calloc(1, sizeof(*event) + idr_size);
 390         if (!event)
 391                 return -1;
 392
 393         filename = event->mmap2.filename;
 394         size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so",
 395                         jd->dir,
 396                         pid,
 397                         count);
 398
 399         size++; /* for \0 */
 400
 401         size = PERF_ALIGN(size, sizeof(u64));
 402         uaddr = (uintptr_t)code;
 403         ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries);
 404
 405         if (jd->debug_data && jd->nr_debug_entries) {
 406                 free(jd->debug_data);
 407                 jd->debug_data = NULL;
 408                 jd->nr_debug_entries = 0;
 409         }
 410
 411         if (ret) {
 412                 free(event);
 413                 return -1;
 414         }
 415         if (stat(filename, &st))
 416                 memset(&st, 0, sizeof(st));
 417
 418         event->mmap2.header.type = PERF_RECORD_MMAP2;
 419         event->mmap2.header.misc = PERF_RECORD_MISC_USER;
 420         event->mmap2.header.size = (sizeof(event->mmap2) -
 421                         (sizeof(event->mmap2.filename) - size) + idr_size);
 422
 423         event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
 424         event->mmap2.start = addr;
 425         event->mmap2.len   = csize;
 426         event->mmap2.pid   = pid;
 427         event->mmap2.tid   = tid;
 428         event->mmap2.ino   = st.st_ino;
 429         event->mmap2.maj   = major(st.st_dev);
 430         event->mmap2.min   = minor(st.st_dev);
 431         event->mmap2.prot  = st.st_mode;
 432         event->mmap2.flags = MAP_SHARED;
 433         event->mmap2.ino_generation = 1;
 434
 435         id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
 436         if (jd->sample_type & PERF_SAMPLE_TID) {
 437                 id->pid  = pid;
 438                 id->tid  = tid;
 439         }
 440         if (jd->sample_type & PERF_SAMPLE_TIME)
 441                 id->time = convert_timestamp(jd, jr->load.p.timestamp);
 442
 443         /*
 444          * create pseudo sample to induce dso hit increment
 445          * use first address as sample address
 446          */
 447         memset(&sample, 0, sizeof(sample));
 448         sample.cpumode = PERF_RECORD_MISC_USER;
 449         sample.pid  = pid;
 450         sample.tid  = tid;
 451         sample.time = id->time;
 452         sample.ip   = addr;
 453
 454         ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
 455         if (ret)
 456                 return ret;
 457
 458         ret = jit_inject_event(jd, event);
 459         /*
 460          * mark dso as use to generate buildid in the header
 461          */
 462         if (!ret)
 463                 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
 464
 465         return ret;
 466 }
 467
 468 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr)
 469 {
 470         struct perf_sample sample;
 471         union perf_event *event;
 472         struct perf_tool *tool = jd->session->tool;
 473         char *filename;
 474         size_t size;
 475         struct stat st;
 476         u16 idr_size;
 477         int ret;
 478         pid_t pid, tid;
 479         struct {
 480                 u32 pid, tid;
 481                 u64 time;
 482         } *id;
 483
 484         pid = jr->move.pid;
 485         tid =  jr->move.tid;
 486         idr_size = jd->machine->id_hdr_size;
 487
 488         /*
 489          * +16 to account for sample_id_all (hack)
 490          */
 491         event = calloc(1, sizeof(*event) + 16);
 492         if (!event)
 493                 return -1;
 494
 495         filename = event->mmap2.filename;
 496         size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64,
 497                  jd->dir,
 498                  pid,
 499                  jr->move.code_index);
 500
 501         size++; /* for \0 */
 502
 503         if (stat(filename, &st))
 504                 memset(&st, 0, sizeof(st));
 505
 506         size = PERF_ALIGN(size, sizeof(u64));
 507
 508         event->mmap2.header.type = PERF_RECORD_MMAP2;
 509         event->mmap2.header.misc = PERF_RECORD_MISC_USER;
 510         event->mmap2.header.size = (sizeof(event->mmap2) -
 511                         (sizeof(event->mmap2.filename) - size) + idr_size);
 512         event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
 513         event->mmap2.start = jr->move.new_code_addr;
 514         event->mmap2.len   = jr->move.code_size;
 515         event->mmap2.pid   = pid;
 516         event->mmap2.tid   = tid;
 517         event->mmap2.ino   = st.st_ino;
 518         event->mmap2.maj   = major(st.st_dev);
 519         event->mmap2.min   = minor(st.st_dev);
 520         event->mmap2.prot  = st.st_mode;
 521         event->mmap2.flags = MAP_SHARED;
 522         event->mmap2.ino_generation = 1;
 523
 524         id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
 525         if (jd->sample_type & PERF_SAMPLE_TID) {
 526                 id->pid  = pid;
 527                 id->tid  = tid;
 528         }
 529         if (jd->sample_type & PERF_SAMPLE_TIME)
 530                 id->time = convert_timestamp(jd, jr->load.p.timestamp);
 531
 532         /*
 533          * create pseudo sample to induce dso hit increment
 534          * use first address as sample address
 535          */
 536         memset(&sample, 0, sizeof(sample));
 537         sample.cpumode = PERF_RECORD_MISC_USER;
 538         sample.pid  = pid;
 539         sample.tid  = tid;
 540         sample.time = id->time;
 541         sample.ip   = jr->move.new_code_addr;
 542
 543         ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
 544         if (ret)
 545                 return ret;
 546
 547         ret = jit_inject_event(jd, event);
 548         if (!ret)
 549                 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
 550
 551         return ret;
 552 }
 553
 554 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr)
 555 {
 556         void *data;
 557         size_t sz;
 558
 559         if (!(jd && jr))
 560                 return -1;
 561
 562         sz  = jr->prefix.total_size - sizeof(jr->info);
 563         data = malloc(sz);
 564         if (!data)
 565                 return -1;
 566
 567         memcpy(data, &jr->info.entries, sz);
 568
 569         jd->debug_data       = data;
 570
 571         /*
 572          * we must use nr_entry instead of size here because
 573          * we cannot distinguish actual entry from padding otherwise
 574          */
 575         jd->nr_debug_entries = jr->info.nr_entry;
 576
 577         return 0;
 578 }
 579
 580 static int
 581 jit_process_dump(struct jit_buf_desc *jd)
 582 {
 583         union jr_entry *jr;
 584         int ret;
 585
 586         while ((jr = jit_get_next_entry(jd))) {
 587                 switch(jr->prefix.id) {
 588                 case JIT_CODE_LOAD:
 589                         ret = jit_repipe_code_load(jd, jr);
 590                         break;
 591                 case JIT_CODE_MOVE:
 592                         ret = jit_repipe_code_move(jd, jr);
 593                         break;
 594                 case JIT_CODE_DEBUG_INFO:
 595                         ret = jit_repipe_debug_info(jd, jr);
 596                         break;
 597                 default:
 598                         ret = 0;
 599                         continue;
 600                 }
 601         }
 602         return ret;
 603 }
 604
 605 static int
 606 jit_inject(struct jit_buf_desc *jd, char *path)
 607 {
 608         int ret;
 609
 610         if (verbose > 0)
 611                 fprintf(stderr, "injecting: %s\n", path);
 612
 613         ret = jit_open(jd, path);
 614         if (ret)
 615                 return -1;
 616
 617         ret = jit_process_dump(jd);
 618
 619         jit_close(jd);
 620
 621         if (verbose > 0)
 622                 fprintf(stderr, "injected: %s (%d)\n", path, ret);
 623
 624         return 0;
 625 }
 626
 627 /*
 628  * File must be with pattern .../jit-XXXX.dump
 629  * where XXXX is the PID of the process which did the mmap()
 630  * as captured in the RECORD_MMAP record
 631  */
 632 static int
 633 jit_detect(char *mmap_name, pid_t pid)
 634  {
 635         char *p;
 636         char *end = NULL;
 637         pid_t pid2;
 638
 639         if (verbose > 2)
 640                 fprintf(stderr, "jit marker trying : %s\n", mmap_name);
 641         /*
 642          * get file name
 643          */
 644         p = strrchr(mmap_name, '/');
 645         if (!p)
 646                 return -1;
 647
 648         /*
 649          * match prefix
 650          */
 651         if (strncmp(p, "/jit-", 5))
 652                 return -1;
 653
 654         /*
 655          * skip prefix
 656          */
 657         p += 5;
 658
 659         /*
 660          * must be followed by a pid
 661          */
 662         if (!isdigit(*p))
 663                 return -1;
 664
 665         pid2 = (int)strtol(p, &end, 10);
 666         if (!end)
 667                 return -1;
 668
 669         /*
 670          * pid does not match mmap pid
 671          * pid==0 in system-wide mode (synthesized)
 672          */
 673         if (pid && pid2 != pid)
 674                 return -1;
 675         /*
 676          * validate suffix
 677          */
 678         if (strcmp(end, ".dump"))
 679                 return -1;
 680
 681         if (verbose > 0)
 682                 fprintf(stderr, "jit marker found: %s\n", mmap_name);
 683
 684         return 0;
 685 }
 686
 687 int
 688 jit_process(struct perf_session *session,
 689             struct perf_data_file *output,
 690             struct machine *machine,
 691             char *filename,
 692             pid_t pid,
 693             u64 *nbytes)
 694 {
 695         struct perf_evsel *first;
 696         struct jit_buf_desc jd;
 697         int ret;
 698
 699         /*
 700          * first, detect marker mmap (i.e., the jitdump mmap)
 701          */
 702         if (jit_detect(filename, pid))
 703                 return 0;
 704
 705         memset(&jd, 0, sizeof(jd));
 706
 707         jd.session = session;
 708         jd.output  = output;
 709         jd.machine = machine;
 710
 711         /*
 712          * track sample_type to compute id_all layout
 713          * perf sets the same sample type to all events as of now
 714          */
 715         first = perf_evlist__first(session->evlist);
 716         jd.sample_type = first->attr.sample_type;
 717
 718         *nbytes = 0;
 719
 720         ret = jit_inject(&jd, filename);
 721         if (!ret) {
 722                 *nbytes = jd.bytes_written;
 723                 ret = 1;
 724         }
 725
 726         return ret;
 727 }