tools/perf/util/bpf-loader.c

   1 /*
   2  * bpf-loader.c
   3  *
   4  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   5  * Copyright (C) 2015 Huawei Inc.
   6  */
   7
   8 #include <linux/bpf.h>
   9 #include <bpf/libbpf.h>
  10 #include <bpf/bpf.h>
  11 #include <linux/err.h>
  12 #include <linux/string.h>
  13 #include "perf.h"
  14 #include "debug.h"
  15 #include "bpf-loader.h"
  16 #include "bpf-prologue.h"
  17 #include "llvm-utils.h"
  18 #include "probe-event.h"
  19 #include "probe-finder.h" // for MAX_PROBES
  20 #include "parse-events.h"
  21 #include "llvm-utils.h"
  22
  23 #define DEFINE_PRINT_FN(name, level) \
  24 static int libbpf_##name(const char *fmt, ...)  \
  25 {                                               \
  26         va_list args;                           \
  27         int ret;                                \
  28                                                 \
  29         va_start(args, fmt);                    \
  30         ret = veprintf(level, verbose, pr_fmt(fmt), args);\
  31         va_end(args);                           \
  32         return ret;                             \
  33 }
  34
  35 DEFINE_PRINT_FN(warning, 1)
  36 DEFINE_PRINT_FN(info, 1)
  37 DEFINE_PRINT_FN(debug, 1)
  38
  39 struct bpf_prog_priv {
  40         bool is_tp;
  41         char *sys_name;
  42         char *evt_name;
  43         struct perf_probe_event pev;
  44         bool need_prologue;
  45         struct bpf_insn *insns_buf;
  46         int nr_types;
  47         int *type_mapping;
  48 };
  49
  50 static bool libbpf_initialized;
  51
  52 struct bpf_object *
  53 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  54 {
  55         struct bpf_object *obj;
  56
  57         if (!libbpf_initialized) {
  58                 libbpf_set_print(libbpf_warning,
  59                                  libbpf_info,
  60                                  libbpf_debug);
  61                 libbpf_initialized = true;
  62         }
  63
  64         obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  65         if (IS_ERR(obj)) {
  66                 pr_debug("bpf: failed to load buffer\n");
  67                 return ERR_PTR(-EINVAL);
  68         }
  69
  70         return obj;
  71 }
  72
  73 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  74 {
  75         struct bpf_object *obj;
  76
  77         if (!libbpf_initialized) {
  78                 libbpf_set_print(libbpf_warning,
  79                                  libbpf_info,
  80                                  libbpf_debug);
  81                 libbpf_initialized = true;
  82         }
  83
  84         if (source) {
  85                 int err;
  86                 void *obj_buf;
  87                 size_t obj_buf_sz;
  88
  89                 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  90                 if (err)
  91                         return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  92                 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  93                 free(obj_buf);
  94         } else
  95                 obj = bpf_object__open(filename);
  96
  97         if (IS_ERR(obj)) {
  98                 pr_debug("bpf: failed to load %s\n", filename);
  99                 return obj;
 100         }
 101
 102         return obj;
 103 }
 104
 105 void bpf__clear(void)
 106 {
 107         struct bpf_object *obj, *tmp;
 108
 109         bpf_object__for_each_safe(obj, tmp) {
 110                 bpf__unprobe(obj);
 111                 bpf_object__close(obj);
 112         }
 113 }
 114
 115 static void
 116 clear_prog_priv(struct bpf_program *prog __maybe_unused,
 117                 void *_priv)
 118 {
 119         struct bpf_prog_priv *priv = _priv;
 120
 121         cleanup_perf_probe_events(&priv->pev, 1);
 122         zfree(&priv->insns_buf);
 123         zfree(&priv->type_mapping);
 124         zfree(&priv->sys_name);
 125         zfree(&priv->evt_name);
 126         free(priv);
 127 }
 128
 129 static int
 130 prog_config__exec(const char *value, struct perf_probe_event *pev)
 131 {
 132         pev->uprobes = true;
 133         pev->target = strdup(value);
 134         if (!pev->target)
 135                 return -ENOMEM;
 136         return 0;
 137 }
 138
 139 static int
 140 prog_config__module(const char *value, struct perf_probe_event *pev)
 141 {
 142         pev->uprobes = false;
 143         pev->target = strdup(value);
 144         if (!pev->target)
 145                 return -ENOMEM;
 146         return 0;
 147 }
 148
 149 static int
 150 prog_config__bool(const char *value, bool *pbool, bool invert)
 151 {
 152         int err;
 153         bool bool_value;
 154
 155         if (!pbool)
 156                 return -EINVAL;
 157
 158         err = strtobool(value, &bool_value);
 159         if (err)
 160                 return err;
 161
 162         *pbool = invert ? !bool_value : bool_value;
 163         return 0;
 164 }
 165
 166 static int
 167 prog_config__inlines(const char *value,
 168                      struct perf_probe_event *pev __maybe_unused)
 169 {
 170         return prog_config__bool(value, &probe_conf.no_inlines, true);
 171 }
 172
 173 static int
 174 prog_config__force(const char *value,
 175                    struct perf_probe_event *pev __maybe_unused)
 176 {
 177         return prog_config__bool(value, &probe_conf.force_add, false);
 178 }
 179
 180 static struct {
 181         const char *key;
 182         const char *usage;
 183         const char *desc;
 184         int (*func)(const char *, struct perf_probe_event *);
 185 } bpf_prog_config_terms[] = {
 186         {
 187                 .key    = "exec",
 188                 .usage  = "exec=<full path of file>",
 189                 .desc   = "Set uprobe target",
 190                 .func   = prog_config__exec,
 191         },
 192         {
 193                 .key    = "module",
 194                 .usage  = "module=<module name>    ",
 195                 .desc   = "Set kprobe module",
 196                 .func   = prog_config__module,
 197         },
 198         {
 199                 .key    = "inlines",
 200                 .usage  = "inlines=[yes|no]        ",
 201                 .desc   = "Probe at inline symbol",
 202                 .func   = prog_config__inlines,
 203         },
 204         {
 205                 .key    = "force",
 206                 .usage  = "force=[yes|no]          ",
 207                 .desc   = "Forcibly add events with existing name",
 208                 .func   = prog_config__force,
 209         },
 210 };
 211
 212 static int
 213 do_prog_config(const char *key, const char *value,
 214                struct perf_probe_event *pev)
 215 {
 216         unsigned int i;
 217
 218         pr_debug("config bpf program: %s=%s\n", key, value);
 219         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 220                 if (strcmp(key, bpf_prog_config_terms[i].key) == 0)
 221                         return bpf_prog_config_terms[i].func(value, pev);
 222
 223         pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
 224                  key, value);
 225
 226         pr_debug("\nHint: Valid options are:\n");
 227         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 228                 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage,
 229                          bpf_prog_config_terms[i].desc);
 230         pr_debug("\n");
 231
 232         return -BPF_LOADER_ERRNO__PROGCONF_TERM;
 233 }
 234
 235 static const char *
 236 parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 237 {
 238         char *text = strdup(config_str);
 239         char *sep, *line;
 240         const char *main_str = NULL;
 241         int err = 0;
 242
 243         if (!text) {
 244                 pr_debug("No enough memory: dup config_str failed\n");
 245                 return ERR_PTR(-ENOMEM);
 246         }
 247
 248         line = text;
 249         while ((sep = strchr(line, ';'))) {
 250                 char *equ;
 251
 252                 *sep = '\0';
 253                 equ = strchr(line, '=');
 254                 if (!equ) {
 255                         pr_warning("WARNING: invalid config in BPF object: %s\n",
 256                                    line);
 257                         pr_warning("\tShould be 'key=value'.\n");
 258                         goto nextline;
 259                 }
 260                 *equ = '\0';
 261
 262                 err = do_prog_config(line, equ + 1, pev);
 263                 if (err)
 264                         break;
 265 nextline:
 266                 line = sep + 1;
 267         }
 268
 269         if (!err)
 270                 main_str = config_str + (line - text);
 271         free(text);
 272
 273         return err ? ERR_PTR(err) : main_str;
 274 }
 275
 276 static int
 277 parse_prog_config(const char *config_str, const char **p_main_str,
 278                   bool *is_tp, struct perf_probe_event *pev)
 279 {
 280         int err;
 281         const char *main_str = parse_prog_config_kvpair(config_str, pev);
 282
 283         if (IS_ERR(main_str))
 284                 return PTR_ERR(main_str);
 285
 286         *p_main_str = main_str;
 287         if (!strchr(main_str, '=')) {
 288                 /* Is a tracepoint event? */
 289                 const char *s = strchr(main_str, ':');
 290
 291                 if (!s) {
 292                         pr_debug("bpf: '%s' is not a valid tracepoint\n",
 293                                  config_str);
 294                         return -BPF_LOADER_ERRNO__CONFIG;
 295                 }
 296
 297                 *is_tp = true;
 298                 return 0;
 299         }
 300
 301         *is_tp = false;
 302         err = parse_perf_probe_command(main_str, pev);
 303         if (err < 0) {
 304                 pr_debug("bpf: '%s' is not a valid config string\n",
 305                          config_str);
 306                 /* parse failed, don't need clear pev. */
 307                 return -BPF_LOADER_ERRNO__CONFIG;
 308         }
 309         return 0;
 310 }
 311
 312 static int
 313 config_bpf_program(struct bpf_program *prog)
 314 {
 315         struct perf_probe_event *pev = NULL;
 316         struct bpf_prog_priv *priv = NULL;
 317         const char *config_str, *main_str;
 318         bool is_tp = false;
 319         int err;
 320
 321         /* Initialize per-program probing setting */
 322         probe_conf.no_inlines = false;
 323         probe_conf.force_add = false;
 324
 325         config_str = bpf_program__title(prog, false);
 326         if (IS_ERR(config_str)) {
 327                 pr_debug("bpf: unable to get title for program\n");
 328                 return PTR_ERR(config_str);
 329         }
 330
 331         priv = calloc(sizeof(*priv), 1);
 332         if (!priv) {
 333                 pr_debug("bpf: failed to alloc priv\n");
 334                 return -ENOMEM;
 335         }
 336         pev = &priv->pev;
 337
 338         pr_debug("bpf: config program '%s'\n", config_str);
 339         err = parse_prog_config(config_str, &main_str, &is_tp, pev);
 340         if (err)
 341                 goto errout;
 342
 343         if (is_tp) {
 344                 char *s = strchr(main_str, ':');
 345
 346                 priv->is_tp = true;
 347                 priv->sys_name = strndup(main_str, s - main_str);
 348                 priv->evt_name = strdup(s + 1);
 349                 goto set_priv;
 350         }
 351
 352         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 353                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 354                          config_str, PERF_BPF_PROBE_GROUP);
 355                 err = -BPF_LOADER_ERRNO__GROUP;
 356                 goto errout;
 357         } else if (!pev->group)
 358                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 359
 360         if (!pev->group) {
 361                 pr_debug("bpf: strdup failed\n");
 362                 err = -ENOMEM;
 363                 goto errout;
 364         }
 365
 366         if (!pev->event) {
 367                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 368                          config_str);
 369                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 370                 goto errout;
 371         }
 372         pr_debug("bpf: config '%s' is ok\n", config_str);
 373
 374 set_priv:
 375         err = bpf_program__set_priv(prog, priv, clear_prog_priv);
 376         if (err) {
 377                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 378                 goto errout;
 379         }
 380
 381         return 0;
 382
 383 errout:
 384         if (pev)
 385                 clear_perf_probe_event(pev);
 386         free(priv);
 387         return err;
 388 }
 389
 390 static int bpf__prepare_probe(void)
 391 {
 392         static int err = 0;
 393         static bool initialized = false;
 394
 395         /*
 396          * Make err static, so if init failed the first, bpf__prepare_probe()
 397          * fails each time without calling init_probe_symbol_maps multiple
 398          * times.
 399          */
 400         if (initialized)
 401                 return err;
 402
 403         initialized = true;
 404         err = init_probe_symbol_maps(false);
 405         if (err < 0)
 406                 pr_debug("Failed to init_probe_symbol_maps\n");
 407         probe_conf.max_probes = MAX_PROBES;
 408         return err;
 409 }
 410
 411 static int
 412 preproc_gen_prologue(struct bpf_program *prog, int n,
 413                      struct bpf_insn *orig_insns, int orig_insns_cnt,
 414                      struct bpf_prog_prep_result *res)
 415 {
 416         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 417         struct probe_trace_event *tev;
 418         struct perf_probe_event *pev;
 419         struct bpf_insn *buf;
 420         size_t prologue_cnt = 0;
 421         int i, err;
 422
 423         if (IS_ERR(priv) || !priv || priv->is_tp)
 424                 goto errout;
 425
 426         pev = &priv->pev;
 427
 428         if (n < 0 || n >= priv->nr_types)
 429                 goto errout;
 430
 431         /* Find a tev belongs to that type */
 432         for (i = 0; i < pev->ntevs; i++) {
 433                 if (priv->type_mapping[i] == n)
 434                         break;
 435         }
 436
 437         if (i >= pev->ntevs) {
 438                 pr_debug("Internal error: prologue type %d not found\n", n);
 439                 return -BPF_LOADER_ERRNO__PROLOGUE;
 440         }
 441
 442         tev = &pev->tevs[i];
 443
 444         buf = priv->insns_buf;
 445         err = bpf__gen_prologue(tev->args, tev->nargs,
 446                                 buf, &prologue_cnt,
 447                                 BPF_MAXINSNS - orig_insns_cnt);
 448         if (err) {
 449                 const char *title;
 450
 451                 title = bpf_program__title(prog, false);
 452                 if (!title)
 453                         title = "[unknown]";
 454
 455                 pr_debug("Failed to generate prologue for program %s\n",
 456                          title);
 457                 return err;
 458         }
 459
 460         memcpy(&buf[prologue_cnt], orig_insns,
 461                sizeof(struct bpf_insn) * orig_insns_cnt);
 462
 463         res->new_insn_ptr = buf;
 464         res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
 465         res->pfd = NULL;
 466         return 0;
 467
 468 errout:
 469         pr_debug("Internal error in preproc_gen_prologue\n");
 470         return -BPF_LOADER_ERRNO__PROLOGUE;
 471 }
 472
 473 /*
 474  * compare_tev_args is reflexive, transitive and antisymmetric.
 475  * I can proof it but this margin is too narrow to contain.
 476  */
 477 static int compare_tev_args(const void *ptev1, const void *ptev2)
 478 {
 479         int i, ret;
 480         const struct probe_trace_event *tev1 =
 481                 *(const struct probe_trace_event **)ptev1;
 482         const struct probe_trace_event *tev2 =
 483                 *(const struct probe_trace_event **)ptev2;
 484
 485         ret = tev2->nargs - tev1->nargs;
 486         if (ret)
 487                 return ret;
 488
 489         for (i = 0; i < tev1->nargs; i++) {
 490                 struct probe_trace_arg *arg1, *arg2;
 491                 struct probe_trace_arg_ref *ref1, *ref2;
 492
 493                 arg1 = &tev1->args[i];
 494                 arg2 = &tev2->args[i];
 495
 496                 ret = strcmp(arg1->value, arg2->value);
 497                 if (ret)
 498                         return ret;
 499
 500                 ref1 = arg1->ref;
 501                 ref2 = arg2->ref;
 502
 503                 while (ref1 && ref2) {
 504                         ret = ref2->offset - ref1->offset;
 505                         if (ret)
 506                                 return ret;
 507
 508                         ref1 = ref1->next;
 509                         ref2 = ref2->next;
 510                 }
 511
 512                 if (ref1 || ref2)
 513                         return ref2 ? 1 : -1;
 514         }
 515
 516         return 0;
 517 }
 518
 519 /*
 520  * Assign a type number to each tevs in a pev.
 521  * mapping is an array with same slots as tevs in that pev.
 522  * nr_types will be set to number of types.
 523  */
 524 static int map_prologue(struct perf_probe_event *pev, int *mapping,
 525                         int *nr_types)
 526 {
 527         int i, type = 0;
 528         struct probe_trace_event **ptevs;
 529
 530         size_t array_sz = sizeof(*ptevs) * pev->ntevs;
 531
 532         ptevs = malloc(array_sz);
 533         if (!ptevs) {
 534                 pr_debug("No enough memory: alloc ptevs failed\n");
 535                 return -ENOMEM;
 536         }
 537
 538         pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs);
 539         for (i = 0; i < pev->ntevs; i++)
 540                 ptevs[i] = &pev->tevs[i];
 541
 542         qsort(ptevs, pev->ntevs, sizeof(*ptevs),
 543               compare_tev_args);
 544
 545         for (i = 0; i < pev->ntevs; i++) {
 546                 int n;
 547
 548                 n = ptevs[i] - pev->tevs;
 549                 if (i == 0) {
 550                         mapping[n] = type;
 551                         pr_debug("mapping[%d]=%d\n", n, type);
 552                         continue;
 553                 }
 554
 555                 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0)
 556                         mapping[n] = type;
 557                 else
 558                         mapping[n] = ++type;
 559
 560                 pr_debug("mapping[%d]=%d\n", n, mapping[n]);
 561         }
 562         free(ptevs);
 563         *nr_types = type + 1;
 564
 565         return 0;
 566 }
 567
 568 static int hook_load_preprocessor(struct bpf_program *prog)
 569 {
 570         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 571         struct perf_probe_event *pev;
 572         bool need_prologue = false;
 573         int err, i;
 574
 575         if (IS_ERR(priv) || !priv) {
 576                 pr_debug("Internal error when hook preprocessor\n");
 577                 return -BPF_LOADER_ERRNO__INTERNAL;
 578         }
 579
 580         if (priv->is_tp) {
 581                 priv->need_prologue = false;
 582                 return 0;
 583         }
 584
 585         pev = &priv->pev;
 586         for (i = 0; i < pev->ntevs; i++) {
 587                 struct probe_trace_event *tev = &pev->tevs[i];
 588
 589                 if (tev->nargs > 0) {
 590                         need_prologue = true;
 591                         break;
 592                 }
 593         }
 594
 595         /*
 596          * Since all tevs don't have argument, we don't need generate
 597          * prologue.
 598          */
 599         if (!need_prologue) {
 600                 priv->need_prologue = false;
 601                 return 0;
 602         }
 603
 604         priv->need_prologue = true;
 605         priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
 606         if (!priv->insns_buf) {
 607                 pr_debug("No enough memory: alloc insns_buf failed\n");
 608                 return -ENOMEM;
 609         }
 610
 611         priv->type_mapping = malloc(sizeof(int) * pev->ntevs);
 612         if (!priv->type_mapping) {
 613                 pr_debug("No enough memory: alloc type_mapping failed\n");
 614                 return -ENOMEM;
 615         }
 616         memset(priv->type_mapping, -1,
 617                sizeof(int) * pev->ntevs);
 618
 619         err = map_prologue(pev, priv->type_mapping, &priv->nr_types);
 620         if (err)
 621                 return err;
 622
 623         err = bpf_program__set_prep(prog, priv->nr_types,
 624                                     preproc_gen_prologue);
 625         return err;
 626 }
 627
 628 int bpf__probe(struct bpf_object *obj)
 629 {
 630         int err = 0;
 631         struct bpf_program *prog;
 632         struct bpf_prog_priv *priv;
 633         struct perf_probe_event *pev;
 634
 635         err = bpf__prepare_probe();
 636         if (err) {
 637                 pr_debug("bpf__prepare_probe failed\n");
 638                 return err;
 639         }
 640
 641         bpf_object__for_each_program(prog, obj) {
 642                 err = config_bpf_program(prog);
 643                 if (err)
 644                         goto out;
 645
 646                 priv = bpf_program__priv(prog);
 647                 if (IS_ERR(priv) || !priv) {
 648                         err = PTR_ERR(priv);
 649                         goto out;
 650                 }
 651
 652                 if (priv->is_tp) {
 653                         bpf_program__set_tracepoint(prog);
 654                         continue;
 655                 }
 656
 657                 bpf_program__set_kprobe(prog);
 658                 pev = &priv->pev;
 659
 660                 err = convert_perf_probe_events(pev, 1);
 661                 if (err < 0) {
 662                         pr_debug("bpf_probe: failed to convert perf probe events");
 663                         goto out;
 664                 }
 665
 666                 err = apply_perf_probe_events(pev, 1);
 667                 if (err < 0) {
 668                         pr_debug("bpf_probe: failed to apply perf probe events");
 669                         goto out;
 670                 }
 671
 672                 /*
 673                  * After probing, let's consider prologue, which
 674                  * adds program fetcher to BPF programs.
 675                  *
 676                  * hook_load_preprocessorr() hooks pre-processor
 677                  * to bpf_program, let it generate prologue
 678                  * dynamically during loading.
 679                  */
 680                 err = hook_load_preprocessor(prog);
 681                 if (err)
 682                         goto out;
 683         }
 684 out:
 685         return err < 0 ? err : 0;
 686 }
 687
 688 #define EVENTS_WRITE_BUFSIZE  4096
 689 int bpf__unprobe(struct bpf_object *obj)
 690 {
 691         int err, ret = 0;
 692         struct bpf_program *prog;
 693
 694         bpf_object__for_each_program(prog, obj) {
 695                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 696                 int i;
 697
 698                 if (IS_ERR(priv) || !priv || priv->is_tp)
 699                         continue;
 700
 701                 for (i = 0; i < priv->pev.ntevs; i++) {
 702                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 703                         char name_buf[EVENTS_WRITE_BUFSIZE];
 704                         struct strfilter *delfilter;
 705
 706                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 707                                  "%s:%s", tev->group, tev->event);
 708                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 709
 710                         delfilter = strfilter__new(name_buf, NULL);
 711                         if (!delfilter) {
 712                                 pr_debug("Failed to create filter for unprobing\n");
 713                                 ret = -ENOMEM;
 714                                 continue;
 715                         }
 716
 717                         err = del_perf_probe_events(delfilter);
 718                         strfilter__delete(delfilter);
 719                         if (err) {
 720                                 pr_debug("Failed to delete %s\n", name_buf);
 721                                 ret = err;
 722                                 continue;
 723                         }
 724                 }
 725         }
 726         return ret;
 727 }
 728
 729 int bpf__load(struct bpf_object *obj)
 730 {
 731         int err;
 732
 733         err = bpf_object__load(obj);
 734         if (err) {
 735                 pr_debug("bpf: load objects failed\n");
 736                 return err;
 737         }
 738         return 0;
 739 }
 740
 741 int bpf__foreach_event(struct bpf_object *obj,
 742                        bpf_prog_iter_callback_t func,
 743                        void *arg)
 744 {
 745         struct bpf_program *prog;
 746         int err;
 747
 748         bpf_object__for_each_program(prog, obj) {
 749                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 750                 struct probe_trace_event *tev;
 751                 struct perf_probe_event *pev;
 752                 int i, fd;
 753
 754                 if (IS_ERR(priv) || !priv) {
 755                         pr_debug("bpf: failed to get private field\n");
 756                         return -BPF_LOADER_ERRNO__INTERNAL;
 757                 }
 758
 759                 if (priv->is_tp) {
 760                         fd = bpf_program__fd(prog);
 761                         err = (*func)(priv->sys_name, priv->evt_name, fd, arg);
 762                         if (err) {
 763                                 pr_debug("bpf: tracepoint call back failed, stop iterate\n");
 764                                 return err;
 765                         }
 766                         continue;
 767                 }
 768
 769                 pev = &priv->pev;
 770                 for (i = 0; i < pev->ntevs; i++) {
 771                         tev = &pev->tevs[i];
 772
 773                         if (priv->need_prologue) {
 774                                 int type = priv->type_mapping[i];
 775
 776                                 fd = bpf_program__nth_fd(prog, type);
 777                         } else {
 778                                 fd = bpf_program__fd(prog);
 779                         }
 780
 781                         if (fd < 0) {
 782                                 pr_debug("bpf: failed to get file descriptor\n");
 783                                 return fd;
 784                         }
 785
 786                         err = (*func)(tev->group, tev->event, fd, arg);
 787                         if (err) {
 788                                 pr_debug("bpf: call back failed, stop iterate\n");
 789                                 return err;
 790                         }
 791                 }
 792         }
 793         return 0;
 794 }
 795
 796 enum bpf_map_op_type {
 797         BPF_MAP_OP_SET_VALUE,
 798         BPF_MAP_OP_SET_EVSEL,
 799 };
 800
 801 enum bpf_map_key_type {
 802         BPF_MAP_KEY_ALL,
 803         BPF_MAP_KEY_RANGES,
 804 };
 805
 806 struct bpf_map_op {
 807         struct list_head list;
 808         enum bpf_map_op_type op_type;
 809         enum bpf_map_key_type key_type;
 810         union {
 811                 struct parse_events_array array;
 812         } k;
 813         union {
 814                 u64 value;
 815                 struct perf_evsel *evsel;
 816         } v;
 817 };
 818
 819 struct bpf_map_priv {
 820         struct list_head ops_list;
 821 };
 822
 823 static void
 824 bpf_map_op__delete(struct bpf_map_op *op)
 825 {
 826         if (!list_empty(&op->list))
 827                 list_del(&op->list);
 828         if (op->key_type == BPF_MAP_KEY_RANGES)
 829                 parse_events__clear_array(&op->k.array);
 830         free(op);
 831 }
 832
 833 static void
 834 bpf_map_priv__purge(struct bpf_map_priv *priv)
 835 {
 836         struct bpf_map_op *pos, *n;
 837
 838         list_for_each_entry_safe(pos, n, &priv->ops_list, list) {
 839                 list_del_init(&pos->list);
 840                 bpf_map_op__delete(pos);
 841         }
 842 }
 843
 844 static void
 845 bpf_map_priv__clear(struct bpf_map *map __maybe_unused,
 846                     void *_priv)
 847 {
 848         struct bpf_map_priv *priv = _priv;
 849
 850         bpf_map_priv__purge(priv);
 851         free(priv);
 852 }
 853
 854 static int
 855 bpf_map_op_setkey(struct bpf_map_op *op, struct parse_events_term *term)
 856 {
 857         op->key_type = BPF_MAP_KEY_ALL;
 858         if (!term)
 859                 return 0;
 860
 861         if (term->array.nr_ranges) {
 862                 size_t memsz = term->array.nr_ranges *
 863                                 sizeof(op->k.array.ranges[0]);
 864
 865                 op->k.array.ranges = memdup(term->array.ranges, memsz);
 866                 if (!op->k.array.ranges) {
 867                         pr_debug("No enough memory to alloc indices for map\n");
 868                         return -ENOMEM;
 869                 }
 870                 op->key_type = BPF_MAP_KEY_RANGES;
 871                 op->k.array.nr_ranges = term->array.nr_ranges;
 872         }
 873         return 0;
 874 }
 875
 876 static struct bpf_map_op *
 877 bpf_map_op__new(struct parse_events_term *term)
 878 {
 879         struct bpf_map_op *op;
 880         int err;
 881
 882         op = zalloc(sizeof(*op));
 883         if (!op) {
 884                 pr_debug("Failed to alloc bpf_map_op\n");
 885                 return ERR_PTR(-ENOMEM);
 886         }
 887         INIT_LIST_HEAD(&op->list);
 888
 889         err = bpf_map_op_setkey(op, term);
 890         if (err) {
 891                 free(op);
 892                 return ERR_PTR(err);
 893         }
 894         return op;
 895 }
 896
 897 static struct bpf_map_op *
 898 bpf_map_op__clone(struct bpf_map_op *op)
 899 {
 900         struct bpf_map_op *newop;
 901
 902         newop = memdup(op, sizeof(*op));
 903         if (!newop) {
 904                 pr_debug("Failed to alloc bpf_map_op\n");
 905                 return NULL;
 906         }
 907
 908         INIT_LIST_HEAD(&newop->list);
 909         if (op->key_type == BPF_MAP_KEY_RANGES) {
 910                 size_t memsz = op->k.array.nr_ranges *
 911                                sizeof(op->k.array.ranges[0]);
 912
 913                 newop->k.array.ranges = memdup(op->k.array.ranges, memsz);
 914                 if (!newop->k.array.ranges) {
 915                         pr_debug("Failed to alloc indices for map\n");
 916                         free(newop);
 917                         return NULL;
 918                 }
 919         }
 920
 921         return newop;
 922 }
 923
 924 static struct bpf_map_priv *
 925 bpf_map_priv__clone(struct bpf_map_priv *priv)
 926 {
 927         struct bpf_map_priv *newpriv;
 928         struct bpf_map_op *pos, *newop;
 929
 930         newpriv = zalloc(sizeof(*newpriv));
 931         if (!newpriv) {
 932                 pr_debug("No enough memory to alloc map private\n");
 933                 return NULL;
 934         }
 935         INIT_LIST_HEAD(&newpriv->ops_list);
 936
 937         list_for_each_entry(pos, &priv->ops_list, list) {
 938                 newop = bpf_map_op__clone(pos);
 939                 if (!newop) {
 940                         bpf_map_priv__purge(newpriv);
 941                         return NULL;
 942                 }
 943                 list_add_tail(&newop->list, &newpriv->ops_list);
 944         }
 945
 946         return newpriv;
 947 }
 948
 949 static int
 950 bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op)
 951 {
 952         const char *map_name = bpf_map__name(map);
 953         struct bpf_map_priv *priv = bpf_map__priv(map);
 954
 955         if (IS_ERR(priv)) {
 956                 pr_debug("Failed to get private from map %s\n", map_name);
 957                 return PTR_ERR(priv);
 958         }
 959
 960         if (!priv) {
 961                 priv = zalloc(sizeof(*priv));
 962                 if (!priv) {
 963                         pr_debug("No enough memory to alloc map private\n");
 964                         return -ENOMEM;
 965                 }
 966                 INIT_LIST_HEAD(&priv->ops_list);
 967
 968                 if (bpf_map__set_priv(map, priv, bpf_map_priv__clear)) {
 969                         free(priv);
 970                         return -BPF_LOADER_ERRNO__INTERNAL;
 971                 }
 972         }
 973
 974         list_add_tail(&op->list, &priv->ops_list);
 975         return 0;
 976 }
 977
 978 static struct bpf_map_op *
 979 bpf_map__add_newop(struct bpf_map *map, struct parse_events_term *term)
 980 {
 981         struct bpf_map_op *op;
 982         int err;
 983
 984         op = bpf_map_op__new(term);
 985         if (IS_ERR(op))
 986                 return op;
 987
 988         err = bpf_map__add_op(map, op);
 989         if (err) {
 990                 bpf_map_op__delete(op);
 991                 return ERR_PTR(err);
 992         }
 993         return op;
 994 }
 995
 996 static int
 997 __bpf_map__config_value(struct bpf_map *map,
 998                         struct parse_events_term *term)
 999 {
1000         struct bpf_map_op *op;
1001         const char *map_name = bpf_map__name(map);
1002         const struct bpf_map_def *def = bpf_map__def(map);
1003
1004         if (IS_ERR(def)) {
1005                 pr_debug("Unable to get map definition from '%s'\n",
1006                          map_name);
1007                 return -BPF_LOADER_ERRNO__INTERNAL;
1008         }
1009
1010         if (def->type != BPF_MAP_TYPE_ARRAY) {
1011                 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
1012                          map_name);
1013                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1014         }
1015         if (def->key_size < sizeof(unsigned int)) {
1016                 pr_debug("Map %s has incorrect key size\n", map_name);
1017                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE;
1018         }
1019         switch (def->value_size) {
1020         case 1:
1021         case 2:
1022         case 4:
1023         case 8:
1024                 break;
1025         default:
1026                 pr_debug("Map %s has incorrect value size\n", map_name);
1027                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1028         }
1029
1030         op = bpf_map__add_newop(map, term);
1031         if (IS_ERR(op))
1032                 return PTR_ERR(op);
1033         op->op_type = BPF_MAP_OP_SET_VALUE;
1034         op->v.value = term->val.num;
1035         return 0;
1036 }
1037
1038 static int
1039 bpf_map__config_value(struct bpf_map *map,
1040                       struct parse_events_term *term,
1041                       struct perf_evlist *evlist __maybe_unused)
1042 {
1043         if (!term->err_val) {
1044                 pr_debug("Config value not set\n");
1045                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1046         }
1047
1048         if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) {
1049                 pr_debug("ERROR: wrong value type for 'value'\n");
1050                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1051         }
1052
1053         return __bpf_map__config_value(map, term);
1054 }
1055
1056 static int
1057 __bpf_map__config_event(struct bpf_map *map,
1058                         struct parse_events_term *term,
1059                         struct perf_evlist *evlist)
1060 {
1061         struct perf_evsel *evsel;
1062         const struct bpf_map_def *def;
1063         struct bpf_map_op *op;
1064         const char *map_name = bpf_map__name(map);
1065
1066         evsel = perf_evlist__find_evsel_by_str(evlist, term->val.str);
1067         if (!evsel) {
1068                 pr_debug("Event (for '%s') '%s' doesn't exist\n",
1069                          map_name, term->val.str);
1070                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT;
1071         }
1072
1073         def = bpf_map__def(map);
1074         if (IS_ERR(def)) {
1075                 pr_debug("Unable to get map definition from '%s'\n",
1076                          map_name);
1077                 return PTR_ERR(def);
1078         }
1079
1080         /*
1081          * No need to check key_size and value_size:
1082          * kernel has already checked them.
1083          */
1084         if (def->type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
1085                 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
1086                          map_name);
1087                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1088         }
1089
1090         op = bpf_map__add_newop(map, term);
1091         if (IS_ERR(op))
1092                 return PTR_ERR(op);
1093         op->op_type = BPF_MAP_OP_SET_EVSEL;
1094         op->v.evsel = evsel;
1095         return 0;
1096 }
1097
1098 static int
1099 bpf_map__config_event(struct bpf_map *map,
1100                       struct parse_events_term *term,
1101                       struct perf_evlist *evlist)
1102 {
1103         if (!term->err_val) {
1104                 pr_debug("Config value not set\n");
1105                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1106         }
1107
1108         if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) {
1109                 pr_debug("ERROR: wrong value type for 'event'\n");
1110                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1111         }
1112
1113         return __bpf_map__config_event(map, term, evlist);
1114 }
1115
1116 struct bpf_obj_config__map_func {
1117         const char *config_opt;
1118         int (*config_func)(struct bpf_map *, struct parse_events_term *,
1119                            struct perf_evlist *);
1120 };
1121
1122 struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = {
1123         {"value", bpf_map__config_value},
1124         {"event", bpf_map__config_event},
1125 };
1126
1127 static int
1128 config_map_indices_range_check(struct parse_events_term *term,
1129                                struct bpf_map *map,
1130                                const char *map_name)
1131 {
1132         struct parse_events_array *array = &term->array;
1133         const struct bpf_map_def *def;
1134         unsigned int i;
1135
1136         if (!array->nr_ranges)
1137                 return 0;
1138         if (!array->ranges) {
1139                 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1140                          map_name, (int)array->nr_ranges);
1141                 return -BPF_LOADER_ERRNO__INTERNAL;
1142         }
1143
1144         def = bpf_map__def(map);
1145         if (IS_ERR(def)) {
1146                 pr_debug("ERROR: Unable to get map definition from '%s'\n",
1147                          map_name);
1148                 return -BPF_LOADER_ERRNO__INTERNAL;
1149         }
1150
1151         for (i = 0; i < array->nr_ranges; i++) {
1152                 unsigned int start = array->ranges[i].start;
1153                 size_t length = array->ranges[i].length;
1154                 unsigned int idx = start + length - 1;
1155
1156                 if (idx >= def->max_entries) {
1157                         pr_debug("ERROR: index %d too large\n", idx);
1158                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG;
1159                 }
1160         }
1161         return 0;
1162 }
1163
1164 static int
1165 bpf__obj_config_map(struct bpf_object *obj,
1166                     struct parse_events_term *term,
1167                     struct perf_evlist *evlist,
1168                     int *key_scan_pos)
1169 {
1170         /* key is "map:<mapname>.<config opt>" */
1171         char *map_name = strdup(term->config + sizeof("map:") - 1);
1172         struct bpf_map *map;
1173         int err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1174         char *map_opt;
1175         size_t i;
1176
1177         if (!map_name)
1178                 return -ENOMEM;
1179
1180         map_opt = strchr(map_name, '.');
1181         if (!map_opt) {
1182                 pr_debug("ERROR: Invalid map config: %s\n", map_name);
1183                 goto out;
1184         }
1185
1186         *map_opt++ = '\0';
1187         if (*map_opt == '\0') {
1188                 pr_debug("ERROR: Invalid map option: %s\n", term->config);
1189                 goto out;
1190         }
1191
1192         map = bpf_object__find_map_by_name(obj, map_name);
1193         if (!map) {
1194                 pr_debug("ERROR: Map %s doesn't exist\n", map_name);
1195                 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST;
1196                 goto out;
1197         }
1198
1199         *key_scan_pos += strlen(map_opt);
1200         err = config_map_indices_range_check(term, map, map_name);
1201         if (err)
1202                 goto out;
1203         *key_scan_pos -= strlen(map_opt);
1204
1205         for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) {
1206                 struct bpf_obj_config__map_func *func =
1207                                 &bpf_obj_config__map_funcs[i];
1208
1209                 if (strcmp(map_opt, func->config_opt) == 0) {
1210                         err = func->config_func(map, term, evlist);
1211                         goto out;
1212                 }
1213         }
1214
1215         pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
1216         err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
1217 out:
1218         free(map_name);
1219         if (!err)
1220                 key_scan_pos += strlen(map_opt);
1221         return err;
1222 }
1223
1224 int bpf__config_obj(struct bpf_object *obj,
1225                     struct parse_events_term *term,
1226                     struct perf_evlist *evlist,
1227                     int *error_pos)
1228 {
1229         int key_scan_pos = 0;
1230         int err;
1231
1232         if (!obj || !term || !term->config)
1233                 return -EINVAL;
1234
1235         if (!prefixcmp(term->config, "map:")) {
1236                 key_scan_pos = sizeof("map:") - 1;
1237                 err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos);
1238                 goto out;
1239         }
1240         err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1241 out:
1242         if (error_pos)
1243                 *error_pos = key_scan_pos;
1244         return err;
1245
1246 }
1247
1248 typedef int (*map_config_func_t)(const char *name, int map_fd,
1249                                  const struct bpf_map_def *pdef,
1250                                  struct bpf_map_op *op,
1251                                  void *pkey, void *arg);
1252
1253 static int
1254 foreach_key_array_all(map_config_func_t func,
1255                       void *arg, const char *name,
1256                       int map_fd, const struct bpf_map_def *pdef,
1257                       struct bpf_map_op *op)
1258 {
1259         unsigned int i;
1260         int err;
1261
1262         for (i = 0; i < pdef->max_entries; i++) {
1263                 err = func(name, map_fd, pdef, op, &i, arg);
1264                 if (err) {
1265                         pr_debug("ERROR: failed to insert value to %s[%u]\n",
1266                                  name, i);
1267                         return err;
1268                 }
1269         }
1270         return 0;
1271 }
1272
1273 static int
1274 foreach_key_array_ranges(map_config_func_t func, void *arg,
1275                          const char *name, int map_fd,
1276                          const struct bpf_map_def *pdef,
1277                          struct bpf_map_op *op)
1278 {
1279         unsigned int i, j;
1280         int err;
1281
1282         for (i = 0; i < op->k.array.nr_ranges; i++) {
1283                 unsigned int start = op->k.array.ranges[i].start;
1284                 size_t length = op->k.array.ranges[i].length;
1285
1286                 for (j = 0; j < length; j++) {
1287                         unsigned int idx = start + j;
1288
1289                         err = func(name, map_fd, pdef, op, &idx, arg);
1290                         if (err) {
1291                                 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1292                                          name, idx);
1293                                 return err;
1294                         }
1295                 }
1296         }
1297         return 0;
1298 }
1299
1300 static int
1301 bpf_map_config_foreach_key(struct bpf_map *map,
1302                            map_config_func_t func,
1303                            void *arg)
1304 {
1305         int err, map_fd;
1306         struct bpf_map_op *op;
1307         const struct bpf_map_def *def;
1308         const char *name = bpf_map__name(map);
1309         struct bpf_map_priv *priv = bpf_map__priv(map);
1310
1311         if (IS_ERR(priv)) {
1312                 pr_debug("ERROR: failed to get private from map %s\n", name);
1313                 return -BPF_LOADER_ERRNO__INTERNAL;
1314         }
1315         if (!priv || list_empty(&priv->ops_list)) {
1316                 pr_debug("INFO: nothing to config for map %s\n", name);
1317                 return 0;
1318         }
1319
1320         def = bpf_map__def(map);
1321         if (IS_ERR(def)) {
1322                 pr_debug("ERROR: failed to get definition from map %s\n", name);
1323                 return -BPF_LOADER_ERRNO__INTERNAL;
1324         }
1325         map_fd = bpf_map__fd(map);
1326         if (map_fd < 0) {
1327                 pr_debug("ERROR: failed to get fd from map %s\n", name);
1328                 return map_fd;
1329         }
1330
1331         list_for_each_entry(op, &priv->ops_list, list) {
1332                 switch (def->type) {
1333                 case BPF_MAP_TYPE_ARRAY:
1334                 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1335                         switch (op->key_type) {
1336                         case BPF_MAP_KEY_ALL:
1337                                 err = foreach_key_array_all(func, arg, name,
1338                                                             map_fd, def, op);
1339                                 break;
1340                         case BPF_MAP_KEY_RANGES:
1341                                 err = foreach_key_array_ranges(func, arg, name,
1342                                                                map_fd, def,
1343                                                                op);
1344                                 break;
1345                         default:
1346                                 pr_debug("ERROR: keytype for map '%s' invalid\n",
1347                                          name);
1348                                 return -BPF_LOADER_ERRNO__INTERNAL;
1349                         }
1350                         if (err)
1351                                 return err;
1352                         break;
1353                 default:
1354                         pr_debug("ERROR: type of '%s' incorrect\n", name);
1355                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1356                 }
1357         }
1358
1359         return 0;
1360 }
1361
1362 static int
1363 apply_config_value_for_key(int map_fd, void *pkey,
1364                            size_t val_size, u64 val)
1365 {
1366         int err = 0;
1367
1368         switch (val_size) {
1369         case 1: {
1370                 u8 _val = (u8)(val);
1371                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1372                 break;
1373         }
1374         case 2: {
1375                 u16 _val = (u16)(val);
1376                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1377                 break;
1378         }
1379         case 4: {
1380                 u32 _val = (u32)(val);
1381                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1382                 break;
1383         }
1384         case 8: {
1385                 err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY);
1386                 break;
1387         }
1388         default:
1389                 pr_debug("ERROR: invalid value size\n");
1390                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1391         }
1392         if (err && errno)
1393                 err = -errno;
1394         return err;
1395 }
1396
1397 static int
1398 apply_config_evsel_for_key(const char *name, int map_fd, void *pkey,
1399                            struct perf_evsel *evsel)
1400 {
1401         struct xyarray *xy = evsel->fd;
1402         struct perf_event_attr *attr;
1403         unsigned int key, events;
1404         bool check_pass = false;
1405         int *evt_fd;
1406         int err;
1407
1408         if (!xy) {
1409                 pr_debug("ERROR: evsel not ready for map %s\n", name);
1410                 return -BPF_LOADER_ERRNO__INTERNAL;
1411         }
1412
1413         if (xy->row_size / xy->entry_size != 1) {
1414                 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1415                          name);
1416                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM;
1417         }
1418
1419         attr = &evsel->attr;
1420         if (attr->inherit) {
1421                 pr_debug("ERROR: Can't put inherit event into map %s\n", name);
1422                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH;
1423         }
1424
1425         if (perf_evsel__is_bpf_output(evsel))
1426                 check_pass = true;
1427         if (attr->type == PERF_TYPE_RAW)
1428                 check_pass = true;
1429         if (attr->type == PERF_TYPE_HARDWARE)
1430                 check_pass = true;
1431         if (!check_pass) {
1432                 pr_debug("ERROR: Event type is wrong for map %s\n", name);
1433                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE;
1434         }
1435
1436         events = xy->entries / (xy->row_size / xy->entry_size);
1437         key = *((unsigned int *)pkey);
1438         if (key >= events) {
1439                 pr_debug("ERROR: there is no event %d for map %s\n",
1440                          key, name);
1441                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE;
1442         }
1443         evt_fd = xyarray__entry(xy, key, 0);
1444         err = bpf_map_update_elem(map_fd, pkey, evt_fd, BPF_ANY);
1445         if (err && errno)
1446                 err = -errno;
1447         return err;
1448 }
1449
1450 static int
1451 apply_obj_config_map_for_key(const char *name, int map_fd,
1452                              const struct bpf_map_def *pdef,
1453                              struct bpf_map_op *op,
1454                              void *pkey, void *arg __maybe_unused)
1455 {
1456         int err;
1457
1458         switch (op->op_type) {
1459         case BPF_MAP_OP_SET_VALUE:
1460                 err = apply_config_value_for_key(map_fd, pkey,
1461                                                  pdef->value_size,
1462                                                  op->v.value);
1463                 break;
1464         case BPF_MAP_OP_SET_EVSEL:
1465                 err = apply_config_evsel_for_key(name, map_fd, pkey,
1466                                                  op->v.evsel);
1467                 break;
1468         default:
1469                 pr_debug("ERROR: unknown value type for '%s'\n", name);
1470                 err = -BPF_LOADER_ERRNO__INTERNAL;
1471         }
1472         return err;
1473 }
1474
1475 static int
1476 apply_obj_config_map(struct bpf_map *map)
1477 {
1478         return bpf_map_config_foreach_key(map,
1479                                           apply_obj_config_map_for_key,
1480                                           NULL);
1481 }
1482
1483 static int
1484 apply_obj_config_object(struct bpf_object *obj)
1485 {
1486         struct bpf_map *map;
1487         int err;
1488
1489         bpf_map__for_each(map, obj) {
1490                 err = apply_obj_config_map(map);
1491                 if (err)
1492                         return err;
1493         }
1494         return 0;
1495 }
1496
1497 int bpf__apply_obj_config(void)
1498 {
1499         struct bpf_object *obj, *tmp;
1500         int err;
1501
1502         bpf_object__for_each_safe(obj, tmp) {
1503                 err = apply_obj_config_object(obj);
1504                 if (err)
1505                         return err;
1506         }
1507
1508         return 0;
1509 }
1510
1511 #define bpf__for_each_map(pos, obj, objtmp)     \
1512         bpf_object__for_each_safe(obj, objtmp)  \
1513                 bpf_map__for_each(pos, obj)
1514
1515 #define bpf__for_each_stdout_map(pos, obj, objtmp)      \
1516         bpf__for_each_map(pos, obj, objtmp)             \
1517                 if (bpf_map__name(pos) &&               \
1518                         (strcmp("__bpf_stdout__",       \
1519                                 bpf_map__name(pos)) == 0))
1520
1521 int bpf__setup_stdout(struct perf_evlist *evlist __maybe_unused)
1522 {
1523         struct bpf_map_priv *tmpl_priv = NULL;
1524         struct bpf_object *obj, *tmp;
1525         struct perf_evsel *evsel = NULL;
1526         struct bpf_map *map;
1527         int err;
1528         bool need_init = false;
1529
1530         bpf__for_each_stdout_map(map, obj, tmp) {
1531                 struct bpf_map_priv *priv = bpf_map__priv(map);
1532
1533                 if (IS_ERR(priv))
1534                         return -BPF_LOADER_ERRNO__INTERNAL;
1535
1536                 /*
1537                  * No need to check map type: type should have been
1538                  * verified by kernel.
1539                  */
1540                 if (!need_init && !priv)
1541                         need_init = !priv;
1542                 if (!tmpl_priv && priv)
1543                         tmpl_priv = priv;
1544         }
1545
1546         if (!need_init)
1547                 return 0;
1548
1549         if (!tmpl_priv) {
1550                 err = parse_events(evlist, "bpf-output/no-inherit=1,name=__bpf_stdout__/",
1551                                    NULL);
1552                 if (err) {
1553                         pr_debug("ERROR: failed to create bpf-output event\n");
1554                         return -err;
1555                 }
1556
1557                 evsel = perf_evlist__last(evlist);
1558         }
1559
1560         bpf__for_each_stdout_map(map, obj, tmp) {
1561                 struct bpf_map_priv *priv = bpf_map__priv(map);
1562
1563                 if (IS_ERR(priv))
1564                         return -BPF_LOADER_ERRNO__INTERNAL;
1565                 if (priv)
1566                         continue;
1567
1568                 if (tmpl_priv) {
1569                         priv = bpf_map_priv__clone(tmpl_priv);
1570                         if (!priv)
1571                                 return -ENOMEM;
1572
1573                         err = bpf_map__set_priv(map, priv, bpf_map_priv__clear);
1574                         if (err) {
1575                                 bpf_map_priv__clear(map, priv);
1576                                 return err;
1577                         }
1578                 } else if (evsel) {
1579                         struct bpf_map_op *op;
1580
1581                         op = bpf_map__add_newop(map, NULL);
1582                         if (IS_ERR(op))
1583                                 return PTR_ERR(op);
1584                         op->op_type = BPF_MAP_OP_SET_EVSEL;
1585                         op->v.evsel = evsel;
1586                 }
1587         }
1588
1589         return 0;
1590 }
1591
1592 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
1593 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1594 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1595
1596 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
1597         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
1598         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
1599         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
1600         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
1601         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
1602         [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string",
1603         [ERRCODE_OFFSET(PROLOGUE)]      = "Failed to generate prologue",
1604         [ERRCODE_OFFSET(PROLOGUE2BIG)]  = "Prologue too big for program",
1605         [ERRCODE_OFFSET(PROLOGUEOOB)]   = "Offset out of bound for prologue",
1606         [ERRCODE_OFFSET(OBJCONF_OPT)]   = "Invalid object config option",
1607         [ERRCODE_OFFSET(OBJCONF_CONF)]  = "Config value not set (missing '=')",
1608         [ERRCODE_OFFSET(OBJCONF_MAP_OPT)]       = "Invalid object map config option",
1609         [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)]  = "Target map doesn't exist",
1610         [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)]     = "Incorrect value type for map",
1611         [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)]      = "Incorrect map type",
1612         [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)]   = "Incorrect map key size",
1613         [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size",
1614         [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT)]     = "Event not found for map setting",
1615         [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE)]   = "Invalid map size for event setting",
1616         [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM)]    = "Event dimension too large",
1617         [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH)]    = "Doesn't support inherit event",
1618         [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE)]   = "Wrong event type for map",
1619         [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG)]   = "Index too large",
1620 };
1621
1622 static int
1623 bpf_loader_strerror(int err, char *buf, size_t size)
1624 {
1625         char sbuf[STRERR_BUFSIZE];
1626         const char *msg;
1627
1628         if (!buf || !size)
1629                 return -1;
1630
1631         err = err > 0 ? err : -err;
1632
1633         if (err >= __LIBBPF_ERRNO__START)
1634                 return libbpf_strerror(err, buf, size);
1635
1636         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
1637                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
1638                 snprintf(buf, size, "%s", msg);
1639                 buf[size - 1] = '\0';
1640                 return 0;
1641         }
1642
1643         if (err >= __BPF_LOADER_ERRNO__END)
1644                 snprintf(buf, size, "Unknown bpf loader error %d", err);
1645         else
1646                 snprintf(buf, size, "%s",
1647                          str_error_r(err, sbuf, sizeof(sbuf)));
1648
1649         buf[size - 1] = '\0';
1650         return -1;
1651 }
1652
1653 #define bpf__strerror_head(err, buf, size) \
1654         char sbuf[STRERR_BUFSIZE], *emsg;\
1655         if (!size)\
1656                 return 0;\
1657         if (err < 0)\
1658                 err = -err;\
1659         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1660         emsg = sbuf;\
1661         switch (err) {\
1662         default:\
1663                 scnprintf(buf, size, "%s", emsg);\
1664                 break;
1665
1666 #define bpf__strerror_entry(val, fmt...)\
1667         case val: {\
1668                 scnprintf(buf, size, fmt);\
1669                 break;\
1670         }
1671
1672 #define bpf__strerror_end(buf, size)\
1673         }\
1674         buf[size - 1] = '\0';
1675
1676 int bpf__strerror_prepare_load(const char *filename, bool source,
1677                                int err, char *buf, size_t size)
1678 {
1679         size_t n;
1680         int ret;
1681
1682         n = snprintf(buf, size, "Failed to load %s%s: ",
1683                          filename, source ? " from source" : "");
1684         if (n >= size) {
1685                 buf[size - 1] = '\0';
1686                 return 0;
1687         }
1688         buf += n;
1689         size -= n;
1690
1691         ret = bpf_loader_strerror(err, buf, size);
1692         buf[size - 1] = '\0';
1693         return ret;
1694 }
1695
1696 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
1697                         int err, char *buf, size_t size)
1698 {
1699         bpf__strerror_head(err, buf, size);
1700         case BPF_LOADER_ERRNO__PROGCONF_TERM: {
1701                 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
1702                 break;
1703         }
1704         bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1705         bpf__strerror_entry(EACCES, "You need to be root");
1706         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1707         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
1708         bpf__strerror_end(buf, size);
1709         return 0;
1710 }
1711
1712 int bpf__strerror_load(struct bpf_object *obj,
1713                        int err, char *buf, size_t size)
1714 {
1715         bpf__strerror_head(err, buf, size);
1716         case LIBBPF_ERRNO__KVER: {
1717                 unsigned int obj_kver = bpf_object__kversion(obj);
1718                 unsigned int real_kver;
1719
1720                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
1721                         scnprintf(buf, size, "Unable to fetch kernel version");
1722                         break;
1723                 }
1724
1725                 if (obj_kver != real_kver) {
1726                         scnprintf(buf, size,
1727                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
1728                                   KVER_PARAM(obj_kver),
1729                                   KVER_PARAM(real_kver));
1730                         break;
1731                 }
1732
1733                 scnprintf(buf, size, "Failed to load program for unknown reason");
1734                 break;
1735         }
1736         bpf__strerror_end(buf, size);
1737         return 0;
1738 }
1739
1740 int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused,
1741                              struct parse_events_term *term __maybe_unused,
1742                              struct perf_evlist *evlist __maybe_unused,
1743                              int *error_pos __maybe_unused, int err,
1744                              char *buf, size_t size)
1745 {
1746         bpf__strerror_head(err, buf, size);
1747         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE,
1748                             "Can't use this config term with this map type");
1749         bpf__strerror_end(buf, size);
1750         return 0;
1751 }
1752
1753 int bpf__strerror_apply_obj_config(int err, char *buf, size_t size)
1754 {
1755         bpf__strerror_head(err, buf, size);
1756         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM,
1757                             "Cannot set event to BPF map in multi-thread tracing");
1758         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH,
1759                             "%s (Hint: use -i to turn off inherit)", emsg);
1760         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE,
1761                             "Can only put raw, hardware and BPF output event into a BPF map");
1762         bpf__strerror_end(buf, size);
1763         return 0;
1764 }
1765
1766 int bpf__strerror_setup_stdout(struct perf_evlist *evlist __maybe_unused,
1767                                int err, char *buf, size_t size)
1768 {
1769         bpf__strerror_head(err, buf, size);
1770         bpf__strerror_end(buf, size);
1771         return 0;
1772 }