kernel/trace/trace_events_hist.c

   1 /*
   2  * trace_events_hist - trace event hist triggers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/mutex.h>
  20 #include <linux/slab.h>
  21 #include <linux/stacktrace.h>
  22
  23 #include "tracing_map.h"
  24 #include "trace.h"
  25
  26 struct hist_field;
  27
  28 typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
  29
  30 struct hist_field {
  31         struct ftrace_event_field       *field;
  32         unsigned long                   flags;
  33         hist_field_fn_t                 fn;
  34         unsigned int                    size;
  35         unsigned int                    offset;
  36 };
  37
  38 static u64 hist_field_none(struct hist_field *field, void *event)
  39 {
  40         return 0;
  41 }
  42
  43 static u64 hist_field_counter(struct hist_field *field, void *event)
  44 {
  45         return 1;
  46 }
  47
  48 static u64 hist_field_string(struct hist_field *hist_field, void *event)
  49 {
  50         char *addr = (char *)(event + hist_field->field->offset);
  51
  52         return (u64)(unsigned long)addr;
  53 }
  54
  55 static u64 hist_field_dynstring(struct hist_field *hist_field, void *event)
  56 {
  57         u32 str_item = *(u32 *)(event + hist_field->field->offset);
  58         int str_loc = str_item & 0xffff;
  59         char *addr = (char *)(event + str_loc);
  60
  61         return (u64)(unsigned long)addr;
  62 }
  63
  64 static u64 hist_field_pstring(struct hist_field *hist_field, void *event)
  65 {
  66         char **addr = (char **)(event + hist_field->field->offset);
  67
  68         return (u64)(unsigned long)*addr;
  69 }
  70
  71 #define DEFINE_HIST_FIELD_FN(type)                                      \
  72 static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
  73 {                                                                       \
  74         type *addr = (type *)(event + hist_field->field->offset);       \
  75                                                                         \
  76         return (u64)(unsigned long)*addr;                               \
  77 }
  78
  79 DEFINE_HIST_FIELD_FN(s64);
  80 DEFINE_HIST_FIELD_FN(u64);
  81 DEFINE_HIST_FIELD_FN(s32);
  82 DEFINE_HIST_FIELD_FN(u32);
  83 DEFINE_HIST_FIELD_FN(s16);
  84 DEFINE_HIST_FIELD_FN(u16);
  85 DEFINE_HIST_FIELD_FN(s8);
  86 DEFINE_HIST_FIELD_FN(u8);
  87
  88 #define for_each_hist_field(i, hist_data)       \
  89         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
  90
  91 #define for_each_hist_val_field(i, hist_data)   \
  92         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
  93
  94 #define for_each_hist_key_field(i, hist_data)   \
  95         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
  96
  97 #define HIST_STACKTRACE_DEPTH   16
  98 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
  99 #define HIST_STACKTRACE_SKIP    5
 100
 101 #define HITCOUNT_IDX            0
 102 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 103
 104 enum hist_field_flags {
 105         HIST_FIELD_FL_HITCOUNT          = 1,
 106         HIST_FIELD_FL_KEY               = 2,
 107         HIST_FIELD_FL_STRING            = 4,
 108         HIST_FIELD_FL_HEX               = 8,
 109         HIST_FIELD_FL_SYM               = 16,
 110         HIST_FIELD_FL_SYM_OFFSET        = 32,
 111         HIST_FIELD_FL_EXECNAME          = 64,
 112         HIST_FIELD_FL_SYSCALL           = 128,
 113         HIST_FIELD_FL_STACKTRACE        = 256,
 114 };
 115
 116 struct hist_trigger_attrs {
 117         char            *keys_str;
 118         char            *vals_str;
 119         char            *sort_key_str;
 120         char            *name;
 121         bool            pause;
 122         bool            cont;
 123         bool            clear;
 124         unsigned int    map_bits;
 125 };
 126
 127 struct hist_trigger_data {
 128         struct hist_field               *fields[TRACING_MAP_FIELDS_MAX];
 129         unsigned int                    n_vals;
 130         unsigned int                    n_keys;
 131         unsigned int                    n_fields;
 132         unsigned int                    key_size;
 133         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 134         unsigned int                    n_sort_keys;
 135         struct trace_event_file         *event_file;
 136         struct hist_trigger_attrs       *attrs;
 137         struct tracing_map              *map;
 138 };
 139
 140 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
 141 {
 142         hist_field_fn_t fn = NULL;
 143
 144         switch (field_size) {
 145         case 8:
 146                 if (field_is_signed)
 147                         fn = hist_field_s64;
 148                 else
 149                         fn = hist_field_u64;
 150                 break;
 151         case 4:
 152                 if (field_is_signed)
 153                         fn = hist_field_s32;
 154                 else
 155                         fn = hist_field_u32;
 156                 break;
 157         case 2:
 158                 if (field_is_signed)
 159                         fn = hist_field_s16;
 160                 else
 161                         fn = hist_field_u16;
 162                 break;
 163         case 1:
 164                 if (field_is_signed)
 165                         fn = hist_field_s8;
 166                 else
 167                         fn = hist_field_u8;
 168                 break;
 169         }
 170
 171         return fn;
 172 }
 173
 174 static int parse_map_size(char *str)
 175 {
 176         unsigned long size, map_bits;
 177         int ret;
 178
 179         strsep(&str, "=");
 180         if (!str) {
 181                 ret = -EINVAL;
 182                 goto out;
 183         }
 184
 185         ret = kstrtoul(str, 0, &size);
 186         if (ret)
 187                 goto out;
 188
 189         map_bits = ilog2(roundup_pow_of_two(size));
 190         if (map_bits < TRACING_MAP_BITS_MIN ||
 191             map_bits > TRACING_MAP_BITS_MAX)
 192                 ret = -EINVAL;
 193         else
 194                 ret = map_bits;
 195  out:
 196         return ret;
 197 }
 198
 199 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
 200 {
 201         if (!attrs)
 202                 return;
 203
 204         kfree(attrs->name);
 205         kfree(attrs->sort_key_str);
 206         kfree(attrs->keys_str);
 207         kfree(attrs->vals_str);
 208         kfree(attrs);
 209 }
 210
 211 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
 212 {
 213         struct hist_trigger_attrs *attrs;
 214         int ret = 0;
 215
 216         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
 217         if (!attrs)
 218                 return ERR_PTR(-ENOMEM);
 219
 220         while (trigger_str) {
 221                 char *str = strsep(&trigger_str, ":");
 222
 223                 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
 224                     (strncmp(str, "keys=", strlen("keys=")) == 0))
 225                         attrs->keys_str = kstrdup(str, GFP_KERNEL);
 226                 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
 227                          (strncmp(str, "vals=", strlen("vals=")) == 0) ||
 228                          (strncmp(str, "values=", strlen("values=")) == 0))
 229                         attrs->vals_str = kstrdup(str, GFP_KERNEL);
 230                 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
 231                         attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
 232                 else if (strncmp(str, "name=", strlen("name=")) == 0)
 233                         attrs->name = kstrdup(str, GFP_KERNEL);
 234                 else if (strcmp(str, "pause") == 0)
 235                         attrs->pause = true;
 236                 else if ((strcmp(str, "cont") == 0) ||
 237                          (strcmp(str, "continue") == 0))
 238                         attrs->cont = true;
 239                 else if (strcmp(str, "clear") == 0)
 240                         attrs->clear = true;
 241                 else if (strncmp(str, "size=", strlen("size=")) == 0) {
 242                         int map_bits = parse_map_size(str);
 243
 244                         if (map_bits < 0) {
 245                                 ret = map_bits;
 246                                 goto free;
 247                         }
 248                         attrs->map_bits = map_bits;
 249                 } else {
 250                         ret = -EINVAL;
 251                         goto free;
 252                 }
 253         }
 254
 255         if (!attrs->keys_str) {
 256                 ret = -EINVAL;
 257                 goto free;
 258         }
 259
 260         return attrs;
 261  free:
 262         destroy_hist_trigger_attrs(attrs);
 263
 264         return ERR_PTR(ret);
 265 }
 266
 267 static inline void save_comm(char *comm, struct task_struct *task)
 268 {
 269         if (!task->pid) {
 270                 strcpy(comm, "<idle>");
 271                 return;
 272         }
 273
 274         if (WARN_ON_ONCE(task->pid < 0)) {
 275                 strcpy(comm, "<XXX>");
 276                 return;
 277         }
 278
 279         memcpy(comm, task->comm, TASK_COMM_LEN);
 280 }
 281
 282 static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
 283 {
 284         kfree((char *)elt->private_data);
 285 }
 286
 287 static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
 288 {
 289         struct hist_trigger_data *hist_data = elt->map->private_data;
 290         struct hist_field *key_field;
 291         unsigned int i;
 292
 293         for_each_hist_key_field(i, hist_data) {
 294                 key_field = hist_data->fields[i];
 295
 296                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
 297                         unsigned int size = TASK_COMM_LEN + 1;
 298
 299                         elt->private_data = kzalloc(size, GFP_KERNEL);
 300                         if (!elt->private_data)
 301                                 return -ENOMEM;
 302                         break;
 303                 }
 304         }
 305
 306         return 0;
 307 }
 308
 309 static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
 310                                        struct tracing_map_elt *from)
 311 {
 312         char *comm_from = from->private_data;
 313         char *comm_to = to->private_data;
 314
 315         if (comm_from)
 316                 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
 317 }
 318
 319 static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
 320 {
 321         char *comm = elt->private_data;
 322
 323         if (comm)
 324                 save_comm(comm, current);
 325 }
 326
 327 static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
 328         .elt_alloc      = hist_trigger_elt_comm_alloc,
 329         .elt_copy       = hist_trigger_elt_comm_copy,
 330         .elt_free       = hist_trigger_elt_comm_free,
 331         .elt_init       = hist_trigger_elt_comm_init,
 332 };
 333
 334 static void destroy_hist_field(struct hist_field *hist_field)
 335 {
 336         kfree(hist_field);
 337 }
 338
 339 static struct hist_field *create_hist_field(struct ftrace_event_field *field,
 340                                             unsigned long flags)
 341 {
 342         struct hist_field *hist_field;
 343
 344         if (field && is_function_field(field))
 345                 return NULL;
 346
 347         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
 348         if (!hist_field)
 349                 return NULL;
 350
 351         if (flags & HIST_FIELD_FL_HITCOUNT) {
 352                 hist_field->fn = hist_field_counter;
 353                 goto out;
 354         }
 355
 356         if (flags & HIST_FIELD_FL_STACKTRACE) {
 357                 hist_field->fn = hist_field_none;
 358                 goto out;
 359         }
 360
 361         if (is_string_field(field)) {
 362                 flags |= HIST_FIELD_FL_STRING;
 363
 364                 if (field->filter_type == FILTER_STATIC_STRING)
 365                         hist_field->fn = hist_field_string;
 366                 else if (field->filter_type == FILTER_DYN_STRING)
 367                         hist_field->fn = hist_field_dynstring;
 368                 else
 369                         hist_field->fn = hist_field_pstring;
 370         } else {
 371                 hist_field->fn = select_value_fn(field->size,
 372                                                  field->is_signed);
 373                 if (!hist_field->fn) {
 374                         destroy_hist_field(hist_field);
 375                         return NULL;
 376                 }
 377         }
 378  out:
 379         hist_field->field = field;
 380         hist_field->flags = flags;
 381
 382         return hist_field;
 383 }
 384
 385 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
 386 {
 387         unsigned int i;
 388
 389         for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
 390                 if (hist_data->fields[i]) {
 391                         destroy_hist_field(hist_data->fields[i]);
 392                         hist_data->fields[i] = NULL;
 393                 }
 394         }
 395 }
 396
 397 static int create_hitcount_val(struct hist_trigger_data *hist_data)
 398 {
 399         hist_data->fields[HITCOUNT_IDX] =
 400                 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
 401         if (!hist_data->fields[HITCOUNT_IDX])
 402                 return -ENOMEM;
 403
 404         hist_data->n_vals++;
 405
 406         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
 407                 return -EINVAL;
 408
 409         return 0;
 410 }
 411
 412 static int create_val_field(struct hist_trigger_data *hist_data,
 413                             unsigned int val_idx,
 414                             struct trace_event_file *file,
 415                             char *field_str)
 416 {
 417         struct ftrace_event_field *field = NULL;
 418         unsigned long flags = 0;
 419         char *field_name;
 420         int ret = 0;
 421
 422         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
 423                 return -EINVAL;
 424
 425         field_name = strsep(&field_str, ".");
 426         if (field_str) {
 427                 if (strcmp(field_str, "hex") == 0)
 428                         flags |= HIST_FIELD_FL_HEX;
 429                 else {
 430                         ret = -EINVAL;
 431                         goto out;
 432                 }
 433         }
 434
 435         field = trace_find_event_field(file->event_call, field_name);
 436         if (!field) {
 437                 ret = -EINVAL;
 438                 goto out;
 439         }
 440
 441         hist_data->fields[val_idx] = create_hist_field(field, flags);
 442         if (!hist_data->fields[val_idx]) {
 443                 ret = -ENOMEM;
 444                 goto out;
 445         }
 446
 447         ++hist_data->n_vals;
 448
 449         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
 450                 ret = -EINVAL;
 451  out:
 452         return ret;
 453 }
 454
 455 static int create_val_fields(struct hist_trigger_data *hist_data,
 456                              struct trace_event_file *file)
 457 {
 458         char *fields_str, *field_str;
 459         unsigned int i, j;
 460         int ret;
 461
 462         ret = create_hitcount_val(hist_data);
 463         if (ret)
 464                 goto out;
 465
 466         fields_str = hist_data->attrs->vals_str;
 467         if (!fields_str)
 468                 goto out;
 469
 470         strsep(&fields_str, "=");
 471         if (!fields_str)
 472                 goto out;
 473
 474         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
 475                      j < TRACING_MAP_VALS_MAX; i++) {
 476                 field_str = strsep(&fields_str, ",");
 477                 if (!field_str)
 478                         break;
 479                 if (strcmp(field_str, "hitcount") == 0)
 480                         continue;
 481                 ret = create_val_field(hist_data, j++, file, field_str);
 482                 if (ret)
 483                         goto out;
 484         }
 485         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
 486                 ret = -EINVAL;
 487  out:
 488         return ret;
 489 }
 490
 491 static int create_key_field(struct hist_trigger_data *hist_data,
 492                             unsigned int key_idx,
 493                             unsigned int key_offset,
 494                             struct trace_event_file *file,
 495                             char *field_str)
 496 {
 497         struct ftrace_event_field *field = NULL;
 498         unsigned long flags = 0;
 499         unsigned int key_size;
 500         int ret = 0;
 501
 502         if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
 503                 return -EINVAL;
 504
 505         flags |= HIST_FIELD_FL_KEY;
 506
 507         if (strcmp(field_str, "stacktrace") == 0) {
 508                 flags |= HIST_FIELD_FL_STACKTRACE;
 509                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
 510         } else {
 511                 char *field_name = strsep(&field_str, ".");
 512
 513                 if (field_str) {
 514                         if (strcmp(field_str, "hex") == 0)
 515                                 flags |= HIST_FIELD_FL_HEX;
 516                         else if (strcmp(field_str, "sym") == 0)
 517                                 flags |= HIST_FIELD_FL_SYM;
 518                         else if (strcmp(field_str, "sym-offset") == 0)
 519                                 flags |= HIST_FIELD_FL_SYM_OFFSET;
 520                         else if ((strcmp(field_str, "execname") == 0) &&
 521                                  (strcmp(field_name, "common_pid") == 0))
 522                                 flags |= HIST_FIELD_FL_EXECNAME;
 523                         else if (strcmp(field_str, "syscall") == 0)
 524                                 flags |= HIST_FIELD_FL_SYSCALL;
 525                         else {
 526                                 ret = -EINVAL;
 527                                 goto out;
 528                         }
 529                 }
 530
 531                 field = trace_find_event_field(file->event_call, field_name);
 532                 if (!field) {
 533                         ret = -EINVAL;
 534                         goto out;
 535                 }
 536
 537                 if (is_string_field(field))
 538                         key_size = MAX_FILTER_STR_VAL;
 539                 else
 540                         key_size = field->size;
 541         }
 542
 543         hist_data->fields[key_idx] = create_hist_field(field, flags);
 544         if (!hist_data->fields[key_idx]) {
 545                 ret = -ENOMEM;
 546                 goto out;
 547         }
 548
 549         key_size = ALIGN(key_size, sizeof(u64));
 550         hist_data->fields[key_idx]->size = key_size;
 551         hist_data->fields[key_idx]->offset = key_offset;
 552         hist_data->key_size += key_size;
 553         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
 554                 ret = -EINVAL;
 555                 goto out;
 556         }
 557
 558         hist_data->n_keys++;
 559
 560         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
 561                 return -EINVAL;
 562
 563         ret = key_size;
 564  out:
 565         return ret;
 566 }
 567
 568 static int create_key_fields(struct hist_trigger_data *hist_data,
 569                              struct trace_event_file *file)
 570 {
 571         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
 572         char *fields_str, *field_str;
 573         int ret = -EINVAL;
 574
 575         fields_str = hist_data->attrs->keys_str;
 576         if (!fields_str)
 577                 goto out;
 578
 579         strsep(&fields_str, "=");
 580         if (!fields_str)
 581                 goto out;
 582
 583         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
 584                 field_str = strsep(&fields_str, ",");
 585                 if (!field_str)
 586                         break;
 587                 ret = create_key_field(hist_data, i, key_offset,
 588                                        file, field_str);
 589                 if (ret < 0)
 590                         goto out;
 591                 key_offset += ret;
 592         }
 593         if (fields_str) {
 594                 ret = -EINVAL;
 595                 goto out;
 596         }
 597         ret = 0;
 598  out:
 599         return ret;
 600 }
 601
 602 static int create_hist_fields(struct hist_trigger_data *hist_data,
 603                               struct trace_event_file *file)
 604 {
 605         int ret;
 606
 607         ret = create_val_fields(hist_data, file);
 608         if (ret)
 609                 goto out;
 610
 611         ret = create_key_fields(hist_data, file);
 612         if (ret)
 613                 goto out;
 614
 615         hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
 616  out:
 617         return ret;
 618 }
 619
 620 static int is_descending(const char *str)
 621 {
 622         if (!str)
 623                 return 0;
 624
 625         if (strcmp(str, "descending") == 0)
 626                 return 1;
 627
 628         if (strcmp(str, "ascending") == 0)
 629                 return 0;
 630
 631         return -EINVAL;
 632 }
 633
 634 static int create_sort_keys(struct hist_trigger_data *hist_data)
 635 {
 636         char *fields_str = hist_data->attrs->sort_key_str;
 637         struct ftrace_event_field *field = NULL;
 638         struct tracing_map_sort_key *sort_key;
 639         int descending, ret = 0;
 640         unsigned int i, j;
 641
 642         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
 643
 644         if (!fields_str)
 645                 goto out;
 646
 647         strsep(&fields_str, "=");
 648         if (!fields_str) {
 649                 ret = -EINVAL;
 650                 goto out;
 651         }
 652
 653         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
 654                 char *field_str, *field_name;
 655
 656                 sort_key = &hist_data->sort_keys[i];
 657
 658                 field_str = strsep(&fields_str, ",");
 659                 if (!field_str) {
 660                         if (i == 0)
 661                                 ret = -EINVAL;
 662                         break;
 663                 }
 664
 665                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
 666                         ret = -EINVAL;
 667                         break;
 668                 }
 669
 670                 field_name = strsep(&field_str, ".");
 671                 if (!field_name) {
 672                         ret = -EINVAL;
 673                         break;
 674                 }
 675
 676                 if (strcmp(field_name, "hitcount") == 0) {
 677                         descending = is_descending(field_str);
 678                         if (descending < 0) {
 679                                 ret = descending;
 680                                 break;
 681                         }
 682                         sort_key->descending = descending;
 683                         continue;
 684                 }
 685
 686                 for (j = 1; j < hist_data->n_fields; j++) {
 687                         field = hist_data->fields[j]->field;
 688                         if (field && (strcmp(field_name, field->name) == 0)) {
 689                                 sort_key->field_idx = j;
 690                                 descending = is_descending(field_str);
 691                                 if (descending < 0) {
 692                                         ret = descending;
 693                                         goto out;
 694                                 }
 695                                 sort_key->descending = descending;
 696                                 break;
 697                         }
 698                 }
 699                 if (j == hist_data->n_fields) {
 700                         ret = -EINVAL;
 701                         break;
 702                 }
 703         }
 704         hist_data->n_sort_keys = i;
 705  out:
 706         return ret;
 707 }
 708
 709 static void destroy_hist_data(struct hist_trigger_data *hist_data)
 710 {
 711         destroy_hist_trigger_attrs(hist_data->attrs);
 712         destroy_hist_fields(hist_data);
 713         tracing_map_destroy(hist_data->map);
 714         kfree(hist_data);
 715 }
 716
 717 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
 718 {
 719         struct tracing_map *map = hist_data->map;
 720         struct ftrace_event_field *field;
 721         struct hist_field *hist_field;
 722         unsigned int i, idx;
 723
 724         for_each_hist_field(i, hist_data) {
 725                 hist_field = hist_data->fields[i];
 726                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
 727                         tracing_map_cmp_fn_t cmp_fn;
 728
 729                         field = hist_field->field;
 730
 731                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
 732                                 cmp_fn = tracing_map_cmp_none;
 733                         else if (is_string_field(field))
 734                                 cmp_fn = tracing_map_cmp_string;
 735                         else
 736                                 cmp_fn = tracing_map_cmp_num(field->size,
 737                                                              field->is_signed);
 738                         idx = tracing_map_add_key_field(map,
 739                                                         hist_field->offset,
 740                                                         cmp_fn);
 741
 742                 } else
 743                         idx = tracing_map_add_sum_field(map);
 744
 745                 if (idx < 0)
 746                         return idx;
 747         }
 748
 749         return 0;
 750 }
 751
 752 static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
 753 {
 754         struct hist_field *key_field;
 755         unsigned int i;
 756
 757         for_each_hist_key_field(i, hist_data) {
 758                 key_field = hist_data->fields[i];
 759
 760                 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
 761                         return true;
 762         }
 763
 764         return false;
 765 }
 766
 767 static struct hist_trigger_data *
 768 create_hist_data(unsigned int map_bits,
 769                  struct hist_trigger_attrs *attrs,
 770                  struct trace_event_file *file)
 771 {
 772         const struct tracing_map_ops *map_ops = NULL;
 773         struct hist_trigger_data *hist_data;
 774         int ret = 0;
 775
 776         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
 777         if (!hist_data)
 778                 return ERR_PTR(-ENOMEM);
 779
 780         hist_data->attrs = attrs;
 781
 782         ret = create_hist_fields(hist_data, file);
 783         if (ret)
 784                 goto free;
 785
 786         ret = create_sort_keys(hist_data);
 787         if (ret)
 788                 goto free;
 789
 790         if (need_tracing_map_ops(hist_data))
 791                 map_ops = &hist_trigger_elt_comm_ops;
 792
 793         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
 794                                             map_ops, hist_data);
 795         if (IS_ERR(hist_data->map)) {
 796                 ret = PTR_ERR(hist_data->map);
 797                 hist_data->map = NULL;
 798                 goto free;
 799         }
 800
 801         ret = create_tracing_map_fields(hist_data);
 802         if (ret)
 803                 goto free;
 804
 805         ret = tracing_map_init(hist_data->map);
 806         if (ret)
 807                 goto free;
 808
 809         hist_data->event_file = file;
 810  out:
 811         return hist_data;
 812  free:
 813         hist_data->attrs = NULL;
 814
 815         destroy_hist_data(hist_data);
 816
 817         hist_data = ERR_PTR(ret);
 818
 819         goto out;
 820 }
 821
 822 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
 823                                     struct tracing_map_elt *elt,
 824                                     void *rec)
 825 {
 826         struct hist_field *hist_field;
 827         unsigned int i;
 828         u64 hist_val;
 829
 830         for_each_hist_val_field(i, hist_data) {
 831                 hist_field = hist_data->fields[i];
 832                 hist_val = hist_field->fn(hist_field, rec);
 833                 tracing_map_update_sum(elt, i, hist_val);
 834         }
 835 }
 836
 837 static inline void add_to_key(char *compound_key, void *key,
 838                               struct hist_field *key_field, void *rec)
 839 {
 840         size_t size = key_field->size;
 841
 842         if (key_field->flags & HIST_FIELD_FL_STRING) {
 843                 struct ftrace_event_field *field;
 844
 845                 field = key_field->field;
 846                 if (field->filter_type == FILTER_DYN_STRING)
 847                         size = *(u32 *)(rec + field->offset) >> 16;
 848                 else if (field->filter_type == FILTER_PTR_STRING)
 849                         size = strlen(key);
 850                 else if (field->filter_type == FILTER_STATIC_STRING)
 851                         size = field->size;
 852
 853                 /* ensure NULL-termination */
 854                 if (size > key_field->size - 1)
 855                         size = key_field->size - 1;
 856         }
 857
 858         memcpy(compound_key + key_field->offset, key, size);
 859 }
 860
 861 static void event_hist_trigger(struct event_trigger_data *data, void *rec)
 862 {
 863         struct hist_trigger_data *hist_data = data->private_data;
 864         bool use_compound_key = (hist_data->n_keys > 1);
 865         unsigned long entries[HIST_STACKTRACE_DEPTH];
 866         char compound_key[HIST_KEY_SIZE_MAX];
 867         struct stack_trace stacktrace;
 868         struct hist_field *key_field;
 869         struct tracing_map_elt *elt;
 870         u64 field_contents;
 871         void *key = NULL;
 872         unsigned int i;
 873
 874         memset(compound_key, 0, hist_data->key_size);
 875
 876         for_each_hist_key_field(i, hist_data) {
 877                 key_field = hist_data->fields[i];
 878
 879                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
 880                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
 881                         stacktrace.entries = entries;
 882                         stacktrace.nr_entries = 0;
 883                         stacktrace.skip = HIST_STACKTRACE_SKIP;
 884
 885                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
 886                         save_stack_trace(&stacktrace);
 887
 888                         key = entries;
 889                 } else {
 890                         field_contents = key_field->fn(key_field, rec);
 891                         if (key_field->flags & HIST_FIELD_FL_STRING) {
 892                                 key = (void *)(unsigned long)field_contents;
 893                                 use_compound_key = true;
 894                         } else
 895                                 key = (void *)&field_contents;
 896                 }
 897
 898                 if (use_compound_key)
 899                         add_to_key(compound_key, key, key_field, rec);
 900         }
 901
 902         if (use_compound_key)
 903                 key = compound_key;
 904
 905         elt = tracing_map_insert(hist_data->map, key);
 906         if (elt)
 907                 hist_trigger_elt_update(hist_data, elt, rec);
 908 }
 909
 910 static void hist_trigger_stacktrace_print(struct seq_file *m,
 911                                           unsigned long *stacktrace_entries,
 912                                           unsigned int max_entries)
 913 {
 914         char str[KSYM_SYMBOL_LEN];
 915         unsigned int spaces = 8;
 916         unsigned int i;
 917
 918         for (i = 0; i < max_entries; i++) {
 919                 if (stacktrace_entries[i] == ULONG_MAX)
 920                         return;
 921
 922                 seq_printf(m, "%*c", 1 + spaces, ' ');
 923                 sprint_symbol(str, stacktrace_entries[i]);
 924                 seq_printf(m, "%s\n", str);
 925         }
 926 }
 927
 928 static void
 929 hist_trigger_entry_print(struct seq_file *m,
 930                          struct hist_trigger_data *hist_data, void *key,
 931                          struct tracing_map_elt *elt)
 932 {
 933         struct hist_field *key_field;
 934         char str[KSYM_SYMBOL_LEN];
 935         bool multiline = false;
 936         unsigned int i;
 937         u64 uval;
 938
 939         seq_puts(m, "{ ");
 940
 941         for_each_hist_key_field(i, hist_data) {
 942                 key_field = hist_data->fields[i];
 943
 944                 if (i > hist_data->n_vals)
 945                         seq_puts(m, ", ");
 946
 947                 if (key_field->flags & HIST_FIELD_FL_HEX) {
 948                         uval = *(u64 *)(key + key_field->offset);
 949                         seq_printf(m, "%s: %llx",
 950                                    key_field->field->name, uval);
 951                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
 952                         uval = *(u64 *)(key + key_field->offset);
 953                         sprint_symbol_no_offset(str, uval);
 954                         seq_printf(m, "%s: [%llx] %-45s",
 955                                    key_field->field->name, uval, str);
 956                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
 957                         uval = *(u64 *)(key + key_field->offset);
 958                         sprint_symbol(str, uval);
 959                         seq_printf(m, "%s: [%llx] %-55s",
 960                                    key_field->field->name, uval, str);
 961                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
 962                         char *comm = elt->private_data;
 963
 964                         uval = *(u64 *)(key + key_field->offset);
 965                         seq_printf(m, "%s: %-16s[%10llu]",
 966                                    key_field->field->name, comm, uval);
 967                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
 968                         const char *syscall_name;
 969
 970                         uval = *(u64 *)(key + key_field->offset);
 971                         syscall_name = get_syscall_name(uval);
 972                         if (!syscall_name)
 973                                 syscall_name = "unknown_syscall";
 974
 975                         seq_printf(m, "%s: %-30s[%3llu]",
 976                                    key_field->field->name, syscall_name, uval);
 977                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
 978                         seq_puts(m, "stacktrace:\n");
 979                         hist_trigger_stacktrace_print(m,
 980                                                       key + key_field->offset,
 981                                                       HIST_STACKTRACE_DEPTH);
 982                         multiline = true;
 983                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
 984                         seq_printf(m, "%s: %-50s", key_field->field->name,
 985                                    (char *)(key + key_field->offset));
 986                 } else {
 987                         uval = *(u64 *)(key + key_field->offset);
 988                         seq_printf(m, "%s: %10llu", key_field->field->name,
 989                                    uval);
 990                 }
 991         }
 992
 993         if (!multiline)
 994                 seq_puts(m, " ");
 995
 996         seq_puts(m, "}");
 997
 998         seq_printf(m, " hitcount: %10llu",
 999                    tracing_map_read_sum(elt, HITCOUNT_IDX));
1000
1001         for (i = 1; i < hist_data->n_vals; i++) {
1002                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
1003                         seq_printf(m, "  %s: %10llx",
1004                                    hist_data->fields[i]->field->name,
1005                                    tracing_map_read_sum(elt, i));
1006                 } else {
1007                         seq_printf(m, "  %s: %10llu",
1008                                    hist_data->fields[i]->field->name,
1009                                    tracing_map_read_sum(elt, i));
1010                 }
1011         }
1012
1013         seq_puts(m, "\n");
1014 }
1015
1016 static int print_entries(struct seq_file *m,
1017                          struct hist_trigger_data *hist_data)
1018 {
1019         struct tracing_map_sort_entry **sort_entries = NULL;
1020         struct tracing_map *map = hist_data->map;
1021         unsigned int i, n_entries;
1022
1023         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1024                                              hist_data->n_sort_keys,
1025                                              &sort_entries);
1026         if (n_entries < 0)
1027                 return n_entries;
1028
1029         for (i = 0; i < n_entries; i++)
1030                 hist_trigger_entry_print(m, hist_data,
1031                                          sort_entries[i]->key,
1032                                          sort_entries[i]->elt);
1033
1034         tracing_map_destroy_sort_entries(sort_entries, n_entries);
1035
1036         return n_entries;
1037 }
1038
1039 static void hist_trigger_show(struct seq_file *m,
1040                               struct event_trigger_data *data, int n)
1041 {
1042         struct hist_trigger_data *hist_data;
1043         int n_entries, ret = 0;
1044
1045         if (n > 0)
1046                 seq_puts(m, "\n\n");
1047
1048         seq_puts(m, "# event histogram\n#\n# trigger info: ");
1049         data->ops->print(m, data->ops, data);
1050         seq_puts(m, "#\n\n");
1051
1052         hist_data = data->private_data;
1053         n_entries = print_entries(m, hist_data);
1054         if (n_entries < 0) {
1055                 ret = n_entries;
1056                 n_entries = 0;
1057         }
1058
1059         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
1060                    (u64)atomic64_read(&hist_data->map->hits),
1061                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
1062 }
1063
1064 static int hist_show(struct seq_file *m, void *v)
1065 {
1066         struct event_trigger_data *data;
1067         struct trace_event_file *event_file;
1068         int n = 0, ret = 0;
1069
1070         mutex_lock(&event_mutex);
1071
1072         event_file = event_file_data(m->private);
1073         if (unlikely(!event_file)) {
1074                 ret = -ENODEV;
1075                 goto out_unlock;
1076         }
1077
1078         list_for_each_entry_rcu(data, &event_file->triggers, list) {
1079                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1080                         hist_trigger_show(m, data, n++);
1081         }
1082
1083  out_unlock:
1084         mutex_unlock(&event_mutex);
1085
1086         return ret;
1087 }
1088
1089 static int event_hist_open(struct inode *inode, struct file *file)
1090 {
1091         return single_open(file, hist_show, file);
1092 }
1093
1094 const struct file_operations event_hist_fops = {
1095         .open = event_hist_open,
1096         .read = seq_read,
1097         .llseek = seq_lseek,
1098         .release = single_release,
1099 };
1100
1101 static const char *get_hist_field_flags(struct hist_field *hist_field)
1102 {
1103         const char *flags_str = NULL;
1104
1105         if (hist_field->flags & HIST_FIELD_FL_HEX)
1106                 flags_str = "hex";
1107         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1108                 flags_str = "sym";
1109         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1110                 flags_str = "sym-offset";
1111         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1112                 flags_str = "execname";
1113         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1114                 flags_str = "syscall";
1115
1116         return flags_str;
1117 }
1118
1119 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1120 {
1121         seq_printf(m, "%s", hist_field->field->name);
1122         if (hist_field->flags) {
1123                 const char *flags_str = get_hist_field_flags(hist_field);
1124
1125                 if (flags_str)
1126                         seq_printf(m, ".%s", flags_str);
1127         }
1128 }
1129
1130 static int event_hist_trigger_print(struct seq_file *m,
1131                                     struct event_trigger_ops *ops,
1132                                     struct event_trigger_data *data)
1133 {
1134         struct hist_trigger_data *hist_data = data->private_data;
1135         struct hist_field *key_field;
1136         unsigned int i;
1137
1138         seq_puts(m, "hist:");
1139
1140         if (data->name)
1141                 seq_printf(m, "%s:", data->name);
1142
1143         seq_puts(m, "keys=");
1144
1145         for_each_hist_key_field(i, hist_data) {
1146                 key_field = hist_data->fields[i];
1147
1148                 if (i > hist_data->n_vals)
1149                         seq_puts(m, ",");
1150
1151                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE)
1152                         seq_puts(m, "stacktrace");
1153                 else
1154                         hist_field_print(m, key_field);
1155         }
1156
1157         seq_puts(m, ":vals=");
1158
1159         for_each_hist_val_field(i, hist_data) {
1160                 if (i == HITCOUNT_IDX)
1161                         seq_puts(m, "hitcount");
1162                 else {
1163                         seq_puts(m, ",");
1164                         hist_field_print(m, hist_data->fields[i]);
1165                 }
1166         }
1167
1168         seq_puts(m, ":sort=");
1169
1170         for (i = 0; i < hist_data->n_sort_keys; i++) {
1171                 struct tracing_map_sort_key *sort_key;
1172
1173                 sort_key = &hist_data->sort_keys[i];
1174
1175                 if (i > 0)
1176                         seq_puts(m, ",");
1177
1178                 if (sort_key->field_idx == HITCOUNT_IDX)
1179                         seq_puts(m, "hitcount");
1180                 else {
1181                         unsigned int idx = sort_key->field_idx;
1182
1183                         if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1184                                 return -EINVAL;
1185
1186                         hist_field_print(m, hist_data->fields[idx]);
1187                 }
1188
1189                 if (sort_key->descending)
1190                         seq_puts(m, ".descending");
1191         }
1192
1193         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1194
1195         if (data->filter_str)
1196                 seq_printf(m, " if %s", data->filter_str);
1197
1198         if (data->paused)
1199                 seq_puts(m, " [paused]");
1200         else
1201                 seq_puts(m, " [active]");
1202
1203         seq_putc(m, '\n');
1204
1205         return 0;
1206 }
1207
1208 static int event_hist_trigger_init(struct event_trigger_ops *ops,
1209                                    struct event_trigger_data *data)
1210 {
1211         struct hist_trigger_data *hist_data = data->private_data;
1212
1213         if (!data->ref && hist_data->attrs->name)
1214                 save_named_trigger(hist_data->attrs->name, data);
1215
1216         data->ref++;
1217
1218         return 0;
1219 }
1220
1221 static void event_hist_trigger_free(struct event_trigger_ops *ops,
1222                                     struct event_trigger_data *data)
1223 {
1224         struct hist_trigger_data *hist_data = data->private_data;
1225
1226         if (WARN_ON_ONCE(data->ref <= 0))
1227                 return;
1228
1229         data->ref--;
1230         if (!data->ref) {
1231                 if (data->name)
1232                         del_named_trigger(data);
1233                 trigger_data_free(data);
1234                 destroy_hist_data(hist_data);
1235         }
1236 }
1237
1238 static struct event_trigger_ops event_hist_trigger_ops = {
1239         .func                   = event_hist_trigger,
1240         .print                  = event_hist_trigger_print,
1241         .init                   = event_hist_trigger_init,
1242         .free                   = event_hist_trigger_free,
1243 };
1244
1245 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
1246                                          struct event_trigger_data *data)
1247 {
1248         data->ref++;
1249
1250         save_named_trigger(data->named_data->name, data);
1251
1252         event_hist_trigger_init(ops, data->named_data);
1253
1254         return 0;
1255 }
1256
1257 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
1258                                           struct event_trigger_data *data)
1259 {
1260         if (WARN_ON_ONCE(data->ref <= 0))
1261                 return;
1262
1263         event_hist_trigger_free(ops, data->named_data);
1264
1265         data->ref--;
1266         if (!data->ref) {
1267                 del_named_trigger(data);
1268                 trigger_data_free(data);
1269         }
1270 }
1271
1272 static struct event_trigger_ops event_hist_trigger_named_ops = {
1273         .func                   = event_hist_trigger,
1274         .print                  = event_hist_trigger_print,
1275         .init                   = event_hist_trigger_named_init,
1276         .free                   = event_hist_trigger_named_free,
1277 };
1278
1279 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1280                                                             char *param)
1281 {
1282         return &event_hist_trigger_ops;
1283 }
1284
1285 static void hist_clear(struct event_trigger_data *data)
1286 {
1287         struct hist_trigger_data *hist_data = data->private_data;
1288
1289         if (data->name)
1290                 pause_named_trigger(data);
1291
1292         synchronize_sched();
1293
1294         tracing_map_clear(hist_data->map);
1295
1296         if (data->name)
1297                 unpause_named_trigger(data);
1298 }
1299
1300 static bool compatible_field(struct ftrace_event_field *field,
1301                              struct ftrace_event_field *test_field)
1302 {
1303         if (field == test_field)
1304                 return true;
1305         if (field == NULL || test_field == NULL)
1306                 return false;
1307         if (strcmp(field->name, test_field->name) != 0)
1308                 return false;
1309         if (strcmp(field->type, test_field->type) != 0)
1310                 return false;
1311         if (field->size != test_field->size)
1312                 return false;
1313         if (field->is_signed != test_field->is_signed)
1314                 return false;
1315
1316         return true;
1317 }
1318
1319 static bool hist_trigger_match(struct event_trigger_data *data,
1320                                struct event_trigger_data *data_test,
1321                                struct event_trigger_data *named_data,
1322                                bool ignore_filter)
1323 {
1324         struct tracing_map_sort_key *sort_key, *sort_key_test;
1325         struct hist_trigger_data *hist_data, *hist_data_test;
1326         struct hist_field *key_field, *key_field_test;
1327         unsigned int i;
1328
1329         if (named_data && (named_data != data_test) &&
1330             (named_data != data_test->named_data))
1331                 return false;
1332
1333         if (!named_data && is_named_trigger(data_test))
1334                 return false;
1335
1336         hist_data = data->private_data;
1337         hist_data_test = data_test->private_data;
1338
1339         if (hist_data->n_vals != hist_data_test->n_vals ||
1340             hist_data->n_fields != hist_data_test->n_fields ||
1341             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
1342                 return false;
1343
1344         if (!ignore_filter) {
1345                 if ((data->filter_str && !data_test->filter_str) ||
1346                    (!data->filter_str && data_test->filter_str))
1347                         return false;
1348         }
1349
1350         for_each_hist_field(i, hist_data) {
1351                 key_field = hist_data->fields[i];
1352                 key_field_test = hist_data_test->fields[i];
1353
1354                 if (key_field->flags != key_field_test->flags)
1355                         return false;
1356                 if (!compatible_field(key_field->field, key_field_test->field))
1357                         return false;
1358                 if (key_field->offset != key_field_test->offset)
1359                         return false;
1360         }
1361
1362         for (i = 0; i < hist_data->n_sort_keys; i++) {
1363                 sort_key = &hist_data->sort_keys[i];
1364                 sort_key_test = &hist_data_test->sort_keys[i];
1365
1366                 if (sort_key->field_idx != sort_key_test->field_idx ||
1367                     sort_key->descending != sort_key_test->descending)
1368                         return false;
1369         }
1370
1371         if (!ignore_filter && data->filter_str &&
1372             (strcmp(data->filter_str, data_test->filter_str) != 0))
1373                 return false;
1374
1375         return true;
1376 }
1377
1378 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1379                                  struct event_trigger_data *data,
1380                                  struct trace_event_file *file)
1381 {
1382         struct hist_trigger_data *hist_data = data->private_data;
1383         struct event_trigger_data *test, *named_data = NULL;
1384         int ret = 0;
1385
1386         if (hist_data->attrs->name) {
1387                 named_data = find_named_trigger(hist_data->attrs->name);
1388                 if (named_data) {
1389                         if (!hist_trigger_match(data, named_data, named_data,
1390                                                 true)) {
1391                                 ret = -EINVAL;
1392                                 goto out;
1393                         }
1394                 }
1395         }
1396
1397         if (hist_data->attrs->name && !named_data)
1398                 goto new;
1399
1400         list_for_each_entry_rcu(test, &file->triggers, list) {
1401                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1402                         if (!hist_trigger_match(data, test, named_data, false))
1403                                 continue;
1404                         if (hist_data->attrs->pause)
1405                                 test->paused = true;
1406                         else if (hist_data->attrs->cont)
1407                                 test->paused = false;
1408                         else if (hist_data->attrs->clear)
1409                                 hist_clear(test);
1410                         else
1411                                 ret = -EEXIST;
1412                         goto out;
1413                 }
1414         }
1415  new:
1416         if (hist_data->attrs->cont || hist_data->attrs->clear) {
1417                 ret = -ENOENT;
1418                 goto out;
1419         }
1420
1421         if (named_data) {
1422                 destroy_hist_data(data->private_data);
1423                 data->private_data = named_data->private_data;
1424                 set_named_trigger_data(data, named_data);
1425                 data->ops = &event_hist_trigger_named_ops;
1426         }
1427
1428         if (hist_data->attrs->pause)
1429                 data->paused = true;
1430
1431         if (data->ops->init) {
1432                 ret = data->ops->init(data->ops, data);
1433                 if (ret < 0)
1434                         goto out;
1435         }
1436
1437         list_add_rcu(&data->list, &file->triggers);
1438         ret++;
1439
1440         update_cond_flag(file);
1441
1442         if (trace_event_trigger_enable_disable(file, 1) < 0) {
1443                 list_del_rcu(&data->list);
1444                 update_cond_flag(file);
1445                 ret--;
1446         }
1447  out:
1448         return ret;
1449 }
1450
1451 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
1452                                     struct event_trigger_data *data,
1453                                     struct trace_event_file *file)
1454 {
1455         struct hist_trigger_data *hist_data = data->private_data;
1456         struct event_trigger_data *test, *named_data = NULL;
1457         bool unregistered = false;
1458
1459         if (hist_data->attrs->name)
1460                 named_data = find_named_trigger(hist_data->attrs->name);
1461
1462         list_for_each_entry_rcu(test, &file->triggers, list) {
1463                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1464                         if (!hist_trigger_match(data, test, named_data, false))
1465                                 continue;
1466                         unregistered = true;
1467                         list_del_rcu(&test->list);
1468                         trace_event_trigger_enable_disable(file, 0);
1469                         update_cond_flag(file);
1470                         break;
1471                 }
1472         }
1473
1474         if (unregistered && test->ops->free)
1475                 test->ops->free(test->ops, test);
1476 }
1477
1478 static void hist_unreg_all(struct trace_event_file *file)
1479 {
1480         struct event_trigger_data *test;
1481
1482         list_for_each_entry_rcu(test, &file->triggers, list) {
1483                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1484                         list_del_rcu(&test->list);
1485                         trace_event_trigger_enable_disable(file, 0);
1486                         update_cond_flag(file);
1487                         if (test->ops->free)
1488                                 test->ops->free(test->ops, test);
1489                 }
1490         }
1491 }
1492
1493 static int event_hist_trigger_func(struct event_command *cmd_ops,
1494                                    struct trace_event_file *file,
1495                                    char *glob, char *cmd, char *param)
1496 {
1497         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1498         struct event_trigger_data *trigger_data;
1499         struct hist_trigger_attrs *attrs;
1500         struct event_trigger_ops *trigger_ops;
1501         struct hist_trigger_data *hist_data;
1502         char *trigger;
1503         int ret = 0;
1504
1505         if (!param)
1506                 return -EINVAL;
1507
1508         /* separate the trigger from the filter (k:v [if filter]) */
1509         trigger = strsep(&param, " \t");
1510         if (!trigger)
1511                 return -EINVAL;
1512
1513         attrs = parse_hist_trigger_attrs(trigger);
1514         if (IS_ERR(attrs))
1515                 return PTR_ERR(attrs);
1516
1517         if (attrs->map_bits)
1518                 hist_trigger_bits = attrs->map_bits;
1519
1520         hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1521         if (IS_ERR(hist_data)) {
1522                 destroy_hist_trigger_attrs(attrs);
1523                 return PTR_ERR(hist_data);
1524         }
1525
1526         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1527
1528         ret = -ENOMEM;
1529         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1530         if (!trigger_data)
1531                 goto out_free;
1532
1533         trigger_data->count = -1;
1534         trigger_data->ops = trigger_ops;
1535         trigger_data->cmd_ops = cmd_ops;
1536
1537         INIT_LIST_HEAD(&trigger_data->list);
1538         RCU_INIT_POINTER(trigger_data->filter, NULL);
1539
1540         trigger_data->private_data = hist_data;
1541
1542         /* if param is non-empty, it's supposed to be a filter */
1543         if (param && cmd_ops->set_filter) {
1544                 ret = cmd_ops->set_filter(param, trigger_data, file);
1545                 if (ret < 0)
1546                         goto out_free;
1547         }
1548
1549         if (glob[0] == '!') {
1550                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1551                 ret = 0;
1552                 goto out_free;
1553         }
1554
1555         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1556         /*
1557          * The above returns on success the # of triggers registered,
1558          * but if it didn't register any it returns zero.  Consider no
1559          * triggers registered a failure too.
1560          */
1561         if (!ret) {
1562                 if (!(attrs->pause || attrs->cont || attrs->clear))
1563                         ret = -ENOENT;
1564                 goto out_free;
1565         } else if (ret < 0)
1566                 goto out_free;
1567         /* Just return zero, not the number of registered triggers */
1568         ret = 0;
1569  out:
1570         return ret;
1571  out_free:
1572         if (cmd_ops->set_filter)
1573                 cmd_ops->set_filter(NULL, trigger_data, NULL);
1574
1575         kfree(trigger_data);
1576
1577         destroy_hist_data(hist_data);
1578         goto out;
1579 }
1580
1581 static struct event_command trigger_hist_cmd = {
1582         .name                   = "hist",
1583         .trigger_type           = ETT_EVENT_HIST,
1584         .flags                  = EVENT_CMD_FL_NEEDS_REC,
1585         .func                   = event_hist_trigger_func,
1586         .reg                    = hist_register_trigger,
1587         .unreg                  = hist_unregister_trigger,
1588         .unreg_all              = hist_unreg_all,
1589         .get_trigger_ops        = event_hist_get_trigger_ops,
1590         .set_filter             = set_trigger_filter,
1591 };
1592
1593 __init int register_trigger_hist_cmd(void)
1594 {
1595         int ret;
1596
1597         ret = register_event_command(&trigger_hist_cmd);
1598         WARN_ON(ret < 0);
1599
1600         return ret;
1601 }
1602
1603 static void
1604 hist_enable_trigger(struct event_trigger_data *data, void *rec)
1605 {
1606         struct enable_trigger_data *enable_data = data->private_data;
1607         struct event_trigger_data *test;
1608
1609         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
1610                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1611                         if (enable_data->enable)
1612                                 test->paused = false;
1613                         else
1614                                 test->paused = true;
1615                 }
1616         }
1617 }
1618
1619 static void
1620 hist_enable_count_trigger(struct event_trigger_data *data, void *rec)
1621 {
1622         if (!data->count)
1623                 return;
1624
1625         if (data->count != -1)
1626                 (data->count)--;
1627
1628         hist_enable_trigger(data, rec);
1629 }
1630
1631 static struct event_trigger_ops hist_enable_trigger_ops = {
1632         .func                   = hist_enable_trigger,
1633         .print                  = event_enable_trigger_print,
1634         .init                   = event_trigger_init,
1635         .free                   = event_enable_trigger_free,
1636 };
1637
1638 static struct event_trigger_ops hist_enable_count_trigger_ops = {
1639         .func                   = hist_enable_count_trigger,
1640         .print                  = event_enable_trigger_print,
1641         .init                   = event_trigger_init,
1642         .free                   = event_enable_trigger_free,
1643 };
1644
1645 static struct event_trigger_ops hist_disable_trigger_ops = {
1646         .func                   = hist_enable_trigger,
1647         .print                  = event_enable_trigger_print,
1648         .init                   = event_trigger_init,
1649         .free                   = event_enable_trigger_free,
1650 };
1651
1652 static struct event_trigger_ops hist_disable_count_trigger_ops = {
1653         .func                   = hist_enable_count_trigger,
1654         .print                  = event_enable_trigger_print,
1655         .init                   = event_trigger_init,
1656         .free                   = event_enable_trigger_free,
1657 };
1658
1659 static struct event_trigger_ops *
1660 hist_enable_get_trigger_ops(char *cmd, char *param)
1661 {
1662         struct event_trigger_ops *ops;
1663         bool enable;
1664
1665         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
1666
1667         if (enable)
1668                 ops = param ? &hist_enable_count_trigger_ops :
1669                         &hist_enable_trigger_ops;
1670         else
1671                 ops = param ? &hist_disable_count_trigger_ops :
1672                         &hist_disable_trigger_ops;
1673
1674         return ops;
1675 }
1676
1677 static void hist_enable_unreg_all(struct trace_event_file *file)
1678 {
1679         struct event_trigger_data *test;
1680
1681         list_for_each_entry_rcu(test, &file->triggers, list) {
1682                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
1683                         list_del_rcu(&test->list);
1684                         update_cond_flag(file);
1685                         trace_event_trigger_enable_disable(file, 0);
1686                         if (test->ops->free)
1687                                 test->ops->free(test->ops, test);
1688                 }
1689         }
1690 }
1691
1692 static struct event_command trigger_hist_enable_cmd = {
1693         .name                   = ENABLE_HIST_STR,
1694         .trigger_type           = ETT_HIST_ENABLE,
1695         .func                   = event_enable_trigger_func,
1696         .reg                    = event_enable_register_trigger,
1697         .unreg                  = event_enable_unregister_trigger,
1698         .unreg_all              = hist_enable_unreg_all,
1699         .get_trigger_ops        = hist_enable_get_trigger_ops,
1700         .set_filter             = set_trigger_filter,
1701 };
1702
1703 static struct event_command trigger_hist_disable_cmd = {
1704         .name                   = DISABLE_HIST_STR,
1705         .trigger_type           = ETT_HIST_ENABLE,
1706         .func                   = event_enable_trigger_func,
1707         .reg                    = event_enable_register_trigger,
1708         .unreg                  = event_enable_unregister_trigger,
1709         .unreg_all              = hist_enable_unreg_all,
1710         .get_trigger_ops        = hist_enable_get_trigger_ops,
1711         .set_filter             = set_trigger_filter,
1712 };
1713
1714 static __init void unregister_trigger_hist_enable_disable_cmds(void)
1715 {
1716         unregister_event_command(&trigger_hist_enable_cmd);
1717         unregister_event_command(&trigger_hist_disable_cmd);
1718 }
1719
1720 __init int register_trigger_hist_enable_disable_cmds(void)
1721 {
1722         int ret;
1723
1724         ret = register_event_command(&trigger_hist_enable_cmd);
1725         if (WARN_ON(ret < 0))
1726                 return ret;
1727         ret = register_event_command(&trigger_hist_disable_cmd);
1728         if (WARN_ON(ret < 0))
1729                 unregister_trigger_hist_enable_disable_cmds();
1730
1731         return ret;
1732 }