2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
78 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled = 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head {
125 unsigned long length;
128 union trace_enum_map_item;
130 struct trace_enum_map_tail {
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item *next;
136 const char *end; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item {
149 struct trace_enum_map map;
150 struct trace_enum_map_head head;
151 struct trace_enum_map_tail tail;
154 static union trace_enum_map_item *trace_enum_maps;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161 static char *default_bootup_tracer;
163 static bool allocate_snapshot;
165 static int __init set_cmdline_ftrace(char *str)
167 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168 default_bootup_tracer = bootup_tracer_buf;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded = true;
173 __setup("ftrace=", set_cmdline_ftrace);
175 static int __init set_ftrace_dump_on_oops(char *str)
177 if (*str++ != '=' || !*str) {
178 ftrace_dump_on_oops = DUMP_ALL;
182 if (!strcmp("orig_cpu", str)) {
183 ftrace_dump_on_oops = DUMP_ORIG;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
191 static int __init stop_trace_on_warning(char *str)
193 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194 __disable_trace_on_warning = 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning);
199 static int __init boot_alloc_snapshot(char *str)
201 allocate_snapshot = true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded = true;
206 __setup("alloc_snapshot", boot_alloc_snapshot);
209 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
211 static int __init set_trace_boot_options(char *str)
213 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
216 __setup("trace_options=", set_trace_boot_options);
218 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219 static char *trace_boot_clock __initdata;
221 static int __init set_trace_boot_clock(char *str)
223 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224 trace_boot_clock = trace_boot_clock_buf;
227 __setup("trace_clock=", set_trace_boot_clock);
229 static int __init set_tracepoint_printk(char *str)
231 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
232 tracepoint_printk = 1;
235 __setup("tp_printk", set_tracepoint_printk);
237 unsigned long long ns2usecs(cycle_t nsec)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace = {
273 .trace_flags = TRACE_DEFAULT_FLAGS,
276 LIST_HEAD(ftrace_trace_arrays);
278 int trace_array_get(struct trace_array *this_tr)
280 struct trace_array *tr;
283 mutex_lock(&trace_types_lock);
284 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
291 mutex_unlock(&trace_types_lock);
296 static void __trace_array_put(struct trace_array *this_tr)
298 WARN_ON(!this_tr->ref);
302 void trace_array_put(struct trace_array *this_tr)
304 mutex_lock(&trace_types_lock);
305 __trace_array_put(this_tr);
306 mutex_unlock(&trace_types_lock);
309 int call_filter_check_discard(struct trace_event_call *call, void *rec,
310 struct ring_buffer *buffer,
311 struct ring_buffer_event *event)
313 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
314 !filter_match_preds(call->filter, rec)) {
315 __trace_event_discard_commit(buffer, event);
322 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
326 /* Early boot up does not have a buffer yet */
328 return trace_clock_local();
330 ts = ring_buffer_time_stamp(buf->buffer, cpu);
331 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
336 cycle_t ftrace_now(int cpu)
338 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
342 * tracing_is_enabled - Show if global_trace has been disabled
344 * Shows if the global trace has been enabled or not. It uses the
345 * mirror flag "buffer_disabled" to be used in fast paths such as for
346 * the irqsoff tracer. But it may be inaccurate due to races. If you
347 * need to know the accurate state, use tracing_is_on() which is a little
348 * slower, but accurate.
350 int tracing_is_enabled(void)
353 * For quick access (irqsoff uses this in fast path), just
354 * return the mirror variable of the state of the ring buffer.
355 * It's a little racy, but we don't really care.
358 return !global_trace.buffer_disabled;
362 * trace_buf_size is the size in bytes that is allocated
363 * for a buffer. Note, the number of bytes is always rounded
366 * This number is purposely set to a low number of 16384.
367 * If the dump on oops happens, it will be much appreciated
368 * to not have to wait for all that output. Anyway this can be
369 * boot time and run time configurable.
371 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
373 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
375 /* trace_types holds a link list of available tracers. */
376 static struct tracer *trace_types __read_mostly;
379 * trace_types_lock is used to protect the trace_types list.
381 DEFINE_MUTEX(trace_types_lock);
384 * serialize the access of the ring buffer
386 * ring buffer serializes readers, but it is low level protection.
387 * The validity of the events (which returns by ring_buffer_peek() ..etc)
388 * are not protected by ring buffer.
390 * The content of events may become garbage if we allow other process consumes
391 * these events concurrently:
392 * A) the page of the consumed events may become a normal page
393 * (not reader page) in ring buffer, and this page will be rewrited
394 * by events producer.
395 * B) The page of the consumed events may become a page for splice_read,
396 * and this page will be returned to system.
398 * These primitives allow multi process access to different cpu ring buffer
401 * These primitives don't distinguish read-only and read-consume access.
402 * Multi read-only access are also serialized.
406 static DECLARE_RWSEM(all_cpu_access_lock);
407 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
409 static inline void trace_access_lock(int cpu)
411 if (cpu == RING_BUFFER_ALL_CPUS) {
412 /* gain it for accessing the whole ring buffer. */
413 down_write(&all_cpu_access_lock);
415 /* gain it for accessing a cpu ring buffer. */
417 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
418 down_read(&all_cpu_access_lock);
420 /* Secondly block other access to this @cpu ring buffer. */
421 mutex_lock(&per_cpu(cpu_access_lock, cpu));
425 static inline void trace_access_unlock(int cpu)
427 if (cpu == RING_BUFFER_ALL_CPUS) {
428 up_write(&all_cpu_access_lock);
430 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
431 up_read(&all_cpu_access_lock);
435 static inline void trace_access_lock_init(void)
439 for_each_possible_cpu(cpu)
440 mutex_init(&per_cpu(cpu_access_lock, cpu));
445 static DEFINE_MUTEX(access_lock);
447 static inline void trace_access_lock(int cpu)
450 mutex_lock(&access_lock);
453 static inline void trace_access_unlock(int cpu)
456 mutex_unlock(&access_lock);
459 static inline void trace_access_lock_init(void)
465 #ifdef CONFIG_STACKTRACE
466 static void __ftrace_trace_stack(struct ring_buffer *buffer,
468 int skip, int pc, struct pt_regs *regs);
469 static inline void ftrace_trace_stack(struct trace_array *tr,
470 struct ring_buffer *buffer,
472 int skip, int pc, struct pt_regs *regs);
475 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
477 int skip, int pc, struct pt_regs *regs)
480 static inline void ftrace_trace_stack(struct trace_array *tr,
481 struct ring_buffer *buffer,
483 int skip, int pc, struct pt_regs *regs)
489 static void tracer_tracing_on(struct trace_array *tr)
491 if (tr->trace_buffer.buffer)
492 ring_buffer_record_on(tr->trace_buffer.buffer);
494 * This flag is looked at when buffers haven't been allocated
495 * yet, or by some tracers (like irqsoff), that just want to
496 * know if the ring buffer has been disabled, but it can handle
497 * races of where it gets disabled but we still do a record.
498 * As the check is in the fast path of the tracers, it is more
499 * important to be fast than accurate.
501 tr->buffer_disabled = 0;
502 /* Make the flag seen by readers */
507 * tracing_on - enable tracing buffers
509 * This function enables tracing buffers that may have been
510 * disabled with tracing_off.
512 void tracing_on(void)
514 tracer_tracing_on(&global_trace);
516 EXPORT_SYMBOL_GPL(tracing_on);
519 * __trace_puts - write a constant string into the trace buffer.
520 * @ip: The address of the caller
521 * @str: The constant string to write
522 * @size: The size of the string.
524 int __trace_puts(unsigned long ip, const char *str, int size)
526 struct ring_buffer_event *event;
527 struct ring_buffer *buffer;
528 struct print_entry *entry;
529 unsigned long irq_flags;
533 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
536 pc = preempt_count();
538 if (unlikely(tracing_selftest_running || tracing_disabled))
541 alloc = sizeof(*entry) + size + 2; /* possible \n added */
543 local_save_flags(irq_flags);
544 buffer = global_trace.trace_buffer.buffer;
545 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
550 entry = ring_buffer_event_data(event);
553 memcpy(&entry->buf, str, size);
555 /* Add a newline if necessary */
556 if (entry->buf[size - 1] != '\n') {
557 entry->buf[size] = '\n';
558 entry->buf[size + 1] = '\0';
560 entry->buf[size] = '\0';
562 __buffer_unlock_commit(buffer, event);
563 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
567 EXPORT_SYMBOL_GPL(__trace_puts);
570 * __trace_bputs - write the pointer to a constant string into trace buffer
571 * @ip: The address of the caller
572 * @str: The constant string to write to the buffer to
574 int __trace_bputs(unsigned long ip, const char *str)
576 struct ring_buffer_event *event;
577 struct ring_buffer *buffer;
578 struct bputs_entry *entry;
579 unsigned long irq_flags;
580 int size = sizeof(struct bputs_entry);
583 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
586 pc = preempt_count();
588 if (unlikely(tracing_selftest_running || tracing_disabled))
591 local_save_flags(irq_flags);
592 buffer = global_trace.trace_buffer.buffer;
593 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
598 entry = ring_buffer_event_data(event);
602 __buffer_unlock_commit(buffer, event);
603 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
607 EXPORT_SYMBOL_GPL(__trace_bputs);
609 #ifdef CONFIG_TRACER_SNAPSHOT
611 * trace_snapshot - take a snapshot of the current buffer.
613 * This causes a swap between the snapshot buffer and the current live
614 * tracing buffer. You can use this to take snapshots of the live
615 * trace when some condition is triggered, but continue to trace.
617 * Note, make sure to allocate the snapshot with either
618 * a tracing_snapshot_alloc(), or by doing it manually
619 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
621 * If the snapshot buffer is not allocated, it will stop tracing.
622 * Basically making a permanent snapshot.
624 void tracing_snapshot(void)
626 struct trace_array *tr = &global_trace;
627 struct tracer *tracer = tr->current_trace;
631 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
632 internal_trace_puts("*** snapshot is being ignored ***\n");
636 if (!tr->allocated_snapshot) {
637 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
638 internal_trace_puts("*** stopping trace here! ***\n");
643 /* Note, snapshot can not be used when the tracer uses it */
644 if (tracer->use_max_tr) {
645 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
646 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
650 local_irq_save(flags);
651 update_max_tr(tr, current, smp_processor_id());
652 local_irq_restore(flags);
654 EXPORT_SYMBOL_GPL(tracing_snapshot);
656 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
657 struct trace_buffer *size_buf, int cpu_id);
658 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
660 static int alloc_snapshot(struct trace_array *tr)
664 if (!tr->allocated_snapshot) {
666 /* allocate spare buffer */
667 ret = resize_buffer_duplicate_size(&tr->max_buffer,
668 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
672 tr->allocated_snapshot = true;
678 static void free_snapshot(struct trace_array *tr)
681 * We don't free the ring buffer. instead, resize it because
682 * The max_tr ring buffer has some state (e.g. ring->clock) and
683 * we want preserve it.
685 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
686 set_buffer_entries(&tr->max_buffer, 1);
687 tracing_reset_online_cpus(&tr->max_buffer);
688 tr->allocated_snapshot = false;
692 * tracing_alloc_snapshot - allocate snapshot buffer.
694 * This only allocates the snapshot buffer if it isn't already
695 * allocated - it doesn't also take a snapshot.
697 * This is meant to be used in cases where the snapshot buffer needs
698 * to be set up for events that can't sleep but need to be able to
699 * trigger a snapshot.
701 int tracing_alloc_snapshot(void)
703 struct trace_array *tr = &global_trace;
706 ret = alloc_snapshot(tr);
711 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
714 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
716 * This is similar to trace_snapshot(), but it will allocate the
717 * snapshot buffer if it isn't already allocated. Use this only
718 * where it is safe to sleep, as the allocation may sleep.
720 * This causes a swap between the snapshot buffer and the current live
721 * tracing buffer. You can use this to take snapshots of the live
722 * trace when some condition is triggered, but continue to trace.
724 void tracing_snapshot_alloc(void)
728 ret = tracing_alloc_snapshot();
734 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
736 void tracing_snapshot(void)
738 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
740 EXPORT_SYMBOL_GPL(tracing_snapshot);
741 int tracing_alloc_snapshot(void)
743 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
746 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
747 void tracing_snapshot_alloc(void)
752 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
753 #endif /* CONFIG_TRACER_SNAPSHOT */
755 static void tracer_tracing_off(struct trace_array *tr)
757 if (tr->trace_buffer.buffer)
758 ring_buffer_record_off(tr->trace_buffer.buffer);
760 * This flag is looked at when buffers haven't been allocated
761 * yet, or by some tracers (like irqsoff), that just want to
762 * know if the ring buffer has been disabled, but it can handle
763 * races of where it gets disabled but we still do a record.
764 * As the check is in the fast path of the tracers, it is more
765 * important to be fast than accurate.
767 tr->buffer_disabled = 1;
768 /* Make the flag seen by readers */
773 * tracing_off - turn off tracing buffers
775 * This function stops the tracing buffers from recording data.
776 * It does not disable any overhead the tracers themselves may
777 * be causing. This function simply causes all recording to
778 * the ring buffers to fail.
780 void tracing_off(void)
782 tracer_tracing_off(&global_trace);
784 EXPORT_SYMBOL_GPL(tracing_off);
786 void disable_trace_on_warning(void)
788 if (__disable_trace_on_warning)
793 * tracer_tracing_is_on - show real state of ring buffer enabled
794 * @tr : the trace array to know if ring buffer is enabled
796 * Shows real state of the ring buffer if it is enabled or not.
798 static int tracer_tracing_is_on(struct trace_array *tr)
800 if (tr->trace_buffer.buffer)
801 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
802 return !tr->buffer_disabled;
806 * tracing_is_on - show state of ring buffers enabled
808 int tracing_is_on(void)
810 return tracer_tracing_is_on(&global_trace);
812 EXPORT_SYMBOL_GPL(tracing_is_on);
814 static int __init set_buf_size(char *str)
816 unsigned long buf_size;
820 buf_size = memparse(str, &str);
821 /* nr_entries can not be zero */
824 trace_buf_size = buf_size;
827 __setup("trace_buf_size=", set_buf_size);
829 static int __init set_tracing_thresh(char *str)
831 unsigned long threshold;
836 ret = kstrtoul(str, 0, &threshold);
839 tracing_thresh = threshold * 1000;
842 __setup("tracing_thresh=", set_tracing_thresh);
844 unsigned long nsecs_to_usecs(unsigned long nsecs)
850 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
851 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
852 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
853 * of strings in the order that the enums were defined.
858 /* These must match the bit postions in trace_iterator_flags */
859 static const char *trace_options[] = {
867 int in_ns; /* is this clock in nanoseconds? */
869 { trace_clock_local, "local", 1 },
870 { trace_clock_global, "global", 1 },
871 { trace_clock_counter, "counter", 0 },
872 { trace_clock_jiffies, "uptime", 0 },
873 { trace_clock, "perf", 1 },
874 { ktime_get_mono_fast_ns, "mono", 1 },
875 { ktime_get_raw_fast_ns, "mono_raw", 1 },
880 * trace_parser_get_init - gets the buffer for trace parser
882 int trace_parser_get_init(struct trace_parser *parser, int size)
884 memset(parser, 0, sizeof(*parser));
886 parser->buffer = kmalloc(size, GFP_KERNEL);
895 * trace_parser_put - frees the buffer for trace parser
897 void trace_parser_put(struct trace_parser *parser)
899 kfree(parser->buffer);
903 * trace_get_user - reads the user input string separated by space
904 * (matched by isspace(ch))
906 * For each string found the 'struct trace_parser' is updated,
907 * and the function returns.
909 * Returns number of bytes read.
911 * See kernel/trace/trace.h for 'struct trace_parser' details.
913 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
914 size_t cnt, loff_t *ppos)
921 trace_parser_clear(parser);
923 ret = get_user(ch, ubuf++);
931 * The parser is not finished with the last write,
932 * continue reading the user input without skipping spaces.
935 /* skip white space */
936 while (cnt && isspace(ch)) {
937 ret = get_user(ch, ubuf++);
944 /* only spaces were written */
954 /* read the non-space input */
955 while (cnt && !isspace(ch)) {
956 if (parser->idx < parser->size - 1)
957 parser->buffer[parser->idx++] = ch;
962 ret = get_user(ch, ubuf++);
969 /* We either got finished input or we have to wait for another call. */
971 parser->buffer[parser->idx] = 0;
972 parser->cont = false;
973 } else if (parser->idx < parser->size - 1) {
975 parser->buffer[parser->idx++] = ch;
988 /* TODO add a seq_buf_to_buffer() */
989 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
993 if (trace_seq_used(s) <= s->seq.readpos)
996 len = trace_seq_used(s) - s->seq.readpos;
999 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1001 s->seq.readpos += cnt;
1005 unsigned long __read_mostly tracing_thresh;
1007 #ifdef CONFIG_TRACER_MAX_TRACE
1009 * Copy the new maximum trace into the separate maximum-trace
1010 * structure. (this way the maximum trace is permanently saved,
1011 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1014 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1016 struct trace_buffer *trace_buf = &tr->trace_buffer;
1017 struct trace_buffer *max_buf = &tr->max_buffer;
1018 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1019 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1022 max_buf->time_start = data->preempt_timestamp;
1024 max_data->saved_latency = tr->max_latency;
1025 max_data->critical_start = data->critical_start;
1026 max_data->critical_end = data->critical_end;
1028 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1029 max_data->pid = tsk->pid;
1031 * If tsk == current, then use current_uid(), as that does not use
1032 * RCU. The irq tracer can be called out of RCU scope.
1035 max_data->uid = current_uid();
1037 max_data->uid = task_uid(tsk);
1039 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1040 max_data->policy = tsk->policy;
1041 max_data->rt_priority = tsk->rt_priority;
1043 /* record this tasks comm */
1044 tracing_record_cmdline(tsk);
1048 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1050 * @tsk: the task with the latency
1051 * @cpu: The cpu that initiated the trace.
1053 * Flip the buffers between the @tr and the max_tr and record information
1054 * about which task was the cause of this latency.
1057 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1059 struct ring_buffer *buf;
1064 WARN_ON_ONCE(!irqs_disabled());
1066 if (!tr->allocated_snapshot) {
1067 /* Only the nop tracer should hit this when disabling */
1068 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1072 arch_spin_lock(&tr->max_lock);
1074 buf = tr->trace_buffer.buffer;
1075 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1076 tr->max_buffer.buffer = buf;
1078 __update_max_tr(tr, tsk, cpu);
1079 arch_spin_unlock(&tr->max_lock);
1083 * update_max_tr_single - only copy one trace over, and reset the rest
1085 * @tsk - task with the latency
1086 * @cpu - the cpu of the buffer to copy.
1088 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1091 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1098 WARN_ON_ONCE(!irqs_disabled());
1099 if (!tr->allocated_snapshot) {
1100 /* Only the nop tracer should hit this when disabling */
1101 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1105 arch_spin_lock(&tr->max_lock);
1107 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1109 if (ret == -EBUSY) {
1111 * We failed to swap the buffer due to a commit taking
1112 * place on this CPU. We fail to record, but we reset
1113 * the max trace buffer (no one writes directly to it)
1114 * and flag that it failed.
1116 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1117 "Failed to swap buffers due to commit in progress\n");
1120 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1122 __update_max_tr(tr, tsk, cpu);
1123 arch_spin_unlock(&tr->max_lock);
1125 #endif /* CONFIG_TRACER_MAX_TRACE */
1127 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1129 /* Iterators are static, they should be filled or empty */
1130 if (trace_buffer_iter(iter, iter->cpu_file))
1133 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1137 #ifdef CONFIG_FTRACE_STARTUP_TEST
1138 static int run_tracer_selftest(struct tracer *type)
1140 struct trace_array *tr = &global_trace;
1141 struct tracer *saved_tracer = tr->current_trace;
1144 if (!type->selftest || tracing_selftest_disabled)
1148 * Run a selftest on this tracer.
1149 * Here we reset the trace buffer, and set the current
1150 * tracer to be this tracer. The tracer can then run some
1151 * internal tracing to verify that everything is in order.
1152 * If we fail, we do not register this tracer.
1154 tracing_reset_online_cpus(&tr->trace_buffer);
1156 tr->current_trace = type;
1158 #ifdef CONFIG_TRACER_MAX_TRACE
1159 if (type->use_max_tr) {
1160 /* If we expanded the buffers, make sure the max is expanded too */
1161 if (ring_buffer_expanded)
1162 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1163 RING_BUFFER_ALL_CPUS);
1164 tr->allocated_snapshot = true;
1168 /* the test is responsible for initializing and enabling */
1169 pr_info("Testing tracer %s: ", type->name);
1170 ret = type->selftest(type, tr);
1171 /* the test is responsible for resetting too */
1172 tr->current_trace = saved_tracer;
1174 printk(KERN_CONT "FAILED!\n");
1175 /* Add the warning after printing 'FAILED' */
1179 /* Only reset on passing, to avoid touching corrupted buffers */
1180 tracing_reset_online_cpus(&tr->trace_buffer);
1182 #ifdef CONFIG_TRACER_MAX_TRACE
1183 if (type->use_max_tr) {
1184 tr->allocated_snapshot = false;
1186 /* Shrink the max buffer again */
1187 if (ring_buffer_expanded)
1188 ring_buffer_resize(tr->max_buffer.buffer, 1,
1189 RING_BUFFER_ALL_CPUS);
1193 printk(KERN_CONT "PASSED\n");
1197 static inline int run_tracer_selftest(struct tracer *type)
1201 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1203 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1205 static void __init apply_trace_boot_options(void);
1208 * register_tracer - register a tracer with the ftrace system.
1209 * @type - the plugin for the tracer
1211 * Register a new plugin tracer.
1213 int __init register_tracer(struct tracer *type)
1219 pr_info("Tracer must have a name\n");
1223 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1224 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1228 mutex_lock(&trace_types_lock);
1230 tracing_selftest_running = true;
1232 for (t = trace_types; t; t = t->next) {
1233 if (strcmp(type->name, t->name) == 0) {
1235 pr_info("Tracer %s already registered\n",
1242 if (!type->set_flag)
1243 type->set_flag = &dummy_set_flag;
1245 /*allocate a dummy tracer_flags*/
1246 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1251 type->flags->val = 0;
1252 type->flags->opts = dummy_tracer_opt;
1254 if (!type->flags->opts)
1255 type->flags->opts = dummy_tracer_opt;
1257 /* store the tracer for __set_tracer_option */
1258 type->flags->trace = type;
1260 ret = run_tracer_selftest(type);
1264 type->next = trace_types;
1266 add_tracer_options(&global_trace, type);
1269 tracing_selftest_running = false;
1270 mutex_unlock(&trace_types_lock);
1272 if (ret || !default_bootup_tracer)
1275 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1278 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1279 /* Do we want this tracer to start on bootup? */
1280 tracing_set_tracer(&global_trace, type->name);
1281 default_bootup_tracer = NULL;
1283 apply_trace_boot_options();
1285 /* disable other selftests, since this will break it. */
1286 tracing_selftest_disabled = true;
1287 #ifdef CONFIG_FTRACE_STARTUP_TEST
1288 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1296 void tracing_reset(struct trace_buffer *buf, int cpu)
1298 struct ring_buffer *buffer = buf->buffer;
1303 ring_buffer_record_disable(buffer);
1305 /* Make sure all commits have finished */
1306 synchronize_sched();
1307 ring_buffer_reset_cpu(buffer, cpu);
1309 ring_buffer_record_enable(buffer);
1312 void tracing_reset_online_cpus(struct trace_buffer *buf)
1314 struct ring_buffer *buffer = buf->buffer;
1320 ring_buffer_record_disable(buffer);
1322 /* Make sure all commits have finished */
1323 synchronize_sched();
1325 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1327 for_each_online_cpu(cpu)
1328 ring_buffer_reset_cpu(buffer, cpu);
1330 ring_buffer_record_enable(buffer);
1333 /* Must have trace_types_lock held */
1334 void tracing_reset_all_online_cpus(void)
1336 struct trace_array *tr;
1338 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1339 tracing_reset_online_cpus(&tr->trace_buffer);
1340 #ifdef CONFIG_TRACER_MAX_TRACE
1341 tracing_reset_online_cpus(&tr->max_buffer);
1346 #define SAVED_CMDLINES_DEFAULT 128
1347 #define NO_CMDLINE_MAP UINT_MAX
1348 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1349 struct saved_cmdlines_buffer {
1350 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1351 unsigned *map_cmdline_to_pid;
1352 unsigned cmdline_num;
1354 char *saved_cmdlines;
1356 static struct saved_cmdlines_buffer *savedcmd;
1358 /* temporary disable recording */
1359 static atomic_t trace_record_cmdline_disabled __read_mostly;
1361 static inline char *get_saved_cmdlines(int idx)
1363 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1366 static inline void set_cmdline(int idx, const char *cmdline)
1368 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1371 static int allocate_cmdlines_buffer(unsigned int val,
1372 struct saved_cmdlines_buffer *s)
1374 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1376 if (!s->map_cmdline_to_pid)
1379 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1380 if (!s->saved_cmdlines) {
1381 kfree(s->map_cmdline_to_pid);
1386 s->cmdline_num = val;
1387 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1388 sizeof(s->map_pid_to_cmdline));
1389 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1390 val * sizeof(*s->map_cmdline_to_pid));
1395 static int trace_create_savedcmd(void)
1399 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1403 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1413 int is_tracing_stopped(void)
1415 return global_trace.stop_count;
1419 * tracing_start - quick start of the tracer
1421 * If tracing is enabled but was stopped by tracing_stop,
1422 * this will start the tracer back up.
1424 void tracing_start(void)
1426 struct ring_buffer *buffer;
1427 unsigned long flags;
1429 if (tracing_disabled)
1432 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1433 if (--global_trace.stop_count) {
1434 if (global_trace.stop_count < 0) {
1435 /* Someone screwed up their debugging */
1437 global_trace.stop_count = 0;
1442 /* Prevent the buffers from switching */
1443 arch_spin_lock(&global_trace.max_lock);
1445 buffer = global_trace.trace_buffer.buffer;
1447 ring_buffer_record_enable(buffer);
1449 #ifdef CONFIG_TRACER_MAX_TRACE
1450 buffer = global_trace.max_buffer.buffer;
1452 ring_buffer_record_enable(buffer);
1455 arch_spin_unlock(&global_trace.max_lock);
1458 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1461 static void tracing_start_tr(struct trace_array *tr)
1463 struct ring_buffer *buffer;
1464 unsigned long flags;
1466 if (tracing_disabled)
1469 /* If global, we need to also start the max tracer */
1470 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1471 return tracing_start();
1473 raw_spin_lock_irqsave(&tr->start_lock, flags);
1475 if (--tr->stop_count) {
1476 if (tr->stop_count < 0) {
1477 /* Someone screwed up their debugging */
1484 buffer = tr->trace_buffer.buffer;
1486 ring_buffer_record_enable(buffer);
1489 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1493 * tracing_stop - quick stop of the tracer
1495 * Light weight way to stop tracing. Use in conjunction with
1498 void tracing_stop(void)
1500 struct ring_buffer *buffer;
1501 unsigned long flags;
1503 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1504 if (global_trace.stop_count++)
1507 /* Prevent the buffers from switching */
1508 arch_spin_lock(&global_trace.max_lock);
1510 buffer = global_trace.trace_buffer.buffer;
1512 ring_buffer_record_disable(buffer);
1514 #ifdef CONFIG_TRACER_MAX_TRACE
1515 buffer = global_trace.max_buffer.buffer;
1517 ring_buffer_record_disable(buffer);
1520 arch_spin_unlock(&global_trace.max_lock);
1523 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1526 static void tracing_stop_tr(struct trace_array *tr)
1528 struct ring_buffer *buffer;
1529 unsigned long flags;
1531 /* If global, we need to also stop the max tracer */
1532 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1533 return tracing_stop();
1535 raw_spin_lock_irqsave(&tr->start_lock, flags);
1536 if (tr->stop_count++)
1539 buffer = tr->trace_buffer.buffer;
1541 ring_buffer_record_disable(buffer);
1544 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1547 void trace_stop_cmdline_recording(void);
1549 static int trace_save_cmdline(struct task_struct *tsk)
1553 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1557 * It's not the end of the world if we don't get
1558 * the lock, but we also don't want to spin
1559 * nor do we want to disable interrupts,
1560 * so if we miss here, then better luck next time.
1562 if (!arch_spin_trylock(&trace_cmdline_lock))
1565 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1566 if (idx == NO_CMDLINE_MAP) {
1567 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1570 * Check whether the cmdline buffer at idx has a pid
1571 * mapped. We are going to overwrite that entry so we
1572 * need to clear the map_pid_to_cmdline. Otherwise we
1573 * would read the new comm for the old pid.
1575 pid = savedcmd->map_cmdline_to_pid[idx];
1576 if (pid != NO_CMDLINE_MAP)
1577 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1579 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1580 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1582 savedcmd->cmdline_idx = idx;
1585 set_cmdline(idx, tsk->comm);
1587 arch_spin_unlock(&trace_cmdline_lock);
1592 static void __trace_find_cmdline(int pid, char comm[])
1597 strcpy(comm, "<idle>");
1601 if (WARN_ON_ONCE(pid < 0)) {
1602 strcpy(comm, "<XXX>");
1606 if (pid > PID_MAX_DEFAULT) {
1607 strcpy(comm, "<...>");
1611 map = savedcmd->map_pid_to_cmdline[pid];
1612 if (map != NO_CMDLINE_MAP)
1613 strcpy(comm, get_saved_cmdlines(map));
1615 strcpy(comm, "<...>");
1618 void trace_find_cmdline(int pid, char comm[])
1621 arch_spin_lock(&trace_cmdline_lock);
1623 __trace_find_cmdline(pid, comm);
1625 arch_spin_unlock(&trace_cmdline_lock);
1629 void tracing_record_cmdline(struct task_struct *tsk)
1631 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1634 if (!__this_cpu_read(trace_cmdline_save))
1637 if (trace_save_cmdline(tsk))
1638 __this_cpu_write(trace_cmdline_save, false);
1642 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1645 struct task_struct *tsk = current;
1647 entry->preempt_count = pc & 0xff;
1648 entry->pid = (tsk) ? tsk->pid : 0;
1650 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1651 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1653 TRACE_FLAG_IRQS_NOSUPPORT |
1655 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
1656 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1657 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1658 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1659 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1661 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1663 static __always_inline void
1664 trace_event_setup(struct ring_buffer_event *event,
1665 int type, unsigned long flags, int pc)
1667 struct trace_entry *ent = ring_buffer_event_data(event);
1669 tracing_generic_entry_update(ent, flags, pc);
1673 struct ring_buffer_event *
1674 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1677 unsigned long flags, int pc)
1679 struct ring_buffer_event *event;
1681 event = ring_buffer_lock_reserve(buffer, len);
1683 trace_event_setup(event, type, flags, pc);
1688 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1689 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1690 static int trace_buffered_event_ref;
1693 * trace_buffered_event_enable - enable buffering events
1695 * When events are being filtered, it is quicker to use a temporary
1696 * buffer to write the event data into if there's a likely chance
1697 * that it will not be committed. The discard of the ring buffer
1698 * is not as fast as committing, and is much slower than copying
1701 * When an event is to be filtered, allocate per cpu buffers to
1702 * write the event data into, and if the event is filtered and discarded
1703 * it is simply dropped, otherwise, the entire data is to be committed
1706 void trace_buffered_event_enable(void)
1708 struct ring_buffer_event *event;
1712 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1714 if (trace_buffered_event_ref++)
1717 for_each_tracing_cpu(cpu) {
1718 page = alloc_pages_node(cpu_to_node(cpu),
1719 GFP_KERNEL | __GFP_NORETRY, 0);
1723 event = page_address(page);
1724 memset(event, 0, sizeof(*event));
1726 per_cpu(trace_buffered_event, cpu) = event;
1729 if (cpu == smp_processor_id() &&
1730 this_cpu_read(trace_buffered_event) !=
1731 per_cpu(trace_buffered_event, cpu))
1738 trace_buffered_event_disable();
1741 static void enable_trace_buffered_event(void *data)
1743 /* Probably not needed, but do it anyway */
1745 this_cpu_dec(trace_buffered_event_cnt);
1748 static void disable_trace_buffered_event(void *data)
1750 this_cpu_inc(trace_buffered_event_cnt);
1754 * trace_buffered_event_disable - disable buffering events
1756 * When a filter is removed, it is faster to not use the buffered
1757 * events, and to commit directly into the ring buffer. Free up
1758 * the temp buffers when there are no more users. This requires
1759 * special synchronization with current events.
1761 void trace_buffered_event_disable(void)
1765 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1767 if (WARN_ON_ONCE(!trace_buffered_event_ref))
1770 if (--trace_buffered_event_ref)
1774 /* For each CPU, set the buffer as used. */
1775 smp_call_function_many(tracing_buffer_mask,
1776 disable_trace_buffered_event, NULL, 1);
1779 /* Wait for all current users to finish */
1780 synchronize_sched();
1782 for_each_tracing_cpu(cpu) {
1783 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
1784 per_cpu(trace_buffered_event, cpu) = NULL;
1787 * Make sure trace_buffered_event is NULL before clearing
1788 * trace_buffered_event_cnt.
1793 /* Do the work on each cpu */
1794 smp_call_function_many(tracing_buffer_mask,
1795 enable_trace_buffered_event, NULL, 1);
1800 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1802 __this_cpu_write(trace_cmdline_save, true);
1804 /* If this is the temp buffer, we need to commit fully */
1805 if (this_cpu_read(trace_buffered_event) == event) {
1806 /* Length is in event->array[0] */
1807 ring_buffer_write(buffer, event->array[0], &event->array[1]);
1808 /* Release the temp buffer */
1809 this_cpu_dec(trace_buffered_event_cnt);
1811 ring_buffer_unlock_commit(buffer, event);
1814 static struct ring_buffer *temp_buffer;
1816 struct ring_buffer_event *
1817 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1818 struct trace_event_file *trace_file,
1819 int type, unsigned long len,
1820 unsigned long flags, int pc)
1822 struct ring_buffer_event *entry;
1825 *current_rb = trace_file->tr->trace_buffer.buffer;
1827 if ((trace_file->flags &
1828 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
1829 (entry = this_cpu_read(trace_buffered_event))) {
1830 /* Try to use the per cpu buffer first */
1831 val = this_cpu_inc_return(trace_buffered_event_cnt);
1833 trace_event_setup(entry, type, flags, pc);
1834 entry->array[0] = len;
1837 this_cpu_dec(trace_buffered_event_cnt);
1840 entry = trace_buffer_lock_reserve(*current_rb,
1841 type, len, flags, pc);
1843 * If tracing is off, but we have triggers enabled
1844 * we still need to look at the event data. Use the temp_buffer
1845 * to store the trace event for the tigger to use. It's recusive
1846 * safe and will not be recorded anywhere.
1848 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1849 *current_rb = temp_buffer;
1850 entry = trace_buffer_lock_reserve(*current_rb,
1851 type, len, flags, pc);
1855 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1857 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1858 struct ring_buffer *buffer,
1859 struct ring_buffer_event *event,
1860 unsigned long flags, int pc,
1861 struct pt_regs *regs)
1863 __buffer_unlock_commit(buffer, event);
1865 ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
1866 ftrace_trace_userstack(buffer, flags, pc);
1870 trace_function(struct trace_array *tr,
1871 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1874 struct trace_event_call *call = &event_function;
1875 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1876 struct ring_buffer_event *event;
1877 struct ftrace_entry *entry;
1879 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1883 entry = ring_buffer_event_data(event);
1885 entry->parent_ip = parent_ip;
1887 if (!call_filter_check_discard(call, entry, buffer, event))
1888 __buffer_unlock_commit(buffer, event);
1891 #ifdef CONFIG_STACKTRACE
1893 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1894 struct ftrace_stack {
1895 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1898 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1899 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1901 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1902 unsigned long flags,
1903 int skip, int pc, struct pt_regs *regs)
1905 struct trace_event_call *call = &event_kernel_stack;
1906 struct ring_buffer_event *event;
1907 struct stack_entry *entry;
1908 struct stack_trace trace;
1910 int size = FTRACE_STACK_ENTRIES;
1912 trace.nr_entries = 0;
1916 * Since events can happen in NMIs there's no safe way to
1917 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1918 * or NMI comes in, it will just have to use the default
1919 * FTRACE_STACK_SIZE.
1921 preempt_disable_notrace();
1923 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1925 * We don't need any atomic variables, just a barrier.
1926 * If an interrupt comes in, we don't care, because it would
1927 * have exited and put the counter back to what we want.
1928 * We just need a barrier to keep gcc from moving things
1932 if (use_stack == 1) {
1933 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1934 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1937 save_stack_trace_regs(regs, &trace);
1939 save_stack_trace(&trace);
1941 if (trace.nr_entries > size)
1942 size = trace.nr_entries;
1944 /* From now on, use_stack is a boolean */
1947 size *= sizeof(unsigned long);
1949 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1950 sizeof(*entry) + size, flags, pc);
1953 entry = ring_buffer_event_data(event);
1955 memset(&entry->caller, 0, size);
1958 memcpy(&entry->caller, trace.entries,
1959 trace.nr_entries * sizeof(unsigned long));
1961 trace.max_entries = FTRACE_STACK_ENTRIES;
1962 trace.entries = entry->caller;
1964 save_stack_trace_regs(regs, &trace);
1966 save_stack_trace(&trace);
1969 entry->size = trace.nr_entries;
1971 if (!call_filter_check_discard(call, entry, buffer, event))
1972 __buffer_unlock_commit(buffer, event);
1975 /* Again, don't let gcc optimize things here */
1977 __this_cpu_dec(ftrace_stack_reserve);
1978 preempt_enable_notrace();
1982 static inline void ftrace_trace_stack(struct trace_array *tr,
1983 struct ring_buffer *buffer,
1984 unsigned long flags,
1985 int skip, int pc, struct pt_regs *regs)
1987 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
1990 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1993 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1996 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
2000 * trace_dump_stack - record a stack back trace in the trace buffer
2001 * @skip: Number of functions to skip (helper handlers)
2003 void trace_dump_stack(int skip)
2005 unsigned long flags;
2007 if (tracing_disabled || tracing_selftest_running)
2010 local_save_flags(flags);
2013 * Skip 3 more, seems to get us at the caller of
2017 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2018 flags, skip, preempt_count(), NULL);
2021 static DEFINE_PER_CPU(int, user_stack_count);
2024 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2026 struct trace_event_call *call = &event_user_stack;
2027 struct ring_buffer_event *event;
2028 struct userstack_entry *entry;
2029 struct stack_trace trace;
2031 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2035 * NMIs can not handle page faults, even with fix ups.
2036 * The save user stack can (and often does) fault.
2038 if (unlikely(in_nmi()))
2042 * prevent recursion, since the user stack tracing may
2043 * trigger other kernel events.
2046 if (__this_cpu_read(user_stack_count))
2049 __this_cpu_inc(user_stack_count);
2051 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2052 sizeof(*entry), flags, pc);
2054 goto out_drop_count;
2055 entry = ring_buffer_event_data(event);
2057 entry->tgid = current->tgid;
2058 memset(&entry->caller, 0, sizeof(entry->caller));
2060 trace.nr_entries = 0;
2061 trace.max_entries = FTRACE_STACK_ENTRIES;
2063 trace.entries = entry->caller;
2065 save_stack_trace_user(&trace);
2066 if (!call_filter_check_discard(call, entry, buffer, event))
2067 __buffer_unlock_commit(buffer, event);
2070 __this_cpu_dec(user_stack_count);
2076 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2078 ftrace_trace_userstack(tr, flags, preempt_count());
2082 #endif /* CONFIG_STACKTRACE */
2084 /* created for use with alloc_percpu */
2085 struct trace_buffer_struct {
2086 char buffer[TRACE_BUF_SIZE];
2089 static struct trace_buffer_struct *trace_percpu_buffer;
2090 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
2091 static struct trace_buffer_struct *trace_percpu_irq_buffer;
2092 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
2095 * The buffer used is dependent on the context. There is a per cpu
2096 * buffer for normal context, softirq contex, hard irq context and
2097 * for NMI context. Thise allows for lockless recording.
2099 * Note, if the buffers failed to be allocated, then this returns NULL
2101 static char *get_trace_buf(void)
2103 struct trace_buffer_struct *percpu_buffer;
2106 * If we have allocated per cpu buffers, then we do not
2107 * need to do any locking.
2110 percpu_buffer = trace_percpu_nmi_buffer;
2112 percpu_buffer = trace_percpu_irq_buffer;
2113 else if (in_softirq())
2114 percpu_buffer = trace_percpu_sirq_buffer;
2116 percpu_buffer = trace_percpu_buffer;
2121 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2124 static int alloc_percpu_trace_buffer(void)
2126 struct trace_buffer_struct *buffers;
2127 struct trace_buffer_struct *sirq_buffers;
2128 struct trace_buffer_struct *irq_buffers;
2129 struct trace_buffer_struct *nmi_buffers;
2131 buffers = alloc_percpu(struct trace_buffer_struct);
2135 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2139 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2143 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2147 trace_percpu_buffer = buffers;
2148 trace_percpu_sirq_buffer = sirq_buffers;
2149 trace_percpu_irq_buffer = irq_buffers;
2150 trace_percpu_nmi_buffer = nmi_buffers;
2155 free_percpu(irq_buffers);
2157 free_percpu(sirq_buffers);
2159 free_percpu(buffers);
2161 WARN(1, "Could not allocate percpu trace_printk buffer");
2165 static int buffers_allocated;
2167 void trace_printk_init_buffers(void)
2169 if (buffers_allocated)
2172 if (alloc_percpu_trace_buffer())
2175 /* trace_printk() is for debug use only. Don't use it in production. */
2178 pr_warn("**********************************************************\n");
2179 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2181 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2183 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2184 pr_warn("** unsafe for production use. **\n");
2186 pr_warn("** If you see this message and you are not debugging **\n");
2187 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2189 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2190 pr_warn("**********************************************************\n");
2192 /* Expand the buffers to set size */
2193 tracing_update_buffers();
2195 buffers_allocated = 1;
2198 * trace_printk_init_buffers() can be called by modules.
2199 * If that happens, then we need to start cmdline recording
2200 * directly here. If the global_trace.buffer is already
2201 * allocated here, then this was called by module code.
2203 if (global_trace.trace_buffer.buffer)
2204 tracing_start_cmdline_record();
2207 void trace_printk_start_comm(void)
2209 /* Start tracing comms if trace printk is set */
2210 if (!buffers_allocated)
2212 tracing_start_cmdline_record();
2215 static void trace_printk_start_stop_comm(int enabled)
2217 if (!buffers_allocated)
2221 tracing_start_cmdline_record();
2223 tracing_stop_cmdline_record();
2227 * trace_vbprintk - write binary msg to tracing buffer
2230 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2232 struct trace_event_call *call = &event_bprint;
2233 struct ring_buffer_event *event;
2234 struct ring_buffer *buffer;
2235 struct trace_array *tr = &global_trace;
2236 struct bprint_entry *entry;
2237 unsigned long flags;
2239 int len = 0, size, pc;
2241 if (unlikely(tracing_selftest_running || tracing_disabled))
2244 /* Don't pollute graph traces with trace_vprintk internals */
2245 pause_graph_tracing();
2247 pc = preempt_count();
2248 preempt_disable_notrace();
2250 tbuffer = get_trace_buf();
2256 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2258 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2261 local_save_flags(flags);
2262 size = sizeof(*entry) + sizeof(u32) * len;
2263 buffer = tr->trace_buffer.buffer;
2264 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2268 entry = ring_buffer_event_data(event);
2272 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2273 if (!call_filter_check_discard(call, entry, buffer, event)) {
2274 __buffer_unlock_commit(buffer, event);
2275 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2279 preempt_enable_notrace();
2280 unpause_graph_tracing();
2284 EXPORT_SYMBOL_GPL(trace_vbprintk);
2287 __trace_array_vprintk(struct ring_buffer *buffer,
2288 unsigned long ip, const char *fmt, va_list args)
2290 struct trace_event_call *call = &event_print;
2291 struct ring_buffer_event *event;
2292 int len = 0, size, pc;
2293 struct print_entry *entry;
2294 unsigned long flags;
2297 if (tracing_disabled || tracing_selftest_running)
2300 /* Don't pollute graph traces with trace_vprintk internals */
2301 pause_graph_tracing();
2303 pc = preempt_count();
2304 preempt_disable_notrace();
2307 tbuffer = get_trace_buf();
2313 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2315 local_save_flags(flags);
2316 size = sizeof(*entry) + len + 1;
2317 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2321 entry = ring_buffer_event_data(event);
2324 memcpy(&entry->buf, tbuffer, len + 1);
2325 if (!call_filter_check_discard(call, entry, buffer, event)) {
2326 __buffer_unlock_commit(buffer, event);
2327 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2330 preempt_enable_notrace();
2331 unpause_graph_tracing();
2336 int trace_array_vprintk(struct trace_array *tr,
2337 unsigned long ip, const char *fmt, va_list args)
2339 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2342 int trace_array_printk(struct trace_array *tr,
2343 unsigned long ip, const char *fmt, ...)
2348 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2352 ret = trace_array_vprintk(tr, ip, fmt, ap);
2357 int trace_array_printk_buf(struct ring_buffer *buffer,
2358 unsigned long ip, const char *fmt, ...)
2363 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2367 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2372 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2374 return trace_array_vprintk(&global_trace, ip, fmt, args);
2376 EXPORT_SYMBOL_GPL(trace_vprintk);
2378 static void trace_iterator_increment(struct trace_iterator *iter)
2380 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2384 ring_buffer_read(buf_iter, NULL);
2387 static struct trace_entry *
2388 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2389 unsigned long *lost_events)
2391 struct ring_buffer_event *event;
2392 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2395 event = ring_buffer_iter_peek(buf_iter, ts);
2397 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2401 iter->ent_size = ring_buffer_event_length(event);
2402 return ring_buffer_event_data(event);
2408 static struct trace_entry *
2409 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2410 unsigned long *missing_events, u64 *ent_ts)
2412 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2413 struct trace_entry *ent, *next = NULL;
2414 unsigned long lost_events = 0, next_lost = 0;
2415 int cpu_file = iter->cpu_file;
2416 u64 next_ts = 0, ts;
2422 * If we are in a per_cpu trace file, don't bother by iterating over
2423 * all cpu and peek directly.
2425 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2426 if (ring_buffer_empty_cpu(buffer, cpu_file))
2428 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2430 *ent_cpu = cpu_file;
2435 for_each_tracing_cpu(cpu) {
2437 if (ring_buffer_empty_cpu(buffer, cpu))
2440 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2443 * Pick the entry with the smallest timestamp:
2445 if (ent && (!next || ts < next_ts)) {
2449 next_lost = lost_events;
2450 next_size = iter->ent_size;
2454 iter->ent_size = next_size;
2457 *ent_cpu = next_cpu;
2463 *missing_events = next_lost;
2468 /* Find the next real entry, without updating the iterator itself */
2469 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2470 int *ent_cpu, u64 *ent_ts)
2472 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2475 /* Find the next real entry, and increment the iterator to the next entry */
2476 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2478 iter->ent = __find_next_entry(iter, &iter->cpu,
2479 &iter->lost_events, &iter->ts);
2482 trace_iterator_increment(iter);
2484 return iter->ent ? iter : NULL;
2487 static void trace_consume(struct trace_iterator *iter)
2489 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2490 &iter->lost_events);
2493 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2495 struct trace_iterator *iter = m->private;
2499 WARN_ON_ONCE(iter->leftover);
2503 /* can't go backwards */
2508 ent = trace_find_next_entry_inc(iter);
2512 while (ent && iter->idx < i)
2513 ent = trace_find_next_entry_inc(iter);
2520 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2522 struct ring_buffer_event *event;
2523 struct ring_buffer_iter *buf_iter;
2524 unsigned long entries = 0;
2527 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2529 buf_iter = trace_buffer_iter(iter, cpu);
2533 ring_buffer_iter_reset(buf_iter);
2536 * We could have the case with the max latency tracers
2537 * that a reset never took place on a cpu. This is evident
2538 * by the timestamp being before the start of the buffer.
2540 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2541 if (ts >= iter->trace_buffer->time_start)
2544 ring_buffer_read(buf_iter, NULL);
2547 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2551 * The current tracer is copied to avoid a global locking
2554 static void *s_start(struct seq_file *m, loff_t *pos)
2556 struct trace_iterator *iter = m->private;
2557 struct trace_array *tr = iter->tr;
2558 int cpu_file = iter->cpu_file;
2564 * copy the tracer to avoid using a global lock all around.
2565 * iter->trace is a copy of current_trace, the pointer to the
2566 * name may be used instead of a strcmp(), as iter->trace->name
2567 * will point to the same string as current_trace->name.
2569 mutex_lock(&trace_types_lock);
2570 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2571 *iter->trace = *tr->current_trace;
2572 mutex_unlock(&trace_types_lock);
2574 #ifdef CONFIG_TRACER_MAX_TRACE
2575 if (iter->snapshot && iter->trace->use_max_tr)
2576 return ERR_PTR(-EBUSY);
2579 if (!iter->snapshot)
2580 atomic_inc(&trace_record_cmdline_disabled);
2582 if (*pos != iter->pos) {
2587 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2588 for_each_tracing_cpu(cpu)
2589 tracing_iter_reset(iter, cpu);
2591 tracing_iter_reset(iter, cpu_file);
2594 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2599 * If we overflowed the seq_file before, then we want
2600 * to just reuse the trace_seq buffer again.
2606 p = s_next(m, p, &l);
2610 trace_event_read_lock();
2611 trace_access_lock(cpu_file);
2615 static void s_stop(struct seq_file *m, void *p)
2617 struct trace_iterator *iter = m->private;
2619 #ifdef CONFIG_TRACER_MAX_TRACE
2620 if (iter->snapshot && iter->trace->use_max_tr)
2624 if (!iter->snapshot)
2625 atomic_dec(&trace_record_cmdline_disabled);
2627 trace_access_unlock(iter->cpu_file);
2628 trace_event_read_unlock();
2632 get_total_entries(struct trace_buffer *buf,
2633 unsigned long *total, unsigned long *entries)
2635 unsigned long count;
2641 for_each_tracing_cpu(cpu) {
2642 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2644 * If this buffer has skipped entries, then we hold all
2645 * entries for the trace and we need to ignore the
2646 * ones before the time stamp.
2648 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2649 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2650 /* total is the same as the entries */
2654 ring_buffer_overrun_cpu(buf->buffer, cpu);
2659 static void print_lat_help_header(struct seq_file *m)
2661 seq_puts(m, "# _------=> CPU# \n"
2662 "# / _-----=> irqs-off \n"
2663 "# | / _----=> need-resched \n"
2664 "# || / _---=> hardirq/softirq \n"
2665 "# ||| / _--=> preempt-depth \n"
2667 "# cmd pid ||||| time | caller \n"
2668 "# \\ / ||||| \\ | / \n");
2671 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2673 unsigned long total;
2674 unsigned long entries;
2676 get_total_entries(buf, &total, &entries);
2677 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2678 entries, total, num_online_cpus());
2682 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2684 print_event_info(buf, m);
2685 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2689 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2691 print_event_info(buf, m);
2692 seq_puts(m, "# _-----=> irqs-off\n"
2693 "# / _----=> need-resched\n"
2694 "# | / _---=> hardirq/softirq\n"
2695 "# || / _--=> preempt-depth\n"
2697 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2698 "# | | | |||| | |\n");
2702 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2704 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2705 struct trace_buffer *buf = iter->trace_buffer;
2706 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2707 struct tracer *type = iter->trace;
2708 unsigned long entries;
2709 unsigned long total;
2710 const char *name = "preemption";
2714 get_total_entries(buf, &total, &entries);
2716 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2718 seq_puts(m, "# -----------------------------------"
2719 "---------------------------------\n");
2720 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2721 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2722 nsecs_to_usecs(data->saved_latency),
2726 #if defined(CONFIG_PREEMPT_NONE)
2728 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2730 #elif defined(CONFIG_PREEMPT)
2735 /* These are reserved for later use */
2738 seq_printf(m, " #P:%d)\n", num_online_cpus());
2742 seq_puts(m, "# -----------------\n");
2743 seq_printf(m, "# | task: %.16s-%d "
2744 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2745 data->comm, data->pid,
2746 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2747 data->policy, data->rt_priority);
2748 seq_puts(m, "# -----------------\n");
2750 if (data->critical_start) {
2751 seq_puts(m, "# => started at: ");
2752 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2753 trace_print_seq(m, &iter->seq);
2754 seq_puts(m, "\n# => ended at: ");
2755 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2756 trace_print_seq(m, &iter->seq);
2757 seq_puts(m, "\n#\n");
2763 static void test_cpu_buff_start(struct trace_iterator *iter)
2765 struct trace_seq *s = &iter->seq;
2766 struct trace_array *tr = iter->tr;
2768 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
2771 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2774 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
2777 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2781 cpumask_set_cpu(iter->cpu, iter->started);
2783 /* Don't print started cpu buffer for the first entry of the trace */
2785 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2789 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2791 struct trace_array *tr = iter->tr;
2792 struct trace_seq *s = &iter->seq;
2793 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
2794 struct trace_entry *entry;
2795 struct trace_event *event;
2799 test_cpu_buff_start(iter);
2801 event = ftrace_find_event(entry->type);
2803 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2804 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2805 trace_print_lat_context(iter);
2807 trace_print_context(iter);
2810 if (trace_seq_has_overflowed(s))
2811 return TRACE_TYPE_PARTIAL_LINE;
2814 return event->funcs->trace(iter, sym_flags, event);
2816 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2818 return trace_handle_return(s);
2821 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2823 struct trace_array *tr = iter->tr;
2824 struct trace_seq *s = &iter->seq;
2825 struct trace_entry *entry;
2826 struct trace_event *event;
2830 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
2831 trace_seq_printf(s, "%d %d %llu ",
2832 entry->pid, iter->cpu, iter->ts);
2834 if (trace_seq_has_overflowed(s))
2835 return TRACE_TYPE_PARTIAL_LINE;
2837 event = ftrace_find_event(entry->type);
2839 return event->funcs->raw(iter, 0, event);
2841 trace_seq_printf(s, "%d ?\n", entry->type);
2843 return trace_handle_return(s);
2846 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2848 struct trace_array *tr = iter->tr;
2849 struct trace_seq *s = &iter->seq;
2850 unsigned char newline = '\n';
2851 struct trace_entry *entry;
2852 struct trace_event *event;
2856 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2857 SEQ_PUT_HEX_FIELD(s, entry->pid);
2858 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2859 SEQ_PUT_HEX_FIELD(s, iter->ts);
2860 if (trace_seq_has_overflowed(s))
2861 return TRACE_TYPE_PARTIAL_LINE;
2864 event = ftrace_find_event(entry->type);
2866 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2867 if (ret != TRACE_TYPE_HANDLED)
2871 SEQ_PUT_FIELD(s, newline);
2873 return trace_handle_return(s);
2876 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2878 struct trace_array *tr = iter->tr;
2879 struct trace_seq *s = &iter->seq;
2880 struct trace_entry *entry;
2881 struct trace_event *event;
2885 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2886 SEQ_PUT_FIELD(s, entry->pid);
2887 SEQ_PUT_FIELD(s, iter->cpu);
2888 SEQ_PUT_FIELD(s, iter->ts);
2889 if (trace_seq_has_overflowed(s))
2890 return TRACE_TYPE_PARTIAL_LINE;
2893 event = ftrace_find_event(entry->type);
2894 return event ? event->funcs->binary(iter, 0, event) :
2898 int trace_empty(struct trace_iterator *iter)
2900 struct ring_buffer_iter *buf_iter;
2903 /* If we are looking at one CPU buffer, only check that one */
2904 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2905 cpu = iter->cpu_file;
2906 buf_iter = trace_buffer_iter(iter, cpu);
2908 if (!ring_buffer_iter_empty(buf_iter))
2911 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2917 for_each_tracing_cpu(cpu) {
2918 buf_iter = trace_buffer_iter(iter, cpu);
2920 if (!ring_buffer_iter_empty(buf_iter))
2923 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2931 /* Called with trace_event_read_lock() held. */
2932 enum print_line_t print_trace_line(struct trace_iterator *iter)
2934 struct trace_array *tr = iter->tr;
2935 unsigned long trace_flags = tr->trace_flags;
2936 enum print_line_t ret;
2938 if (iter->lost_events) {
2939 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2940 iter->cpu, iter->lost_events);
2941 if (trace_seq_has_overflowed(&iter->seq))
2942 return TRACE_TYPE_PARTIAL_LINE;
2945 if (iter->trace && iter->trace->print_line) {
2946 ret = iter->trace->print_line(iter);
2947 if (ret != TRACE_TYPE_UNHANDLED)
2951 if (iter->ent->type == TRACE_BPUTS &&
2952 trace_flags & TRACE_ITER_PRINTK &&
2953 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2954 return trace_print_bputs_msg_only(iter);
2956 if (iter->ent->type == TRACE_BPRINT &&
2957 trace_flags & TRACE_ITER_PRINTK &&
2958 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2959 return trace_print_bprintk_msg_only(iter);
2961 if (iter->ent->type == TRACE_PRINT &&
2962 trace_flags & TRACE_ITER_PRINTK &&
2963 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2964 return trace_print_printk_msg_only(iter);
2966 if (trace_flags & TRACE_ITER_BIN)
2967 return print_bin_fmt(iter);
2969 if (trace_flags & TRACE_ITER_HEX)
2970 return print_hex_fmt(iter);
2972 if (trace_flags & TRACE_ITER_RAW)
2973 return print_raw_fmt(iter);
2975 return print_trace_fmt(iter);
2978 void trace_latency_header(struct seq_file *m)
2980 struct trace_iterator *iter = m->private;
2981 struct trace_array *tr = iter->tr;
2983 /* print nothing if the buffers are empty */
2984 if (trace_empty(iter))
2987 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2988 print_trace_header(m, iter);
2990 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
2991 print_lat_help_header(m);
2994 void trace_default_header(struct seq_file *m)
2996 struct trace_iterator *iter = m->private;
2997 struct trace_array *tr = iter->tr;
2998 unsigned long trace_flags = tr->trace_flags;
3000 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3003 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3004 /* print nothing if the buffers are empty */
3005 if (trace_empty(iter))
3007 print_trace_header(m, iter);
3008 if (!(trace_flags & TRACE_ITER_VERBOSE))
3009 print_lat_help_header(m);
3011 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3012 if (trace_flags & TRACE_ITER_IRQ_INFO)
3013 print_func_help_header_irq(iter->trace_buffer, m);
3015 print_func_help_header(iter->trace_buffer, m);
3020 static void test_ftrace_alive(struct seq_file *m)
3022 if (!ftrace_is_dead())
3024 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3025 "# MAY BE MISSING FUNCTION EVENTS\n");
3028 #ifdef CONFIG_TRACER_MAX_TRACE
3029 static void show_snapshot_main_help(struct seq_file *m)
3031 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3032 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3033 "# Takes a snapshot of the main buffer.\n"
3034 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3035 "# (Doesn't have to be '2' works with any number that\n"
3036 "# is not a '0' or '1')\n");
3039 static void show_snapshot_percpu_help(struct seq_file *m)
3041 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3042 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3043 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3044 "# Takes a snapshot of the main buffer for this cpu.\n");
3046 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3047 "# Must use main snapshot file to allocate.\n");
3049 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3050 "# (Doesn't have to be '2' works with any number that\n"
3051 "# is not a '0' or '1')\n");
3054 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3056 if (iter->tr->allocated_snapshot)
3057 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3059 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3061 seq_puts(m, "# Snapshot commands:\n");
3062 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3063 show_snapshot_main_help(m);
3065 show_snapshot_percpu_help(m);
3068 /* Should never be called */
3069 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3072 static int s_show(struct seq_file *m, void *v)
3074 struct trace_iterator *iter = v;
3077 if (iter->ent == NULL) {
3079 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3081 test_ftrace_alive(m);
3083 if (iter->snapshot && trace_empty(iter))
3084 print_snapshot_help(m, iter);
3085 else if (iter->trace && iter->trace->print_header)
3086 iter->trace->print_header(m);
3088 trace_default_header(m);
3090 } else if (iter->leftover) {
3092 * If we filled the seq_file buffer earlier, we
3093 * want to just show it now.
3095 ret = trace_print_seq(m, &iter->seq);
3097 /* ret should this time be zero, but you never know */
3098 iter->leftover = ret;
3101 print_trace_line(iter);
3102 ret = trace_print_seq(m, &iter->seq);
3104 * If we overflow the seq_file buffer, then it will
3105 * ask us for this data again at start up.
3107 * ret is 0 if seq_file write succeeded.
3110 iter->leftover = ret;
3117 * Should be used after trace_array_get(), trace_types_lock
3118 * ensures that i_cdev was already initialized.
3120 static inline int tracing_get_cpu(struct inode *inode)
3122 if (inode->i_cdev) /* See trace_create_cpu_file() */
3123 return (long)inode->i_cdev - 1;
3124 return RING_BUFFER_ALL_CPUS;
3127 static const struct seq_operations tracer_seq_ops = {
3134 static struct trace_iterator *
3135 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3137 struct trace_array *tr = inode->i_private;
3138 struct trace_iterator *iter;
3141 if (tracing_disabled)
3142 return ERR_PTR(-ENODEV);
3144 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3146 return ERR_PTR(-ENOMEM);
3148 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3150 if (!iter->buffer_iter)
3154 * We make a copy of the current tracer to avoid concurrent
3155 * changes on it while we are reading.
3157 mutex_lock(&trace_types_lock);
3158 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3162 *iter->trace = *tr->current_trace;
3164 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3169 #ifdef CONFIG_TRACER_MAX_TRACE
3170 /* Currently only the top directory has a snapshot */
3171 if (tr->current_trace->print_max || snapshot)
3172 iter->trace_buffer = &tr->max_buffer;
3175 iter->trace_buffer = &tr->trace_buffer;
3176 iter->snapshot = snapshot;
3178 iter->cpu_file = tracing_get_cpu(inode);
3179 mutex_init(&iter->mutex);
3181 /* Notify the tracer early; before we stop tracing. */
3182 if (iter->trace && iter->trace->open)
3183 iter->trace->open(iter);
3185 /* Annotate start of buffers if we had overruns */
3186 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3187 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3189 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3190 if (trace_clocks[tr->clock_id].in_ns)
3191 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3193 /* stop the trace while dumping if we are not opening "snapshot" */
3194 if (!iter->snapshot)
3195 tracing_stop_tr(tr);
3197 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3198 for_each_tracing_cpu(cpu) {
3199 iter->buffer_iter[cpu] =
3200 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3202 ring_buffer_read_prepare_sync();
3203 for_each_tracing_cpu(cpu) {
3204 ring_buffer_read_start(iter->buffer_iter[cpu]);
3205 tracing_iter_reset(iter, cpu);
3208 cpu = iter->cpu_file;
3209 iter->buffer_iter[cpu] =
3210 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3211 ring_buffer_read_prepare_sync();
3212 ring_buffer_read_start(iter->buffer_iter[cpu]);
3213 tracing_iter_reset(iter, cpu);
3216 mutex_unlock(&trace_types_lock);
3221 mutex_unlock(&trace_types_lock);
3223 kfree(iter->buffer_iter);
3225 seq_release_private(inode, file);
3226 return ERR_PTR(-ENOMEM);
3229 int tracing_open_generic(struct inode *inode, struct file *filp)
3231 if (tracing_disabled)
3234 filp->private_data = inode->i_private;
3238 bool tracing_is_disabled(void)
3240 return (tracing_disabled) ? true: false;
3244 * Open and update trace_array ref count.
3245 * Must have the current trace_array passed to it.
3247 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3249 struct trace_array *tr = inode->i_private;
3251 if (tracing_disabled)
3254 if (trace_array_get(tr) < 0)
3257 filp->private_data = inode->i_private;
3262 static int tracing_release(struct inode *inode, struct file *file)
3264 struct trace_array *tr = inode->i_private;
3265 struct seq_file *m = file->private_data;
3266 struct trace_iterator *iter;
3269 if (!(file->f_mode & FMODE_READ)) {
3270 trace_array_put(tr);
3274 /* Writes do not use seq_file */
3276 mutex_lock(&trace_types_lock);
3278 for_each_tracing_cpu(cpu) {
3279 if (iter->buffer_iter[cpu])
3280 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3283 if (iter->trace && iter->trace->close)
3284 iter->trace->close(iter);
3286 if (!iter->snapshot)
3287 /* reenable tracing if it was previously enabled */
3288 tracing_start_tr(tr);
3290 __trace_array_put(tr);
3292 mutex_unlock(&trace_types_lock);
3294 mutex_destroy(&iter->mutex);
3295 free_cpumask_var(iter->started);
3297 kfree(iter->buffer_iter);
3298 seq_release_private(inode, file);
3303 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3305 struct trace_array *tr = inode->i_private;
3307 trace_array_put(tr);
3311 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3313 struct trace_array *tr = inode->i_private;
3315 trace_array_put(tr);
3317 return single_release(inode, file);
3320 static int tracing_open(struct inode *inode, struct file *file)
3322 struct trace_array *tr = inode->i_private;
3323 struct trace_iterator *iter;
3326 if (trace_array_get(tr) < 0)
3329 /* If this file was open for write, then erase contents */
3330 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3331 int cpu = tracing_get_cpu(inode);
3333 if (cpu == RING_BUFFER_ALL_CPUS)
3334 tracing_reset_online_cpus(&tr->trace_buffer);
3336 tracing_reset(&tr->trace_buffer, cpu);
3339 if (file->f_mode & FMODE_READ) {
3340 iter = __tracing_open(inode, file, false);
3342 ret = PTR_ERR(iter);
3343 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3344 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3348 trace_array_put(tr);
3354 * Some tracers are not suitable for instance buffers.
3355 * A tracer is always available for the global array (toplevel)
3356 * or if it explicitly states that it is.
3359 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3361 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3364 /* Find the next tracer that this trace array may use */
3365 static struct tracer *
3366 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3368 while (t && !trace_ok_for_array(t, tr))
3375 t_next(struct seq_file *m, void *v, loff_t *pos)
3377 struct trace_array *tr = m->private;
3378 struct tracer *t = v;
3383 t = get_tracer_for_array(tr, t->next);
3388 static void *t_start(struct seq_file *m, loff_t *pos)
3390 struct trace_array *tr = m->private;
3394 mutex_lock(&trace_types_lock);
3396 t = get_tracer_for_array(tr, trace_types);
3397 for (; t && l < *pos; t = t_next(m, t, &l))
3403 static void t_stop(struct seq_file *m, void *p)
3405 mutex_unlock(&trace_types_lock);
3408 static int t_show(struct seq_file *m, void *v)
3410 struct tracer *t = v;
3415 seq_puts(m, t->name);
3424 static const struct seq_operations show_traces_seq_ops = {
3431 static int show_traces_open(struct inode *inode, struct file *file)
3433 struct trace_array *tr = inode->i_private;
3437 if (tracing_disabled)
3440 ret = seq_open(file, &show_traces_seq_ops);
3444 m = file->private_data;
3451 tracing_write_stub(struct file *filp, const char __user *ubuf,
3452 size_t count, loff_t *ppos)
3457 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3461 if (file->f_mode & FMODE_READ)
3462 ret = seq_lseek(file, offset, whence);
3464 file->f_pos = ret = 0;
3469 static const struct file_operations tracing_fops = {
3470 .open = tracing_open,
3472 .write = tracing_write_stub,
3473 .llseek = tracing_lseek,
3474 .release = tracing_release,
3477 static const struct file_operations show_traces_fops = {
3478 .open = show_traces_open,
3480 .release = seq_release,
3481 .llseek = seq_lseek,
3485 * The tracer itself will not take this lock, but still we want
3486 * to provide a consistent cpumask to user-space:
3488 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3491 * Temporary storage for the character representation of the
3492 * CPU bitmask (and one more byte for the newline):
3494 static char mask_str[NR_CPUS + 1];
3497 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3498 size_t count, loff_t *ppos)
3500 struct trace_array *tr = file_inode(filp)->i_private;
3503 mutex_lock(&tracing_cpumask_update_lock);
3505 len = snprintf(mask_str, count, "%*pb\n",
3506 cpumask_pr_args(tr->tracing_cpumask));
3511 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3514 mutex_unlock(&tracing_cpumask_update_lock);
3520 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3521 size_t count, loff_t *ppos)
3523 struct trace_array *tr = file_inode(filp)->i_private;
3524 cpumask_var_t tracing_cpumask_new;
3527 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3530 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3534 mutex_lock(&tracing_cpumask_update_lock);
3536 local_irq_disable();
3537 arch_spin_lock(&tr->max_lock);
3538 for_each_tracing_cpu(cpu) {
3540 * Increase/decrease the disabled counter if we are
3541 * about to flip a bit in the cpumask:
3543 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3544 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3545 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3546 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3548 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3549 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3550 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3551 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3554 arch_spin_unlock(&tr->max_lock);
3557 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3559 mutex_unlock(&tracing_cpumask_update_lock);
3560 free_cpumask_var(tracing_cpumask_new);
3565 free_cpumask_var(tracing_cpumask_new);
3570 static const struct file_operations tracing_cpumask_fops = {
3571 .open = tracing_open_generic_tr,
3572 .read = tracing_cpumask_read,
3573 .write = tracing_cpumask_write,
3574 .release = tracing_release_generic_tr,
3575 .llseek = generic_file_llseek,
3578 static int tracing_trace_options_show(struct seq_file *m, void *v)
3580 struct tracer_opt *trace_opts;
3581 struct trace_array *tr = m->private;
3585 mutex_lock(&trace_types_lock);
3586 tracer_flags = tr->current_trace->flags->val;
3587 trace_opts = tr->current_trace->flags->opts;
3589 for (i = 0; trace_options[i]; i++) {
3590 if (tr->trace_flags & (1 << i))
3591 seq_printf(m, "%s\n", trace_options[i]);
3593 seq_printf(m, "no%s\n", trace_options[i]);
3596 for (i = 0; trace_opts[i].name; i++) {
3597 if (tracer_flags & trace_opts[i].bit)
3598 seq_printf(m, "%s\n", trace_opts[i].name);
3600 seq_printf(m, "no%s\n", trace_opts[i].name);
3602 mutex_unlock(&trace_types_lock);
3607 static int __set_tracer_option(struct trace_array *tr,
3608 struct tracer_flags *tracer_flags,
3609 struct tracer_opt *opts, int neg)
3611 struct tracer *trace = tracer_flags->trace;
3614 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3619 tracer_flags->val &= ~opts->bit;
3621 tracer_flags->val |= opts->bit;
3625 /* Try to assign a tracer specific option */
3626 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3628 struct tracer *trace = tr->current_trace;
3629 struct tracer_flags *tracer_flags = trace->flags;
3630 struct tracer_opt *opts = NULL;
3633 for (i = 0; tracer_flags->opts[i].name; i++) {
3634 opts = &tracer_flags->opts[i];
3636 if (strcmp(cmp, opts->name) == 0)
3637 return __set_tracer_option(tr, trace->flags, opts, neg);
3643 /* Some tracers require overwrite to stay enabled */
3644 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3646 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3652 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3654 /* do nothing if flag is already set */
3655 if (!!(tr->trace_flags & mask) == !!enabled)
3658 /* Give the tracer a chance to approve the change */
3659 if (tr->current_trace->flag_changed)
3660 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3664 tr->trace_flags |= mask;
3666 tr->trace_flags &= ~mask;
3668 if (mask == TRACE_ITER_RECORD_CMD)
3669 trace_event_enable_cmd_record(enabled);
3671 if (mask == TRACE_ITER_EVENT_FORK)
3672 trace_event_follow_fork(tr, enabled);
3674 if (mask == TRACE_ITER_OVERWRITE) {
3675 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3676 #ifdef CONFIG_TRACER_MAX_TRACE
3677 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3681 if (mask == TRACE_ITER_PRINTK) {
3682 trace_printk_start_stop_comm(enabled);
3683 trace_printk_control(enabled);
3689 static int trace_set_options(struct trace_array *tr, char *option)
3695 size_t orig_len = strlen(option);
3697 cmp = strstrip(option);
3699 if (strncmp(cmp, "no", 2) == 0) {
3704 mutex_lock(&trace_types_lock);
3706 for (i = 0; trace_options[i]; i++) {
3707 if (strcmp(cmp, trace_options[i]) == 0) {
3708 ret = set_tracer_flag(tr, 1 << i, !neg);
3713 /* If no option could be set, test the specific tracer options */
3714 if (!trace_options[i])
3715 ret = set_tracer_option(tr, cmp, neg);
3717 mutex_unlock(&trace_types_lock);
3720 * If the first trailing whitespace is replaced with '\0' by strstrip,
3721 * turn it back into a space.
3723 if (orig_len > strlen(option))
3724 option[strlen(option)] = ' ';
3729 static void __init apply_trace_boot_options(void)
3731 char *buf = trace_boot_options_buf;
3735 option = strsep(&buf, ",");
3741 trace_set_options(&global_trace, option);
3743 /* Put back the comma to allow this to be called again */
3750 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3751 size_t cnt, loff_t *ppos)
3753 struct seq_file *m = filp->private_data;
3754 struct trace_array *tr = m->private;
3758 if (cnt >= sizeof(buf))
3761 if (copy_from_user(buf, ubuf, cnt))
3766 ret = trace_set_options(tr, buf);
3775 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3777 struct trace_array *tr = inode->i_private;
3780 if (tracing_disabled)
3783 if (trace_array_get(tr) < 0)
3786 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3788 trace_array_put(tr);
3793 static const struct file_operations tracing_iter_fops = {
3794 .open = tracing_trace_options_open,
3796 .llseek = seq_lseek,
3797 .release = tracing_single_release_tr,
3798 .write = tracing_trace_options_write,
3801 static const char readme_msg[] =
3802 "tracing mini-HOWTO:\n\n"
3803 "# echo 0 > tracing_on : quick way to disable tracing\n"
3804 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3805 " Important files:\n"
3806 " trace\t\t\t- The static contents of the buffer\n"
3807 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3808 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3809 " current_tracer\t- function and latency tracers\n"
3810 " available_tracers\t- list of configured tracers for current_tracer\n"
3811 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3812 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3813 " trace_clock\t\t-change the clock used to order events\n"
3814 " local: Per cpu clock but may not be synced across CPUs\n"
3815 " global: Synced across CPUs but slows tracing down.\n"
3816 " counter: Not a clock, but just an increment\n"
3817 " uptime: Jiffy counter from time of boot\n"
3818 " perf: Same clock that perf events use\n"
3819 #ifdef CONFIG_X86_64
3820 " x86-tsc: TSC cycle counter\n"
3822 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3823 " tracing_cpumask\t- Limit which CPUs to trace\n"
3824 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3825 "\t\t\t Remove sub-buffer with rmdir\n"
3826 " trace_options\t\t- Set format or modify how tracing happens\n"
3827 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3828 "\t\t\t option name\n"
3829 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3830 #ifdef CONFIG_DYNAMIC_FTRACE
3831 "\n available_filter_functions - list of functions that can be filtered on\n"
3832 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3833 "\t\t\t functions\n"
3834 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3835 "\t modules: Can select a group via module\n"
3836 "\t Format: :mod:<module-name>\n"
3837 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3838 "\t triggers: a command to perform when function is hit\n"
3839 "\t Format: <function>:<trigger>[:count]\n"
3840 "\t trigger: traceon, traceoff\n"
3841 "\t\t enable_event:<system>:<event>\n"
3842 "\t\t disable_event:<system>:<event>\n"
3843 #ifdef CONFIG_STACKTRACE
3846 #ifdef CONFIG_TRACER_SNAPSHOT
3851 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3852 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3853 "\t The first one will disable tracing every time do_fault is hit\n"
3854 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3855 "\t The first time do trap is hit and it disables tracing, the\n"
3856 "\t counter will decrement to 2. If tracing is already disabled,\n"
3857 "\t the counter will not decrement. It only decrements when the\n"
3858 "\t trigger did work\n"
3859 "\t To remove trigger without count:\n"
3860 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3861 "\t To remove trigger with a count:\n"
3862 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3863 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3864 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3865 "\t modules: Can select a group via module command :mod:\n"
3866 "\t Does not accept triggers\n"
3867 #endif /* CONFIG_DYNAMIC_FTRACE */
3868 #ifdef CONFIG_FUNCTION_TRACER
3869 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3872 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3873 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3874 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3875 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3877 #ifdef CONFIG_TRACER_SNAPSHOT
3878 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3879 "\t\t\t snapshot buffer. Read the contents for more\n"
3880 "\t\t\t information\n"
3882 #ifdef CONFIG_STACK_TRACER
3883 " stack_trace\t\t- Shows the max stack trace when active\n"
3884 " stack_max_size\t- Shows current max stack size that was traced\n"
3885 "\t\t\t Write into this file to reset the max size (trigger a\n"
3886 "\t\t\t new trace)\n"
3887 #ifdef CONFIG_DYNAMIC_FTRACE
3888 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3891 #endif /* CONFIG_STACK_TRACER */
3892 " events/\t\t- Directory containing all trace event subsystems:\n"
3893 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3894 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3895 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3897 " filter\t\t- If set, only events passing filter are traced\n"
3898 " events/<system>/<event>/\t- Directory containing control files for\n"
3900 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3901 " filter\t\t- If set, only events passing filter are traced\n"
3902 " trigger\t\t- If set, a command to perform when event is hit\n"
3903 "\t Format: <trigger>[:count][if <filter>]\n"
3904 "\t trigger: traceon, traceoff\n"
3905 "\t enable_event:<system>:<event>\n"
3906 "\t disable_event:<system>:<event>\n"
3907 #ifdef CONFIG_HIST_TRIGGERS
3908 "\t enable_hist:<system>:<event>\n"
3909 "\t disable_hist:<system>:<event>\n"
3911 #ifdef CONFIG_STACKTRACE
3914 #ifdef CONFIG_TRACER_SNAPSHOT
3917 #ifdef CONFIG_HIST_TRIGGERS
3918 "\t\t hist (see below)\n"
3920 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3921 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3922 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3923 "\t events/block/block_unplug/trigger\n"
3924 "\t The first disables tracing every time block_unplug is hit.\n"
3925 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3926 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3927 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3928 "\t Like function triggers, the counter is only decremented if it\n"
3929 "\t enabled or disabled tracing.\n"
3930 "\t To remove a trigger without a count:\n"
3931 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3932 "\t To remove a trigger with a count:\n"
3933 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3934 "\t Filters can be ignored when removing a trigger.\n"
3935 #ifdef CONFIG_HIST_TRIGGERS
3936 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
3937 "\t Format: hist:keys=<field1[,field2,...]>\n"
3938 "\t [:values=<field1[,field2,...]>]\n"
3939 "\t [:sort=<field1[,field2,...]>]\n"
3940 "\t [:size=#entries]\n"
3941 "\t [:pause][:continue][:clear]\n"
3942 "\t [:name=histname1]\n"
3943 "\t [if <filter>]\n\n"
3944 "\t When a matching event is hit, an entry is added to a hash\n"
3945 "\t table using the key(s) and value(s) named, and the value of a\n"
3946 "\t sum called 'hitcount' is incremented. Keys and values\n"
3947 "\t correspond to fields in the event's format description. Keys\n"
3948 "\t can be any field, or the special string 'stacktrace'.\n"
3949 "\t Compound keys consisting of up to two fields can be specified\n"
3950 "\t by the 'keys' keyword. Values must correspond to numeric\n"
3951 "\t fields. Sort keys consisting of up to two fields can be\n"
3952 "\t specified using the 'sort' keyword. The sort direction can\n"
3953 "\t be modified by appending '.descending' or '.ascending' to a\n"
3954 "\t sort field. The 'size' parameter can be used to specify more\n"
3955 "\t or fewer than the default 2048 entries for the hashtable size.\n"
3956 "\t If a hist trigger is given a name using the 'name' parameter,\n"
3957 "\t its histogram data will be shared with other triggers of the\n"
3958 "\t same name, and trigger hits will update this common data.\n\n"
3959 "\t Reading the 'hist' file for the event will dump the hash\n"
3960 "\t table in its entirety to stdout. If there are multiple hist\n"
3961 "\t triggers attached to an event, there will be a table for each\n"
3962 "\t trigger in the output. The table displayed for a named\n"
3963 "\t trigger will be the same as any other instance having the\n"
3964 "\t same name. The default format used to display a given field\n"
3965 "\t can be modified by appending any of the following modifiers\n"
3966 "\t to the field name, as applicable:\n\n"
3967 "\t .hex display a number as a hex value\n"
3968 "\t .sym display an address as a symbol\n"
3969 "\t .sym-offset display an address as a symbol and offset\n"
3970 "\t .execname display a common_pid as a program name\n"
3971 "\t .syscall display a syscall id as a syscall name\n\n"
3972 "\t .log2 display log2 value rather than raw number\n\n"
3973 "\t The 'pause' parameter can be used to pause an existing hist\n"
3974 "\t trigger or to start a hist trigger but not log any events\n"
3975 "\t until told to do so. 'continue' can be used to start or\n"
3976 "\t restart a paused hist trigger.\n\n"
3977 "\t The 'clear' parameter will clear the contents of a running\n"
3978 "\t hist trigger and leave its current paused/active state\n"
3980 "\t The enable_hist and disable_hist triggers can be used to\n"
3981 "\t have one event conditionally start and stop another event's\n"
3982 "\t already-attached hist trigger. The syntax is analagous to\n"
3983 "\t the enable_event and disable_event triggers.\n"
3988 tracing_readme_read(struct file *filp, char __user *ubuf,
3989 size_t cnt, loff_t *ppos)
3991 return simple_read_from_buffer(ubuf, cnt, ppos,
3992 readme_msg, strlen(readme_msg));
3995 static const struct file_operations tracing_readme_fops = {
3996 .open = tracing_open_generic,
3997 .read = tracing_readme_read,
3998 .llseek = generic_file_llseek,
4001 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4003 unsigned int *ptr = v;
4005 if (*pos || m->count)
4010 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4012 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4021 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4027 arch_spin_lock(&trace_cmdline_lock);
4029 v = &savedcmd->map_cmdline_to_pid[0];
4031 v = saved_cmdlines_next(m, v, &l);
4039 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4041 arch_spin_unlock(&trace_cmdline_lock);
4045 static int saved_cmdlines_show(struct seq_file *m, void *v)
4047 char buf[TASK_COMM_LEN];
4048 unsigned int *pid = v;
4050 __trace_find_cmdline(*pid, buf);
4051 seq_printf(m, "%d %s\n", *pid, buf);
4055 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4056 .start = saved_cmdlines_start,
4057 .next = saved_cmdlines_next,
4058 .stop = saved_cmdlines_stop,
4059 .show = saved_cmdlines_show,
4062 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4064 if (tracing_disabled)
4067 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4070 static const struct file_operations tracing_saved_cmdlines_fops = {
4071 .open = tracing_saved_cmdlines_open,
4073 .llseek = seq_lseek,
4074 .release = seq_release,
4078 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4079 size_t cnt, loff_t *ppos)
4084 arch_spin_lock(&trace_cmdline_lock);
4085 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4086 arch_spin_unlock(&trace_cmdline_lock);
4088 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4091 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4093 kfree(s->saved_cmdlines);
4094 kfree(s->map_cmdline_to_pid);
4098 static int tracing_resize_saved_cmdlines(unsigned int val)
4100 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4102 s = kmalloc(sizeof(*s), GFP_KERNEL);
4106 if (allocate_cmdlines_buffer(val, s) < 0) {
4111 arch_spin_lock(&trace_cmdline_lock);
4112 savedcmd_temp = savedcmd;
4114 arch_spin_unlock(&trace_cmdline_lock);
4115 free_saved_cmdlines_buffer(savedcmd_temp);
4121 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4122 size_t cnt, loff_t *ppos)
4127 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4131 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4132 if (!val || val > PID_MAX_DEFAULT)
4135 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4144 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4145 .open = tracing_open_generic,
4146 .read = tracing_saved_cmdlines_size_read,
4147 .write = tracing_saved_cmdlines_size_write,
4150 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4151 static union trace_enum_map_item *
4152 update_enum_map(union trace_enum_map_item *ptr)
4154 if (!ptr->map.enum_string) {
4155 if (ptr->tail.next) {
4156 ptr = ptr->tail.next;
4157 /* Set ptr to the next real item (skip head) */
4165 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4167 union trace_enum_map_item *ptr = v;
4170 * Paranoid! If ptr points to end, we don't want to increment past it.
4171 * This really should never happen.
4173 ptr = update_enum_map(ptr);
4174 if (WARN_ON_ONCE(!ptr))
4181 ptr = update_enum_map(ptr);
4186 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4188 union trace_enum_map_item *v;
4191 mutex_lock(&trace_enum_mutex);
4193 v = trace_enum_maps;
4197 while (v && l < *pos) {
4198 v = enum_map_next(m, v, &l);
4204 static void enum_map_stop(struct seq_file *m, void *v)
4206 mutex_unlock(&trace_enum_mutex);
4209 static int enum_map_show(struct seq_file *m, void *v)
4211 union trace_enum_map_item *ptr = v;
4213 seq_printf(m, "%s %ld (%s)\n",
4214 ptr->map.enum_string, ptr->map.enum_value,
4220 static const struct seq_operations tracing_enum_map_seq_ops = {
4221 .start = enum_map_start,
4222 .next = enum_map_next,
4223 .stop = enum_map_stop,
4224 .show = enum_map_show,
4227 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4229 if (tracing_disabled)
4232 return seq_open(filp, &tracing_enum_map_seq_ops);
4235 static const struct file_operations tracing_enum_map_fops = {
4236 .open = tracing_enum_map_open,
4238 .llseek = seq_lseek,
4239 .release = seq_release,
4242 static inline union trace_enum_map_item *
4243 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4245 /* Return tail of array given the head */
4246 return ptr + ptr->head.length + 1;
4250 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4253 struct trace_enum_map **stop;
4254 struct trace_enum_map **map;
4255 union trace_enum_map_item *map_array;
4256 union trace_enum_map_item *ptr;
4261 * The trace_enum_maps contains the map plus a head and tail item,
4262 * where the head holds the module and length of array, and the
4263 * tail holds a pointer to the next list.
4265 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4267 pr_warn("Unable to allocate trace enum mapping\n");
4271 mutex_lock(&trace_enum_mutex);
4273 if (!trace_enum_maps)
4274 trace_enum_maps = map_array;
4276 ptr = trace_enum_maps;
4278 ptr = trace_enum_jmp_to_tail(ptr);
4279 if (!ptr->tail.next)
4281 ptr = ptr->tail.next;
4284 ptr->tail.next = map_array;
4286 map_array->head.mod = mod;
4287 map_array->head.length = len;
4290 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4291 map_array->map = **map;
4294 memset(map_array, 0, sizeof(*map_array));
4296 mutex_unlock(&trace_enum_mutex);
4299 static void trace_create_enum_file(struct dentry *d_tracer)
4301 trace_create_file("enum_map", 0444, d_tracer,
4302 NULL, &tracing_enum_map_fops);
4305 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4306 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4307 static inline void trace_insert_enum_map_file(struct module *mod,
4308 struct trace_enum_map **start, int len) { }
4309 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4311 static void trace_insert_enum_map(struct module *mod,
4312 struct trace_enum_map **start, int len)
4314 struct trace_enum_map **map;
4321 trace_event_enum_update(map, len);
4323 trace_insert_enum_map_file(mod, start, len);
4327 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4328 size_t cnt, loff_t *ppos)
4330 struct trace_array *tr = filp->private_data;
4331 char buf[MAX_TRACER_SIZE+2];
4334 mutex_lock(&trace_types_lock);
4335 r = sprintf(buf, "%s\n", tr->current_trace->name);
4336 mutex_unlock(&trace_types_lock);
4338 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4341 int tracer_init(struct tracer *t, struct trace_array *tr)
4343 tracing_reset_online_cpus(&tr->trace_buffer);
4347 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4351 for_each_tracing_cpu(cpu)
4352 per_cpu_ptr(buf->data, cpu)->entries = val;
4355 #ifdef CONFIG_TRACER_MAX_TRACE
4356 /* resize @tr's buffer to the size of @size_tr's entries */
4357 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4358 struct trace_buffer *size_buf, int cpu_id)
4362 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4363 for_each_tracing_cpu(cpu) {
4364 ret = ring_buffer_resize(trace_buf->buffer,
4365 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4368 per_cpu_ptr(trace_buf->data, cpu)->entries =
4369 per_cpu_ptr(size_buf->data, cpu)->entries;
4372 ret = ring_buffer_resize(trace_buf->buffer,
4373 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4375 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4376 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4381 #endif /* CONFIG_TRACER_MAX_TRACE */
4383 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4384 unsigned long size, int cpu)
4389 * If kernel or user changes the size of the ring buffer
4390 * we use the size that was given, and we can forget about
4391 * expanding it later.
4393 ring_buffer_expanded = true;
4395 /* May be called before buffers are initialized */
4396 if (!tr->trace_buffer.buffer)
4399 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4403 #ifdef CONFIG_TRACER_MAX_TRACE
4404 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4405 !tr->current_trace->use_max_tr)
4408 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4410 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4411 &tr->trace_buffer, cpu);
4414 * AARGH! We are left with different
4415 * size max buffer!!!!
4416 * The max buffer is our "snapshot" buffer.
4417 * When a tracer needs a snapshot (one of the
4418 * latency tracers), it swaps the max buffer
4419 * with the saved snap shot. We succeeded to
4420 * update the size of the main buffer, but failed to
4421 * update the size of the max buffer. But when we tried
4422 * to reset the main buffer to the original size, we
4423 * failed there too. This is very unlikely to
4424 * happen, but if it does, warn and kill all
4428 tracing_disabled = 1;
4433 if (cpu == RING_BUFFER_ALL_CPUS)
4434 set_buffer_entries(&tr->max_buffer, size);
4436 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4439 #endif /* CONFIG_TRACER_MAX_TRACE */
4441 if (cpu == RING_BUFFER_ALL_CPUS)
4442 set_buffer_entries(&tr->trace_buffer, size);
4444 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4449 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4450 unsigned long size, int cpu_id)
4454 mutex_lock(&trace_types_lock);
4456 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4457 /* make sure, this cpu is enabled in the mask */
4458 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4464 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4469 mutex_unlock(&trace_types_lock);
4476 * tracing_update_buffers - used by tracing facility to expand ring buffers
4478 * To save on memory when the tracing is never used on a system with it
4479 * configured in. The ring buffers are set to a minimum size. But once
4480 * a user starts to use the tracing facility, then they need to grow
4481 * to their default size.
4483 * This function is to be called when a tracer is about to be used.
4485 int tracing_update_buffers(void)
4489 mutex_lock(&trace_types_lock);
4490 if (!ring_buffer_expanded)
4491 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4492 RING_BUFFER_ALL_CPUS);
4493 mutex_unlock(&trace_types_lock);
4498 struct trace_option_dentry;
4501 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4504 * Used to clear out the tracer before deletion of an instance.
4505 * Must have trace_types_lock held.
4507 static void tracing_set_nop(struct trace_array *tr)
4509 if (tr->current_trace == &nop_trace)
4512 tr->current_trace->enabled--;
4514 if (tr->current_trace->reset)
4515 tr->current_trace->reset(tr);
4517 tr->current_trace = &nop_trace;
4520 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4522 /* Only enable if the directory has been created already. */
4526 create_trace_option_files(tr, t);
4529 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4532 #ifdef CONFIG_TRACER_MAX_TRACE
4537 mutex_lock(&trace_types_lock);
4539 if (!ring_buffer_expanded) {
4540 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4541 RING_BUFFER_ALL_CPUS);
4547 for (t = trace_types; t; t = t->next) {
4548 if (strcmp(t->name, buf) == 0)
4555 if (t == tr->current_trace)
4558 /* Some tracers are only allowed for the top level buffer */
4559 if (!trace_ok_for_array(t, tr)) {
4564 /* If trace pipe files are being read, we can't change the tracer */
4565 if (tr->current_trace->ref) {
4570 trace_branch_disable();
4572 tr->current_trace->enabled--;
4574 if (tr->current_trace->reset)
4575 tr->current_trace->reset(tr);
4577 /* Current trace needs to be nop_trace before synchronize_sched */
4578 tr->current_trace = &nop_trace;
4580 #ifdef CONFIG_TRACER_MAX_TRACE
4581 had_max_tr = tr->allocated_snapshot;
4583 if (had_max_tr && !t->use_max_tr) {
4585 * We need to make sure that the update_max_tr sees that
4586 * current_trace changed to nop_trace to keep it from
4587 * swapping the buffers after we resize it.
4588 * The update_max_tr is called from interrupts disabled
4589 * so a synchronized_sched() is sufficient.
4591 synchronize_sched();
4596 #ifdef CONFIG_TRACER_MAX_TRACE
4597 if (t->use_max_tr && !had_max_tr) {
4598 ret = alloc_snapshot(tr);
4605 ret = tracer_init(t, tr);
4610 tr->current_trace = t;
4611 tr->current_trace->enabled++;
4612 trace_branch_enable(tr);
4614 mutex_unlock(&trace_types_lock);
4620 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4621 size_t cnt, loff_t *ppos)
4623 struct trace_array *tr = filp->private_data;
4624 char buf[MAX_TRACER_SIZE+1];
4631 if (cnt > MAX_TRACER_SIZE)
4632 cnt = MAX_TRACER_SIZE;
4634 if (copy_from_user(buf, ubuf, cnt))
4639 /* strip ending whitespace. */
4640 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4643 err = tracing_set_tracer(tr, buf);
4653 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4654 size_t cnt, loff_t *ppos)
4659 r = snprintf(buf, sizeof(buf), "%ld\n",
4660 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4661 if (r > sizeof(buf))
4663 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4667 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4668 size_t cnt, loff_t *ppos)
4673 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4683 tracing_thresh_read(struct file *filp, char __user *ubuf,
4684 size_t cnt, loff_t *ppos)
4686 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4690 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4691 size_t cnt, loff_t *ppos)
4693 struct trace_array *tr = filp->private_data;
4696 mutex_lock(&trace_types_lock);
4697 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4701 if (tr->current_trace->update_thresh) {
4702 ret = tr->current_trace->update_thresh(tr);
4709 mutex_unlock(&trace_types_lock);
4714 #ifdef CONFIG_TRACER_MAX_TRACE
4717 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4718 size_t cnt, loff_t *ppos)
4720 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4724 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4725 size_t cnt, loff_t *ppos)
4727 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4732 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4734 struct trace_array *tr = inode->i_private;
4735 struct trace_iterator *iter;
4738 if (tracing_disabled)
4741 if (trace_array_get(tr) < 0)
4744 mutex_lock(&trace_types_lock);
4746 /* create a buffer to store the information to pass to userspace */
4747 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4750 __trace_array_put(tr);
4754 trace_seq_init(&iter->seq);
4755 iter->trace = tr->current_trace;
4757 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4762 /* trace pipe does not show start of buffer */
4763 cpumask_setall(iter->started);
4765 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4766 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4768 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4769 if (trace_clocks[tr->clock_id].in_ns)
4770 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4773 iter->trace_buffer = &tr->trace_buffer;
4774 iter->cpu_file = tracing_get_cpu(inode);
4775 mutex_init(&iter->mutex);
4776 filp->private_data = iter;
4778 if (iter->trace->pipe_open)
4779 iter->trace->pipe_open(iter);
4781 nonseekable_open(inode, filp);
4783 tr->current_trace->ref++;
4785 mutex_unlock(&trace_types_lock);
4791 __trace_array_put(tr);
4792 mutex_unlock(&trace_types_lock);
4796 static int tracing_release_pipe(struct inode *inode, struct file *file)
4798 struct trace_iterator *iter = file->private_data;
4799 struct trace_array *tr = inode->i_private;
4801 mutex_lock(&trace_types_lock);
4803 tr->current_trace->ref--;
4805 if (iter->trace->pipe_close)
4806 iter->trace->pipe_close(iter);
4808 mutex_unlock(&trace_types_lock);
4810 free_cpumask_var(iter->started);
4811 mutex_destroy(&iter->mutex);
4814 trace_array_put(tr);
4820 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4822 struct trace_array *tr = iter->tr;
4824 /* Iterators are static, they should be filled or empty */
4825 if (trace_buffer_iter(iter, iter->cpu_file))
4826 return POLLIN | POLLRDNORM;
4828 if (tr->trace_flags & TRACE_ITER_BLOCK)
4830 * Always select as readable when in blocking mode
4832 return POLLIN | POLLRDNORM;
4834 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4839 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4841 struct trace_iterator *iter = filp->private_data;
4843 return trace_poll(iter, filp, poll_table);
4846 /* Must be called with iter->mutex held. */
4847 static int tracing_wait_pipe(struct file *filp)
4849 struct trace_iterator *iter = filp->private_data;
4852 while (trace_empty(iter)) {
4854 if ((filp->f_flags & O_NONBLOCK)) {
4859 * We block until we read something and tracing is disabled.
4860 * We still block if tracing is disabled, but we have never
4861 * read anything. This allows a user to cat this file, and
4862 * then enable tracing. But after we have read something,
4863 * we give an EOF when tracing is again disabled.
4865 * iter->pos will be 0 if we haven't read anything.
4867 if (!tracing_is_on() && iter->pos)
4870 mutex_unlock(&iter->mutex);
4872 ret = wait_on_pipe(iter, false);
4874 mutex_lock(&iter->mutex);
4887 tracing_read_pipe(struct file *filp, char __user *ubuf,
4888 size_t cnt, loff_t *ppos)
4890 struct trace_iterator *iter = filp->private_data;
4894 * Avoid more than one consumer on a single file descriptor
4895 * This is just a matter of traces coherency, the ring buffer itself
4898 mutex_lock(&iter->mutex);
4900 /* return any leftover data */
4901 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4905 trace_seq_init(&iter->seq);
4907 if (iter->trace->read) {
4908 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4914 sret = tracing_wait_pipe(filp);
4918 /* stop when tracing is finished */
4919 if (trace_empty(iter)) {
4924 if (cnt >= PAGE_SIZE)
4925 cnt = PAGE_SIZE - 1;
4927 /* reset all but tr, trace, and overruns */
4928 memset(&iter->seq, 0,
4929 sizeof(struct trace_iterator) -
4930 offsetof(struct trace_iterator, seq));
4931 cpumask_clear(iter->started);
4934 trace_event_read_lock();
4935 trace_access_lock(iter->cpu_file);
4936 while (trace_find_next_entry_inc(iter) != NULL) {
4937 enum print_line_t ret;
4938 int save_len = iter->seq.seq.len;
4940 ret = print_trace_line(iter);
4941 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4942 /* don't print partial lines */
4943 iter->seq.seq.len = save_len;
4946 if (ret != TRACE_TYPE_NO_CONSUME)
4947 trace_consume(iter);
4949 if (trace_seq_used(&iter->seq) >= cnt)
4953 * Setting the full flag means we reached the trace_seq buffer
4954 * size and we should leave by partial output condition above.
4955 * One of the trace_seq_* functions is not used properly.
4957 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4960 trace_access_unlock(iter->cpu_file);
4961 trace_event_read_unlock();
4963 /* Now copy what we have to the user */
4964 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4965 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4966 trace_seq_init(&iter->seq);
4969 * If there was nothing to send to user, in spite of consuming trace
4970 * entries, go back to wait for more entries.
4976 mutex_unlock(&iter->mutex);
4981 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4984 __free_page(spd->pages[idx]);
4987 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4989 .confirm = generic_pipe_buf_confirm,
4990 .release = generic_pipe_buf_release,
4991 .steal = generic_pipe_buf_steal,
4992 .get = generic_pipe_buf_get,
4996 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5002 /* Seq buffer is page-sized, exactly what we need. */
5004 save_len = iter->seq.seq.len;
5005 ret = print_trace_line(iter);
5007 if (trace_seq_has_overflowed(&iter->seq)) {
5008 iter->seq.seq.len = save_len;
5013 * This should not be hit, because it should only
5014 * be set if the iter->seq overflowed. But check it
5015 * anyway to be safe.
5017 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5018 iter->seq.seq.len = save_len;
5022 count = trace_seq_used(&iter->seq) - save_len;
5025 iter->seq.seq.len = save_len;
5029 if (ret != TRACE_TYPE_NO_CONSUME)
5030 trace_consume(iter);
5032 if (!trace_find_next_entry_inc(iter)) {
5042 static ssize_t tracing_splice_read_pipe(struct file *filp,
5044 struct pipe_inode_info *pipe,
5048 struct page *pages_def[PIPE_DEF_BUFFERS];
5049 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5050 struct trace_iterator *iter = filp->private_data;
5051 struct splice_pipe_desc spd = {
5053 .partial = partial_def,
5054 .nr_pages = 0, /* This gets updated below. */
5055 .nr_pages_max = PIPE_DEF_BUFFERS,
5057 .ops = &tracing_pipe_buf_ops,
5058 .spd_release = tracing_spd_release_pipe,
5064 if (splice_grow_spd(pipe, &spd))
5067 mutex_lock(&iter->mutex);
5069 if (iter->trace->splice_read) {
5070 ret = iter->trace->splice_read(iter, filp,
5071 ppos, pipe, len, flags);
5076 ret = tracing_wait_pipe(filp);
5080 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5085 trace_event_read_lock();
5086 trace_access_lock(iter->cpu_file);
5088 /* Fill as many pages as possible. */
5089 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5090 spd.pages[i] = alloc_page(GFP_KERNEL);
5094 rem = tracing_fill_pipe_page(rem, iter);
5096 /* Copy the data into the page, so we can start over. */
5097 ret = trace_seq_to_buffer(&iter->seq,
5098 page_address(spd.pages[i]),
5099 trace_seq_used(&iter->seq));
5101 __free_page(spd.pages[i]);
5104 spd.partial[i].offset = 0;
5105 spd.partial[i].len = trace_seq_used(&iter->seq);
5107 trace_seq_init(&iter->seq);
5110 trace_access_unlock(iter->cpu_file);
5111 trace_event_read_unlock();
5112 mutex_unlock(&iter->mutex);
5117 ret = splice_to_pipe(pipe, &spd);
5121 splice_shrink_spd(&spd);
5125 mutex_unlock(&iter->mutex);
5130 tracing_entries_read(struct file *filp, char __user *ubuf,
5131 size_t cnt, loff_t *ppos)
5133 struct inode *inode = file_inode(filp);
5134 struct trace_array *tr = inode->i_private;
5135 int cpu = tracing_get_cpu(inode);
5140 mutex_lock(&trace_types_lock);
5142 if (cpu == RING_BUFFER_ALL_CPUS) {
5143 int cpu, buf_size_same;
5148 /* check if all cpu sizes are same */
5149 for_each_tracing_cpu(cpu) {
5150 /* fill in the size from first enabled cpu */
5152 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5153 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5159 if (buf_size_same) {
5160 if (!ring_buffer_expanded)
5161 r = sprintf(buf, "%lu (expanded: %lu)\n",
5163 trace_buf_size >> 10);
5165 r = sprintf(buf, "%lu\n", size >> 10);
5167 r = sprintf(buf, "X\n");
5169 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5171 mutex_unlock(&trace_types_lock);
5173 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5178 tracing_entries_write(struct file *filp, const char __user *ubuf,
5179 size_t cnt, loff_t *ppos)
5181 struct inode *inode = file_inode(filp);
5182 struct trace_array *tr = inode->i_private;
5186 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5190 /* must have at least 1 entry */
5194 /* value is in KB */
5196 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5206 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5207 size_t cnt, loff_t *ppos)
5209 struct trace_array *tr = filp->private_data;
5212 unsigned long size = 0, expanded_size = 0;
5214 mutex_lock(&trace_types_lock);
5215 for_each_tracing_cpu(cpu) {
5216 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5217 if (!ring_buffer_expanded)
5218 expanded_size += trace_buf_size >> 10;
5220 if (ring_buffer_expanded)
5221 r = sprintf(buf, "%lu\n", size);
5223 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5224 mutex_unlock(&trace_types_lock);
5226 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5230 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5231 size_t cnt, loff_t *ppos)
5234 * There is no need to read what the user has written, this function
5235 * is just to make sure that there is no error when "echo" is used
5244 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5246 struct trace_array *tr = inode->i_private;
5248 /* disable tracing ? */
5249 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5250 tracer_tracing_off(tr);
5251 /* resize the ring buffer to 0 */
5252 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5254 trace_array_put(tr);
5260 tracing_mark_write(struct file *filp, const char __user *ubuf,
5261 size_t cnt, loff_t *fpos)
5263 unsigned long addr = (unsigned long)ubuf;
5264 struct trace_array *tr = filp->private_data;
5265 struct ring_buffer_event *event;
5266 struct ring_buffer *buffer;
5267 struct print_entry *entry;
5268 unsigned long irq_flags;
5269 struct page *pages[2];
5279 if (tracing_disabled)
5282 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5285 if (cnt > TRACE_BUF_SIZE)
5286 cnt = TRACE_BUF_SIZE;
5289 * Userspace is injecting traces into the kernel trace buffer.
5290 * We want to be as non intrusive as possible.
5291 * To do so, we do not want to allocate any special buffers
5292 * or take any locks, but instead write the userspace data
5293 * straight into the ring buffer.
5295 * First we need to pin the userspace buffer into memory,
5296 * which, most likely it is, because it just referenced it.
5297 * But there's no guarantee that it is. By using get_user_pages_fast()
5298 * and kmap_atomic/kunmap_atomic() we can get access to the
5299 * pages directly. We then write the data directly into the
5302 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5304 /* check if we cross pages */
5305 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5308 offset = addr & (PAGE_SIZE - 1);
5311 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5312 if (ret < nr_pages) {
5314 put_page(pages[ret]);
5319 for (i = 0; i < nr_pages; i++)
5320 map_page[i] = kmap_atomic(pages[i]);
5322 local_save_flags(irq_flags);
5323 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5324 buffer = tr->trace_buffer.buffer;
5325 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5326 irq_flags, preempt_count());
5328 /* Ring buffer disabled, return as if not open for write */
5333 entry = ring_buffer_event_data(event);
5334 entry->ip = _THIS_IP_;
5336 if (nr_pages == 2) {
5337 len = PAGE_SIZE - offset;
5338 memcpy(&entry->buf, map_page[0] + offset, len);
5339 memcpy(&entry->buf[len], map_page[1], cnt - len);
5341 memcpy(&entry->buf, map_page[0] + offset, cnt);
5343 if (entry->buf[cnt - 1] != '\n') {
5344 entry->buf[cnt] = '\n';
5345 entry->buf[cnt + 1] = '\0';
5347 entry->buf[cnt] = '\0';
5349 __buffer_unlock_commit(buffer, event);
5356 for (i = nr_pages - 1; i >= 0; i--) {
5357 kunmap_atomic(map_page[i]);
5364 static int tracing_clock_show(struct seq_file *m, void *v)
5366 struct trace_array *tr = m->private;
5369 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5371 "%s%s%s%s", i ? " " : "",
5372 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5373 i == tr->clock_id ? "]" : "");
5379 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5383 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5384 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5387 if (i == ARRAY_SIZE(trace_clocks))
5390 mutex_lock(&trace_types_lock);
5394 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5397 * New clock may not be consistent with the previous clock.
5398 * Reset the buffer so that it doesn't have incomparable timestamps.
5400 tracing_reset_online_cpus(&tr->trace_buffer);
5402 #ifdef CONFIG_TRACER_MAX_TRACE
5403 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5404 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5405 tracing_reset_online_cpus(&tr->max_buffer);
5408 mutex_unlock(&trace_types_lock);
5413 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5414 size_t cnt, loff_t *fpos)
5416 struct seq_file *m = filp->private_data;
5417 struct trace_array *tr = m->private;
5419 const char *clockstr;
5422 if (cnt >= sizeof(buf))
5425 if (copy_from_user(buf, ubuf, cnt))
5430 clockstr = strstrip(buf);
5432 ret = tracing_set_clock(tr, clockstr);
5441 static int tracing_clock_open(struct inode *inode, struct file *file)
5443 struct trace_array *tr = inode->i_private;
5446 if (tracing_disabled)
5449 if (trace_array_get(tr))
5452 ret = single_open(file, tracing_clock_show, inode->i_private);
5454 trace_array_put(tr);
5459 struct ftrace_buffer_info {
5460 struct trace_iterator iter;
5465 #ifdef CONFIG_TRACER_SNAPSHOT
5466 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5468 struct trace_array *tr = inode->i_private;
5469 struct trace_iterator *iter;
5473 if (trace_array_get(tr) < 0)
5476 if (file->f_mode & FMODE_READ) {
5477 iter = __tracing_open(inode, file, true);
5479 ret = PTR_ERR(iter);
5481 /* Writes still need the seq_file to hold the private data */
5483 m = kzalloc(sizeof(*m), GFP_KERNEL);
5486 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5494 iter->trace_buffer = &tr->max_buffer;
5495 iter->cpu_file = tracing_get_cpu(inode);
5497 file->private_data = m;
5501 trace_array_put(tr);
5507 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5510 struct seq_file *m = filp->private_data;
5511 struct trace_iterator *iter = m->private;
5512 struct trace_array *tr = iter->tr;
5516 ret = tracing_update_buffers();
5520 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5524 mutex_lock(&trace_types_lock);
5526 if (tr->current_trace->use_max_tr) {
5533 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5537 if (tr->allocated_snapshot)
5541 /* Only allow per-cpu swap if the ring buffer supports it */
5542 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5543 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5548 if (!tr->allocated_snapshot) {
5549 ret = alloc_snapshot(tr);
5553 local_irq_disable();
5554 /* Now, we're going to swap */
5555 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5556 update_max_tr(tr, current, smp_processor_id());
5558 update_max_tr_single(tr, current, iter->cpu_file);
5562 if (tr->allocated_snapshot) {
5563 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5564 tracing_reset_online_cpus(&tr->max_buffer);
5566 tracing_reset(&tr->max_buffer, iter->cpu_file);
5576 mutex_unlock(&trace_types_lock);
5580 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5582 struct seq_file *m = file->private_data;
5585 ret = tracing_release(inode, file);
5587 if (file->f_mode & FMODE_READ)
5590 /* If write only, the seq_file is just a stub */
5598 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5599 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5600 size_t count, loff_t *ppos);
5601 static int tracing_buffers_release(struct inode *inode, struct file *file);
5602 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5603 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5605 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5607 struct ftrace_buffer_info *info;
5610 ret = tracing_buffers_open(inode, filp);
5614 info = filp->private_data;
5616 if (info->iter.trace->use_max_tr) {
5617 tracing_buffers_release(inode, filp);
5621 info->iter.snapshot = true;
5622 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5627 #endif /* CONFIG_TRACER_SNAPSHOT */
5630 static const struct file_operations tracing_thresh_fops = {
5631 .open = tracing_open_generic,
5632 .read = tracing_thresh_read,
5633 .write = tracing_thresh_write,
5634 .llseek = generic_file_llseek,
5637 #ifdef CONFIG_TRACER_MAX_TRACE
5638 static const struct file_operations tracing_max_lat_fops = {
5639 .open = tracing_open_generic,
5640 .read = tracing_max_lat_read,
5641 .write = tracing_max_lat_write,
5642 .llseek = generic_file_llseek,
5646 static const struct file_operations set_tracer_fops = {
5647 .open = tracing_open_generic,
5648 .read = tracing_set_trace_read,
5649 .write = tracing_set_trace_write,
5650 .llseek = generic_file_llseek,
5653 static const struct file_operations tracing_pipe_fops = {
5654 .open = tracing_open_pipe,
5655 .poll = tracing_poll_pipe,
5656 .read = tracing_read_pipe,
5657 .splice_read = tracing_splice_read_pipe,
5658 .release = tracing_release_pipe,
5659 .llseek = no_llseek,
5662 static const struct file_operations tracing_entries_fops = {
5663 .open = tracing_open_generic_tr,
5664 .read = tracing_entries_read,
5665 .write = tracing_entries_write,
5666 .llseek = generic_file_llseek,
5667 .release = tracing_release_generic_tr,
5670 static const struct file_operations tracing_total_entries_fops = {
5671 .open = tracing_open_generic_tr,
5672 .read = tracing_total_entries_read,
5673 .llseek = generic_file_llseek,
5674 .release = tracing_release_generic_tr,
5677 static const struct file_operations tracing_free_buffer_fops = {
5678 .open = tracing_open_generic_tr,
5679 .write = tracing_free_buffer_write,
5680 .release = tracing_free_buffer_release,
5683 static const struct file_operations tracing_mark_fops = {
5684 .open = tracing_open_generic_tr,
5685 .write = tracing_mark_write,
5686 .llseek = generic_file_llseek,
5687 .release = tracing_release_generic_tr,
5690 static const struct file_operations trace_clock_fops = {
5691 .open = tracing_clock_open,
5693 .llseek = seq_lseek,
5694 .release = tracing_single_release_tr,
5695 .write = tracing_clock_write,
5698 #ifdef CONFIG_TRACER_SNAPSHOT
5699 static const struct file_operations snapshot_fops = {
5700 .open = tracing_snapshot_open,
5702 .write = tracing_snapshot_write,
5703 .llseek = tracing_lseek,
5704 .release = tracing_snapshot_release,
5707 static const struct file_operations snapshot_raw_fops = {
5708 .open = snapshot_raw_open,
5709 .read = tracing_buffers_read,
5710 .release = tracing_buffers_release,
5711 .splice_read = tracing_buffers_splice_read,
5712 .llseek = no_llseek,
5715 #endif /* CONFIG_TRACER_SNAPSHOT */
5717 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5719 struct trace_array *tr = inode->i_private;
5720 struct ftrace_buffer_info *info;
5723 if (tracing_disabled)
5726 if (trace_array_get(tr) < 0)
5729 info = kzalloc(sizeof(*info), GFP_KERNEL);
5731 trace_array_put(tr);
5735 mutex_lock(&trace_types_lock);
5738 info->iter.cpu_file = tracing_get_cpu(inode);
5739 info->iter.trace = tr->current_trace;
5740 info->iter.trace_buffer = &tr->trace_buffer;
5742 /* Force reading ring buffer for first read */
5743 info->read = (unsigned int)-1;
5745 filp->private_data = info;
5747 tr->current_trace->ref++;
5749 mutex_unlock(&trace_types_lock);
5751 ret = nonseekable_open(inode, filp);
5753 trace_array_put(tr);
5759 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5761 struct ftrace_buffer_info *info = filp->private_data;
5762 struct trace_iterator *iter = &info->iter;
5764 return trace_poll(iter, filp, poll_table);
5768 tracing_buffers_read(struct file *filp, char __user *ubuf,
5769 size_t count, loff_t *ppos)
5771 struct ftrace_buffer_info *info = filp->private_data;
5772 struct trace_iterator *iter = &info->iter;
5779 #ifdef CONFIG_TRACER_MAX_TRACE
5780 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5785 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5790 /* Do we have previous read data to read? */
5791 if (info->read < PAGE_SIZE)
5795 trace_access_lock(iter->cpu_file);
5796 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5800 trace_access_unlock(iter->cpu_file);
5803 if (trace_empty(iter)) {
5804 if ((filp->f_flags & O_NONBLOCK))
5807 ret = wait_on_pipe(iter, false);
5818 size = PAGE_SIZE - info->read;
5822 ret = copy_to_user(ubuf, info->spare + info->read, size);
5834 static int tracing_buffers_release(struct inode *inode, struct file *file)
5836 struct ftrace_buffer_info *info = file->private_data;
5837 struct trace_iterator *iter = &info->iter;
5839 mutex_lock(&trace_types_lock);
5841 iter->tr->current_trace->ref--;
5843 __trace_array_put(iter->tr);
5846 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5849 mutex_unlock(&trace_types_lock);
5855 struct ring_buffer *buffer;
5860 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5861 struct pipe_buffer *buf)
5863 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5868 ring_buffer_free_read_page(ref->buffer, ref->page);
5873 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5874 struct pipe_buffer *buf)
5876 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5881 /* Pipe buffer operations for a buffer. */
5882 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5884 .confirm = generic_pipe_buf_confirm,
5885 .release = buffer_pipe_buf_release,
5886 .steal = generic_pipe_buf_steal,
5887 .get = buffer_pipe_buf_get,
5891 * Callback from splice_to_pipe(), if we need to release some pages
5892 * at the end of the spd in case we error'ed out in filling the pipe.
5894 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5896 struct buffer_ref *ref =
5897 (struct buffer_ref *)spd->partial[i].private;
5902 ring_buffer_free_read_page(ref->buffer, ref->page);
5904 spd->partial[i].private = 0;
5908 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5909 struct pipe_inode_info *pipe, size_t len,
5912 struct ftrace_buffer_info *info = file->private_data;
5913 struct trace_iterator *iter = &info->iter;
5914 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5915 struct page *pages_def[PIPE_DEF_BUFFERS];
5916 struct splice_pipe_desc spd = {
5918 .partial = partial_def,
5919 .nr_pages_max = PIPE_DEF_BUFFERS,
5921 .ops = &buffer_pipe_buf_ops,
5922 .spd_release = buffer_spd_release,
5924 struct buffer_ref *ref;
5925 int entries, size, i;
5928 #ifdef CONFIG_TRACER_MAX_TRACE
5929 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5933 if (splice_grow_spd(pipe, &spd))
5936 if (*ppos & (PAGE_SIZE - 1))
5939 if (len & (PAGE_SIZE - 1)) {
5940 if (len < PAGE_SIZE)
5946 trace_access_lock(iter->cpu_file);
5947 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5949 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5953 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5960 ref->buffer = iter->trace_buffer->buffer;
5961 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5968 r = ring_buffer_read_page(ref->buffer, &ref->page,
5969 len, iter->cpu_file, 1);
5971 ring_buffer_free_read_page(ref->buffer, ref->page);
5977 * zero out any left over data, this is going to
5980 size = ring_buffer_page_len(ref->page);
5981 if (size < PAGE_SIZE)
5982 memset(ref->page + size, 0, PAGE_SIZE - size);
5984 page = virt_to_page(ref->page);
5986 spd.pages[i] = page;
5987 spd.partial[i].len = PAGE_SIZE;
5988 spd.partial[i].offset = 0;
5989 spd.partial[i].private = (unsigned long)ref;
5993 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5996 trace_access_unlock(iter->cpu_file);
5999 /* did we read anything? */
6000 if (!spd.nr_pages) {
6004 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6007 ret = wait_on_pipe(iter, true);
6014 ret = splice_to_pipe(pipe, &spd);
6015 splice_shrink_spd(&spd);
6020 static const struct file_operations tracing_buffers_fops = {
6021 .open = tracing_buffers_open,
6022 .read = tracing_buffers_read,
6023 .poll = tracing_buffers_poll,
6024 .release = tracing_buffers_release,
6025 .splice_read = tracing_buffers_splice_read,
6026 .llseek = no_llseek,
6030 tracing_stats_read(struct file *filp, char __user *ubuf,
6031 size_t count, loff_t *ppos)
6033 struct inode *inode = file_inode(filp);
6034 struct trace_array *tr = inode->i_private;
6035 struct trace_buffer *trace_buf = &tr->trace_buffer;
6036 int cpu = tracing_get_cpu(inode);
6037 struct trace_seq *s;
6039 unsigned long long t;
6040 unsigned long usec_rem;
6042 s = kmalloc(sizeof(*s), GFP_KERNEL);
6048 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6049 trace_seq_printf(s, "entries: %ld\n", cnt);
6051 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6052 trace_seq_printf(s, "overrun: %ld\n", cnt);
6054 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6055 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6057 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6058 trace_seq_printf(s, "bytes: %ld\n", cnt);
6060 if (trace_clocks[tr->clock_id].in_ns) {
6061 /* local or global for trace_clock */
6062 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6063 usec_rem = do_div(t, USEC_PER_SEC);
6064 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6067 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6068 usec_rem = do_div(t, USEC_PER_SEC);
6069 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6071 /* counter or tsc mode for trace_clock */
6072 trace_seq_printf(s, "oldest event ts: %llu\n",
6073 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6075 trace_seq_printf(s, "now ts: %llu\n",
6076 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6079 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6080 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6082 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6083 trace_seq_printf(s, "read events: %ld\n", cnt);
6085 count = simple_read_from_buffer(ubuf, count, ppos,
6086 s->buffer, trace_seq_used(s));
6093 static const struct file_operations tracing_stats_fops = {
6094 .open = tracing_open_generic_tr,
6095 .read = tracing_stats_read,
6096 .llseek = generic_file_llseek,
6097 .release = tracing_release_generic_tr,
6100 #ifdef CONFIG_DYNAMIC_FTRACE
6102 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6108 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6109 size_t cnt, loff_t *ppos)
6111 static char ftrace_dyn_info_buffer[1024];
6112 static DEFINE_MUTEX(dyn_info_mutex);
6113 unsigned long *p = filp->private_data;
6114 char *buf = ftrace_dyn_info_buffer;
6115 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
6118 mutex_lock(&dyn_info_mutex);
6119 r = sprintf(buf, "%ld ", *p);
6121 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
6124 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6126 mutex_unlock(&dyn_info_mutex);
6131 static const struct file_operations tracing_dyn_info_fops = {
6132 .open = tracing_open_generic,
6133 .read = tracing_read_dyn_info,
6134 .llseek = generic_file_llseek,
6136 #endif /* CONFIG_DYNAMIC_FTRACE */
6138 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6140 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6146 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6148 unsigned long *count = (long *)data;
6160 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6161 struct ftrace_probe_ops *ops, void *data)
6163 long count = (long)data;
6165 seq_printf(m, "%ps:", (void *)ip);
6167 seq_puts(m, "snapshot");
6170 seq_puts(m, ":unlimited\n");
6172 seq_printf(m, ":count=%ld\n", count);
6177 static struct ftrace_probe_ops snapshot_probe_ops = {
6178 .func = ftrace_snapshot,
6179 .print = ftrace_snapshot_print,
6182 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6183 .func = ftrace_count_snapshot,
6184 .print = ftrace_snapshot_print,
6188 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6189 char *glob, char *cmd, char *param, int enable)
6191 struct ftrace_probe_ops *ops;
6192 void *count = (void *)-1;
6196 /* hash funcs only work with set_ftrace_filter */
6200 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6202 if (glob[0] == '!') {
6203 unregister_ftrace_function_probe_func(glob+1, ops);
6210 number = strsep(¶m, ":");
6212 if (!strlen(number))
6216 * We use the callback data field (which is a pointer)
6219 ret = kstrtoul(number, 0, (unsigned long *)&count);
6224 ret = register_ftrace_function_probe(glob, ops, count);
6227 alloc_snapshot(&global_trace);
6229 return ret < 0 ? ret : 0;
6232 static struct ftrace_func_command ftrace_snapshot_cmd = {
6234 .func = ftrace_trace_snapshot_callback,
6237 static __init int register_snapshot_cmd(void)
6239 return register_ftrace_command(&ftrace_snapshot_cmd);
6242 static inline __init int register_snapshot_cmd(void) { return 0; }
6243 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6245 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6247 if (WARN_ON(!tr->dir))
6248 return ERR_PTR(-ENODEV);
6250 /* Top directory uses NULL as the parent */
6251 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6254 /* All sub buffers have a descriptor */
6258 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6260 struct dentry *d_tracer;
6263 return tr->percpu_dir;
6265 d_tracer = tracing_get_dentry(tr);
6266 if (IS_ERR(d_tracer))
6269 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6271 WARN_ONCE(!tr->percpu_dir,
6272 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6274 return tr->percpu_dir;
6277 static struct dentry *
6278 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6279 void *data, long cpu, const struct file_operations *fops)
6281 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6283 if (ret) /* See tracing_get_cpu() */
6284 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6289 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6291 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6292 struct dentry *d_cpu;
6293 char cpu_dir[30]; /* 30 characters should be more than enough */
6298 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6299 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6301 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6305 /* per cpu trace_pipe */
6306 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6307 tr, cpu, &tracing_pipe_fops);
6310 trace_create_cpu_file("trace", 0644, d_cpu,
6311 tr, cpu, &tracing_fops);
6313 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6314 tr, cpu, &tracing_buffers_fops);
6316 trace_create_cpu_file("stats", 0444, d_cpu,
6317 tr, cpu, &tracing_stats_fops);
6319 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6320 tr, cpu, &tracing_entries_fops);
6322 #ifdef CONFIG_TRACER_SNAPSHOT
6323 trace_create_cpu_file("snapshot", 0644, d_cpu,
6324 tr, cpu, &snapshot_fops);
6326 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6327 tr, cpu, &snapshot_raw_fops);
6331 #ifdef CONFIG_FTRACE_SELFTEST
6332 /* Let selftest have access to static functions in this file */
6333 #include "trace_selftest.c"
6337 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6340 struct trace_option_dentry *topt = filp->private_data;
6343 if (topt->flags->val & topt->opt->bit)
6348 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6352 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6355 struct trace_option_dentry *topt = filp->private_data;
6359 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6363 if (val != 0 && val != 1)
6366 if (!!(topt->flags->val & topt->opt->bit) != val) {
6367 mutex_lock(&trace_types_lock);
6368 ret = __set_tracer_option(topt->tr, topt->flags,
6370 mutex_unlock(&trace_types_lock);
6381 static const struct file_operations trace_options_fops = {
6382 .open = tracing_open_generic,
6383 .read = trace_options_read,
6384 .write = trace_options_write,
6385 .llseek = generic_file_llseek,
6389 * In order to pass in both the trace_array descriptor as well as the index
6390 * to the flag that the trace option file represents, the trace_array
6391 * has a character array of trace_flags_index[], which holds the index
6392 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6393 * The address of this character array is passed to the flag option file
6394 * read/write callbacks.
6396 * In order to extract both the index and the trace_array descriptor,
6397 * get_tr_index() uses the following algorithm.
6401 * As the pointer itself contains the address of the index (remember
6404 * Then to get the trace_array descriptor, by subtracting that index
6405 * from the ptr, we get to the start of the index itself.
6407 * ptr - idx == &index[0]
6409 * Then a simple container_of() from that pointer gets us to the
6410 * trace_array descriptor.
6412 static void get_tr_index(void *data, struct trace_array **ptr,
6413 unsigned int *pindex)
6415 *pindex = *(unsigned char *)data;
6417 *ptr = container_of(data - *pindex, struct trace_array,
6422 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6425 void *tr_index = filp->private_data;
6426 struct trace_array *tr;
6430 get_tr_index(tr_index, &tr, &index);
6432 if (tr->trace_flags & (1 << index))
6437 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6441 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6444 void *tr_index = filp->private_data;
6445 struct trace_array *tr;
6450 get_tr_index(tr_index, &tr, &index);
6452 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6456 if (val != 0 && val != 1)
6459 mutex_lock(&trace_types_lock);
6460 ret = set_tracer_flag(tr, 1 << index, val);
6461 mutex_unlock(&trace_types_lock);
6471 static const struct file_operations trace_options_core_fops = {
6472 .open = tracing_open_generic,
6473 .read = trace_options_core_read,
6474 .write = trace_options_core_write,
6475 .llseek = generic_file_llseek,
6478 struct dentry *trace_create_file(const char *name,
6480 struct dentry *parent,
6482 const struct file_operations *fops)
6486 ret = tracefs_create_file(name, mode, parent, data, fops);
6488 pr_warn("Could not create tracefs '%s' entry\n", name);
6494 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6496 struct dentry *d_tracer;
6501 d_tracer = tracing_get_dentry(tr);
6502 if (IS_ERR(d_tracer))
6505 tr->options = tracefs_create_dir("options", d_tracer);
6507 pr_warn("Could not create tracefs directory 'options'\n");
6515 create_trace_option_file(struct trace_array *tr,
6516 struct trace_option_dentry *topt,
6517 struct tracer_flags *flags,
6518 struct tracer_opt *opt)
6520 struct dentry *t_options;
6522 t_options = trace_options_init_dentry(tr);
6526 topt->flags = flags;
6530 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6531 &trace_options_fops);
6536 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6538 struct trace_option_dentry *topts;
6539 struct trace_options *tr_topts;
6540 struct tracer_flags *flags;
6541 struct tracer_opt *opts;
6548 flags = tracer->flags;
6550 if (!flags || !flags->opts)
6554 * If this is an instance, only create flags for tracers
6555 * the instance may have.
6557 if (!trace_ok_for_array(tracer, tr))
6560 for (i = 0; i < tr->nr_topts; i++) {
6561 /* Make sure there's no duplicate flags. */
6562 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6568 for (cnt = 0; opts[cnt].name; cnt++)
6571 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6575 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6582 tr->topts = tr_topts;
6583 tr->topts[tr->nr_topts].tracer = tracer;
6584 tr->topts[tr->nr_topts].topts = topts;
6587 for (cnt = 0; opts[cnt].name; cnt++) {
6588 create_trace_option_file(tr, &topts[cnt], flags,
6590 WARN_ONCE(topts[cnt].entry == NULL,
6591 "Failed to create trace option: %s",
6596 static struct dentry *
6597 create_trace_option_core_file(struct trace_array *tr,
6598 const char *option, long index)
6600 struct dentry *t_options;
6602 t_options = trace_options_init_dentry(tr);
6606 return trace_create_file(option, 0644, t_options,
6607 (void *)&tr->trace_flags_index[index],
6608 &trace_options_core_fops);
6611 static void create_trace_options_dir(struct trace_array *tr)
6613 struct dentry *t_options;
6614 bool top_level = tr == &global_trace;
6617 t_options = trace_options_init_dentry(tr);
6621 for (i = 0; trace_options[i]; i++) {
6623 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6624 create_trace_option_core_file(tr, trace_options[i], i);
6629 rb_simple_read(struct file *filp, char __user *ubuf,
6630 size_t cnt, loff_t *ppos)
6632 struct trace_array *tr = filp->private_data;
6636 r = tracer_tracing_is_on(tr);
6637 r = sprintf(buf, "%d\n", r);
6639 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6643 rb_simple_write(struct file *filp, const char __user *ubuf,
6644 size_t cnt, loff_t *ppos)
6646 struct trace_array *tr = filp->private_data;
6647 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6651 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6656 mutex_lock(&trace_types_lock);
6658 tracer_tracing_on(tr);
6659 if (tr->current_trace->start)
6660 tr->current_trace->start(tr);
6662 tracer_tracing_off(tr);
6663 if (tr->current_trace->stop)
6664 tr->current_trace->stop(tr);
6666 mutex_unlock(&trace_types_lock);
6674 static const struct file_operations rb_simple_fops = {
6675 .open = tracing_open_generic_tr,
6676 .read = rb_simple_read,
6677 .write = rb_simple_write,
6678 .release = tracing_release_generic_tr,
6679 .llseek = default_llseek,
6682 struct dentry *trace_instance_dir;
6685 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6688 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6690 enum ring_buffer_flags rb_flags;
6692 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6696 buf->buffer = ring_buffer_alloc(size, rb_flags);
6700 buf->data = alloc_percpu(struct trace_array_cpu);
6702 ring_buffer_free(buf->buffer);
6706 /* Allocate the first page for all buffers */
6707 set_buffer_entries(&tr->trace_buffer,
6708 ring_buffer_size(tr->trace_buffer.buffer, 0));
6713 static int allocate_trace_buffers(struct trace_array *tr, int size)
6717 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6721 #ifdef CONFIG_TRACER_MAX_TRACE
6722 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6723 allocate_snapshot ? size : 1);
6725 ring_buffer_free(tr->trace_buffer.buffer);
6726 free_percpu(tr->trace_buffer.data);
6729 tr->allocated_snapshot = allocate_snapshot;
6732 * Only the top level trace array gets its snapshot allocated
6733 * from the kernel command line.
6735 allocate_snapshot = false;
6740 static void free_trace_buffer(struct trace_buffer *buf)
6743 ring_buffer_free(buf->buffer);
6745 free_percpu(buf->data);
6750 static void free_trace_buffers(struct trace_array *tr)
6755 free_trace_buffer(&tr->trace_buffer);
6757 #ifdef CONFIG_TRACER_MAX_TRACE
6758 free_trace_buffer(&tr->max_buffer);
6762 static void init_trace_flags_index(struct trace_array *tr)
6766 /* Used by the trace options files */
6767 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6768 tr->trace_flags_index[i] = i;
6771 static void __update_tracer_options(struct trace_array *tr)
6775 for (t = trace_types; t; t = t->next)
6776 add_tracer_options(tr, t);
6779 static void update_tracer_options(struct trace_array *tr)
6781 mutex_lock(&trace_types_lock);
6782 __update_tracer_options(tr);
6783 mutex_unlock(&trace_types_lock);
6786 static int instance_mkdir(const char *name)
6788 struct trace_array *tr;
6791 mutex_lock(&trace_types_lock);
6794 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6795 if (tr->name && strcmp(tr->name, name) == 0)
6800 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6804 tr->name = kstrdup(name, GFP_KERNEL);
6808 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6811 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
6813 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6815 raw_spin_lock_init(&tr->start_lock);
6817 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6819 tr->current_trace = &nop_trace;
6821 INIT_LIST_HEAD(&tr->systems);
6822 INIT_LIST_HEAD(&tr->events);
6824 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6827 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6831 ret = event_trace_add_tracer(tr->dir, tr);
6833 tracefs_remove_recursive(tr->dir);
6837 init_tracer_tracefs(tr, tr->dir);
6838 init_trace_flags_index(tr);
6839 __update_tracer_options(tr);
6841 list_add(&tr->list, &ftrace_trace_arrays);
6843 mutex_unlock(&trace_types_lock);
6848 free_trace_buffers(tr);
6849 free_cpumask_var(tr->tracing_cpumask);
6854 mutex_unlock(&trace_types_lock);
6860 static int instance_rmdir(const char *name)
6862 struct trace_array *tr;
6867 mutex_lock(&trace_types_lock);
6870 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6871 if (tr->name && strcmp(tr->name, name) == 0) {
6880 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6883 list_del(&tr->list);
6885 /* Disable all the flags that were enabled coming in */
6886 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
6887 if ((1 << i) & ZEROED_TRACE_FLAGS)
6888 set_tracer_flag(tr, 1 << i, 0);
6891 tracing_set_nop(tr);
6892 event_trace_del_tracer(tr);
6893 ftrace_destroy_function_files(tr);
6894 tracefs_remove_recursive(tr->dir);
6895 free_trace_buffers(tr);
6897 for (i = 0; i < tr->nr_topts; i++) {
6898 kfree(tr->topts[i].topts);
6908 mutex_unlock(&trace_types_lock);
6913 static __init void create_trace_instances(struct dentry *d_tracer)
6915 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6918 if (WARN_ON(!trace_instance_dir))
6923 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6927 trace_create_file("available_tracers", 0444, d_tracer,
6928 tr, &show_traces_fops);
6930 trace_create_file("current_tracer", 0644, d_tracer,
6931 tr, &set_tracer_fops);
6933 trace_create_file("tracing_cpumask", 0644, d_tracer,
6934 tr, &tracing_cpumask_fops);
6936 trace_create_file("trace_options", 0644, d_tracer,
6937 tr, &tracing_iter_fops);
6939 trace_create_file("trace", 0644, d_tracer,
6942 trace_create_file("trace_pipe", 0444, d_tracer,
6943 tr, &tracing_pipe_fops);
6945 trace_create_file("buffer_size_kb", 0644, d_tracer,
6946 tr, &tracing_entries_fops);
6948 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6949 tr, &tracing_total_entries_fops);
6951 trace_create_file("free_buffer", 0200, d_tracer,
6952 tr, &tracing_free_buffer_fops);
6954 trace_create_file("trace_marker", 0220, d_tracer,
6955 tr, &tracing_mark_fops);
6957 trace_create_file("trace_clock", 0644, d_tracer, tr,
6960 trace_create_file("tracing_on", 0644, d_tracer,
6961 tr, &rb_simple_fops);
6963 create_trace_options_dir(tr);
6965 #ifdef CONFIG_TRACER_MAX_TRACE
6966 trace_create_file("tracing_max_latency", 0644, d_tracer,
6967 &tr->max_latency, &tracing_max_lat_fops);
6970 if (ftrace_create_function_files(tr, d_tracer))
6971 WARN(1, "Could not allocate function filter files");
6973 #ifdef CONFIG_TRACER_SNAPSHOT
6974 trace_create_file("snapshot", 0644, d_tracer,
6975 tr, &snapshot_fops);
6978 for_each_tracing_cpu(cpu)
6979 tracing_init_tracefs_percpu(tr, cpu);
6983 static struct vfsmount *trace_automount(void *ingore)
6985 struct vfsmount *mnt;
6986 struct file_system_type *type;
6989 * To maintain backward compatibility for tools that mount
6990 * debugfs to get to the tracing facility, tracefs is automatically
6991 * mounted to the debugfs/tracing directory.
6993 type = get_fs_type("tracefs");
6996 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6997 put_filesystem(type);
7006 * tracing_init_dentry - initialize top level trace array
7008 * This is called when creating files or directories in the tracing
7009 * directory. It is called via fs_initcall() by any of the boot up code
7010 * and expects to return the dentry of the top level tracing directory.
7012 struct dentry *tracing_init_dentry(void)
7014 struct trace_array *tr = &global_trace;
7016 /* The top level trace array uses NULL as parent */
7020 if (WARN_ON(!tracefs_initialized()) ||
7021 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7022 WARN_ON(!debugfs_initialized())))
7023 return ERR_PTR(-ENODEV);
7026 * As there may still be users that expect the tracing
7027 * files to exist in debugfs/tracing, we must automount
7028 * the tracefs file system there, so older tools still
7029 * work with the newer kerenl.
7031 tr->dir = debugfs_create_automount("tracing", NULL,
7032 trace_automount, NULL);
7034 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7035 return ERR_PTR(-ENOMEM);
7041 extern struct trace_enum_map *__start_ftrace_enum_maps[];
7042 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7044 static void __init trace_enum_init(void)
7048 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
7049 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
7052 #ifdef CONFIG_MODULES
7053 static void trace_module_add_enums(struct module *mod)
7055 if (!mod->num_trace_enums)
7059 * Modules with bad taint do not have events created, do
7060 * not bother with enums either.
7062 if (trace_module_has_bad_taint(mod))
7065 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
7068 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7069 static void trace_module_remove_enums(struct module *mod)
7071 union trace_enum_map_item *map;
7072 union trace_enum_map_item **last = &trace_enum_maps;
7074 if (!mod->num_trace_enums)
7077 mutex_lock(&trace_enum_mutex);
7079 map = trace_enum_maps;
7082 if (map->head.mod == mod)
7084 map = trace_enum_jmp_to_tail(map);
7085 last = &map->tail.next;
7086 map = map->tail.next;
7091 *last = trace_enum_jmp_to_tail(map)->tail.next;
7094 mutex_unlock(&trace_enum_mutex);
7097 static inline void trace_module_remove_enums(struct module *mod) { }
7098 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7100 static int trace_module_notify(struct notifier_block *self,
7101 unsigned long val, void *data)
7103 struct module *mod = data;
7106 case MODULE_STATE_COMING:
7107 trace_module_add_enums(mod);
7109 case MODULE_STATE_GOING:
7110 trace_module_remove_enums(mod);
7117 static struct notifier_block trace_module_nb = {
7118 .notifier_call = trace_module_notify,
7121 #endif /* CONFIG_MODULES */
7123 static __init int tracer_init_tracefs(void)
7125 struct dentry *d_tracer;
7127 trace_access_lock_init();
7129 d_tracer = tracing_init_dentry();
7130 if (IS_ERR(d_tracer))
7133 init_tracer_tracefs(&global_trace, d_tracer);
7135 trace_create_file("tracing_thresh", 0644, d_tracer,
7136 &global_trace, &tracing_thresh_fops);
7138 trace_create_file("README", 0444, d_tracer,
7139 NULL, &tracing_readme_fops);
7141 trace_create_file("saved_cmdlines", 0444, d_tracer,
7142 NULL, &tracing_saved_cmdlines_fops);
7144 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7145 NULL, &tracing_saved_cmdlines_size_fops);
7149 trace_create_enum_file(d_tracer);
7151 #ifdef CONFIG_MODULES
7152 register_module_notifier(&trace_module_nb);
7155 #ifdef CONFIG_DYNAMIC_FTRACE
7156 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7157 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7160 create_trace_instances(d_tracer);
7162 update_tracer_options(&global_trace);
7167 static int trace_panic_handler(struct notifier_block *this,
7168 unsigned long event, void *unused)
7170 if (ftrace_dump_on_oops)
7171 ftrace_dump(ftrace_dump_on_oops);
7175 static struct notifier_block trace_panic_notifier = {
7176 .notifier_call = trace_panic_handler,
7178 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7181 static int trace_die_handler(struct notifier_block *self,
7187 if (ftrace_dump_on_oops)
7188 ftrace_dump(ftrace_dump_on_oops);
7196 static struct notifier_block trace_die_notifier = {
7197 .notifier_call = trace_die_handler,
7202 * printk is set to max of 1024, we really don't need it that big.
7203 * Nothing should be printing 1000 characters anyway.
7205 #define TRACE_MAX_PRINT 1000
7208 * Define here KERN_TRACE so that we have one place to modify
7209 * it if we decide to change what log level the ftrace dump
7212 #define KERN_TRACE KERN_EMERG
7215 trace_printk_seq(struct trace_seq *s)
7217 /* Probably should print a warning here. */
7218 if (s->seq.len >= TRACE_MAX_PRINT)
7219 s->seq.len = TRACE_MAX_PRINT;
7222 * More paranoid code. Although the buffer size is set to
7223 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7224 * an extra layer of protection.
7226 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7227 s->seq.len = s->seq.size - 1;
7229 /* should be zero ended, but we are paranoid. */
7230 s->buffer[s->seq.len] = 0;
7232 printk(KERN_TRACE "%s", s->buffer);
7237 void trace_init_global_iter(struct trace_iterator *iter)
7239 iter->tr = &global_trace;
7240 iter->trace = iter->tr->current_trace;
7241 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7242 iter->trace_buffer = &global_trace.trace_buffer;
7244 if (iter->trace && iter->trace->open)
7245 iter->trace->open(iter);
7247 /* Annotate start of buffers if we had overruns */
7248 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7249 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7251 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7252 if (trace_clocks[iter->tr->clock_id].in_ns)
7253 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7256 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7258 /* use static because iter can be a bit big for the stack */
7259 static struct trace_iterator iter;
7260 static atomic_t dump_running;
7261 struct trace_array *tr = &global_trace;
7262 unsigned int old_userobj;
7263 unsigned long flags;
7266 /* Only allow one dump user at a time. */
7267 if (atomic_inc_return(&dump_running) != 1) {
7268 atomic_dec(&dump_running);
7273 * Always turn off tracing when we dump.
7274 * We don't need to show trace output of what happens
7275 * between multiple crashes.
7277 * If the user does a sysrq-z, then they can re-enable
7278 * tracing with echo 1 > tracing_on.
7282 local_irq_save(flags);
7284 /* Simulate the iterator */
7285 trace_init_global_iter(&iter);
7287 for_each_tracing_cpu(cpu) {
7288 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7291 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7293 /* don't look at user memory in panic mode */
7294 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7296 switch (oops_dump_mode) {
7298 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7301 iter.cpu_file = raw_smp_processor_id();
7306 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7307 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7310 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7312 /* Did function tracer already get disabled? */
7313 if (ftrace_is_dead()) {
7314 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7315 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7319 * We need to stop all tracing on all CPUS to read the
7320 * the next buffer. This is a bit expensive, but is
7321 * not done often. We fill all what we can read,
7322 * and then release the locks again.
7325 while (!trace_empty(&iter)) {
7328 printk(KERN_TRACE "---------------------------------\n");
7332 /* reset all but tr, trace, and overruns */
7333 memset(&iter.seq, 0,
7334 sizeof(struct trace_iterator) -
7335 offsetof(struct trace_iterator, seq));
7336 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7339 if (trace_find_next_entry_inc(&iter) != NULL) {
7342 ret = print_trace_line(&iter);
7343 if (ret != TRACE_TYPE_NO_CONSUME)
7344 trace_consume(&iter);
7346 touch_nmi_watchdog();
7348 trace_printk_seq(&iter.seq);
7352 printk(KERN_TRACE " (ftrace buffer empty)\n");
7354 printk(KERN_TRACE "---------------------------------\n");
7357 tr->trace_flags |= old_userobj;
7359 for_each_tracing_cpu(cpu) {
7360 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7362 atomic_dec(&dump_running);
7363 local_irq_restore(flags);
7365 EXPORT_SYMBOL_GPL(ftrace_dump);
7367 __init static int tracer_alloc_buffers(void)
7373 * Make sure we don't accidently add more trace options
7374 * than we have bits for.
7376 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7378 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7381 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7382 goto out_free_buffer_mask;
7384 /* Only allocate trace_printk buffers if a trace_printk exists */
7385 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7386 /* Must be called before global_trace.buffer is allocated */
7387 trace_printk_init_buffers();
7389 /* To save memory, keep the ring buffer size to its minimum */
7390 if (ring_buffer_expanded)
7391 ring_buf_size = trace_buf_size;
7395 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7396 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7398 raw_spin_lock_init(&global_trace.start_lock);
7400 /* Used for event triggers */
7401 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7403 goto out_free_cpumask;
7405 if (trace_create_savedcmd() < 0)
7406 goto out_free_temp_buffer;
7408 /* TODO: make the number of buffers hot pluggable with CPUS */
7409 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7410 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7412 goto out_free_savedcmd;
7415 if (global_trace.buffer_disabled)
7418 if (trace_boot_clock) {
7419 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7421 pr_warn("Trace clock %s not defined, going back to default\n",
7426 * register_tracer() might reference current_trace, so it
7427 * needs to be set before we register anything. This is
7428 * just a bootstrap of current_trace anyway.
7430 global_trace.current_trace = &nop_trace;
7432 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7434 ftrace_init_global_array_ops(&global_trace);
7436 init_trace_flags_index(&global_trace);
7438 register_tracer(&nop_trace);
7440 /* All seems OK, enable tracing */
7441 tracing_disabled = 0;
7443 atomic_notifier_chain_register(&panic_notifier_list,
7444 &trace_panic_notifier);
7446 register_die_notifier(&trace_die_notifier);
7448 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7450 INIT_LIST_HEAD(&global_trace.systems);
7451 INIT_LIST_HEAD(&global_trace.events);
7452 list_add(&global_trace.list, &ftrace_trace_arrays);
7454 apply_trace_boot_options();
7456 register_snapshot_cmd();
7461 free_saved_cmdlines_buffer(savedcmd);
7462 out_free_temp_buffer:
7463 ring_buffer_free(temp_buffer);
7465 free_cpumask_var(global_trace.tracing_cpumask);
7466 out_free_buffer_mask:
7467 free_cpumask_var(tracing_buffer_mask);
7472 void __init trace_init(void)
7474 if (tracepoint_printk) {
7475 tracepoint_print_iter =
7476 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7477 if (WARN_ON(!tracepoint_print_iter))
7478 tracepoint_printk = 0;
7480 tracer_alloc_buffers();
7484 __init static int clear_boot_tracer(void)
7487 * The default tracer at boot buffer is an init section.
7488 * This function is called in lateinit. If we did not
7489 * find the boot tracer, then clear it out, to prevent
7490 * later registration from accessing the buffer that is
7491 * about to be freed.
7493 if (!default_bootup_tracer)
7496 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7497 default_bootup_tracer);
7498 default_bootup_tracer = NULL;
7503 fs_initcall(tracer_init_tracefs);
7504 late_initcall(clear_boot_tracer);