2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 __disable_trace_on_warning = 1;
161 __setup("traceoff_on_warning=", stop_trace_on_warning);
163 static int __init boot_alloc_snapshot(char *str)
165 allocate_snapshot = true;
166 /* We also need the main ring buffer expanded */
167 ring_buffer_expanded = true;
170 __setup("alloc_snapshot", boot_alloc_snapshot);
173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
174 static char *trace_boot_options __initdata;
176 static int __init set_trace_boot_options(char *str)
178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 trace_boot_options = trace_boot_options_buf;
182 __setup("trace_options=", set_trace_boot_options);
184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
185 static char *trace_boot_clock __initdata;
187 static int __init set_trace_boot_clock(char *str)
189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
190 trace_boot_clock = trace_boot_clock_buf;
193 __setup("trace_clock=", set_trace_boot_clock);
196 unsigned long long ns2usecs(cycle_t nsec)
204 * The global_trace is the descriptor that holds the tracing
205 * buffers for the live tracing. For each CPU, it contains
206 * a link list of pages that will store trace entries. The
207 * page descriptor of the pages in the memory is used to hold
208 * the link list by linking the lru item in the page descriptor
209 * to each of the pages in the buffer per CPU.
211 * For each active CPU there is a data field that holds the
212 * pages for the buffer for that CPU. Each CPU has the same number
213 * of pages allocated for its buffer.
215 static struct trace_array global_trace;
217 LIST_HEAD(ftrace_trace_arrays);
219 int trace_array_get(struct trace_array *this_tr)
221 struct trace_array *tr;
224 mutex_lock(&trace_types_lock);
225 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
232 mutex_unlock(&trace_types_lock);
237 static void __trace_array_put(struct trace_array *this_tr)
239 WARN_ON(!this_tr->ref);
243 void trace_array_put(struct trace_array *this_tr)
245 mutex_lock(&trace_types_lock);
246 __trace_array_put(this_tr);
247 mutex_unlock(&trace_types_lock);
250 int filter_check_discard(struct ftrace_event_file *file, void *rec,
251 struct ring_buffer *buffer,
252 struct ring_buffer_event *event)
254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
255 !filter_match_preds(file->filter, rec)) {
256 ring_buffer_discard_commit(buffer, event);
262 EXPORT_SYMBOL_GPL(filter_check_discard);
264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
265 struct ring_buffer *buffer,
266 struct ring_buffer_event *event)
268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
269 !filter_match_preds(call->filter, rec)) {
270 ring_buffer_discard_commit(buffer, event);
276 EXPORT_SYMBOL_GPL(call_filter_check_discard);
278 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
282 /* Early boot up does not have a buffer yet */
284 return trace_clock_local();
286 ts = ring_buffer_time_stamp(buf->buffer, cpu);
287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
292 cycle_t ftrace_now(int cpu)
294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
298 * tracing_is_enabled - Show if global_trace has been disabled
300 * Shows if the global trace has been enabled or not. It uses the
301 * mirror flag "buffer_disabled" to be used in fast paths such as for
302 * the irqsoff tracer. But it may be inaccurate due to races. If you
303 * need to know the accurate state, use tracing_is_on() which is a little
304 * slower, but accurate.
306 int tracing_is_enabled(void)
309 * For quick access (irqsoff uses this in fast path), just
310 * return the mirror variable of the state of the ring buffer.
311 * It's a little racy, but we don't really care.
314 return !global_trace.buffer_disabled;
318 * trace_buf_size is the size in bytes that is allocated
319 * for a buffer. Note, the number of bytes is always rounded
322 * This number is purposely set to a low number of 16384.
323 * If the dump on oops happens, it will be much appreciated
324 * to not have to wait for all that output. Anyway this can be
325 * boot time and run time configurable.
327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
331 /* trace_types holds a link list of available tracers. */
332 static struct tracer *trace_types __read_mostly;
335 * trace_types_lock is used to protect the trace_types list.
337 DEFINE_MUTEX(trace_types_lock);
340 * serialize the access of the ring buffer
342 * ring buffer serializes readers, but it is low level protection.
343 * The validity of the events (which returns by ring_buffer_peek() ..etc)
344 * are not protected by ring buffer.
346 * The content of events may become garbage if we allow other process consumes
347 * these events concurrently:
348 * A) the page of the consumed events may become a normal page
349 * (not reader page) in ring buffer, and this page will be rewrited
350 * by events producer.
351 * B) The page of the consumed events may become a page for splice_read,
352 * and this page will be returned to system.
354 * These primitives allow multi process access to different cpu ring buffer
357 * These primitives don't distinguish read-only and read-consume access.
358 * Multi read-only access are also serialized.
362 static DECLARE_RWSEM(all_cpu_access_lock);
363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365 static inline void trace_access_lock(int cpu)
367 if (cpu == RING_BUFFER_ALL_CPUS) {
368 /* gain it for accessing the whole ring buffer. */
369 down_write(&all_cpu_access_lock);
371 /* gain it for accessing a cpu ring buffer. */
373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 down_read(&all_cpu_access_lock);
376 /* Secondly block other access to this @cpu ring buffer. */
377 mutex_lock(&per_cpu(cpu_access_lock, cpu));
381 static inline void trace_access_unlock(int cpu)
383 if (cpu == RING_BUFFER_ALL_CPUS) {
384 up_write(&all_cpu_access_lock);
386 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
387 up_read(&all_cpu_access_lock);
391 static inline void trace_access_lock_init(void)
395 for_each_possible_cpu(cpu)
396 mutex_init(&per_cpu(cpu_access_lock, cpu));
401 static DEFINE_MUTEX(access_lock);
403 static inline void trace_access_lock(int cpu)
406 mutex_lock(&access_lock);
409 static inline void trace_access_unlock(int cpu)
412 mutex_unlock(&access_lock);
415 static inline void trace_access_lock_init(void)
421 /* trace_flags holds trace_options default values */
422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
427 static void tracer_tracing_on(struct trace_array *tr)
429 if (tr->trace_buffer.buffer)
430 ring_buffer_record_on(tr->trace_buffer.buffer);
432 * This flag is looked at when buffers haven't been allocated
433 * yet, or by some tracers (like irqsoff), that just want to
434 * know if the ring buffer has been disabled, but it can handle
435 * races of where it gets disabled but we still do a record.
436 * As the check is in the fast path of the tracers, it is more
437 * important to be fast than accurate.
439 tr->buffer_disabled = 0;
440 /* Make the flag seen by readers */
445 * tracing_on - enable tracing buffers
447 * This function enables tracing buffers that may have been
448 * disabled with tracing_off.
450 void tracing_on(void)
452 tracer_tracing_on(&global_trace);
454 EXPORT_SYMBOL_GPL(tracing_on);
457 * __trace_puts - write a constant string into the trace buffer.
458 * @ip: The address of the caller
459 * @str: The constant string to write
460 * @size: The size of the string.
462 int __trace_puts(unsigned long ip, const char *str, int size)
464 struct ring_buffer_event *event;
465 struct ring_buffer *buffer;
466 struct print_entry *entry;
467 unsigned long irq_flags;
470 if (unlikely(tracing_selftest_running || tracing_disabled))
473 alloc = sizeof(*entry) + size + 2; /* possible \n added */
475 local_save_flags(irq_flags);
476 buffer = global_trace.trace_buffer.buffer;
477 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
478 irq_flags, preempt_count());
482 entry = ring_buffer_event_data(event);
485 memcpy(&entry->buf, str, size);
487 /* Add a newline if necessary */
488 if (entry->buf[size - 1] != '\n') {
489 entry->buf[size] = '\n';
490 entry->buf[size + 1] = '\0';
492 entry->buf[size] = '\0';
494 __buffer_unlock_commit(buffer, event);
498 EXPORT_SYMBOL_GPL(__trace_puts);
501 * __trace_bputs - write the pointer to a constant string into trace buffer
502 * @ip: The address of the caller
503 * @str: The constant string to write to the buffer to
505 int __trace_bputs(unsigned long ip, const char *str)
507 struct ring_buffer_event *event;
508 struct ring_buffer *buffer;
509 struct bputs_entry *entry;
510 unsigned long irq_flags;
511 int size = sizeof(struct bputs_entry);
513 if (unlikely(tracing_selftest_running || tracing_disabled))
516 local_save_flags(irq_flags);
517 buffer = global_trace.trace_buffer.buffer;
518 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
519 irq_flags, preempt_count());
523 entry = ring_buffer_event_data(event);
527 __buffer_unlock_commit(buffer, event);
531 EXPORT_SYMBOL_GPL(__trace_bputs);
533 #ifdef CONFIG_TRACER_SNAPSHOT
535 * trace_snapshot - take a snapshot of the current buffer.
537 * This causes a swap between the snapshot buffer and the current live
538 * tracing buffer. You can use this to take snapshots of the live
539 * trace when some condition is triggered, but continue to trace.
541 * Note, make sure to allocate the snapshot with either
542 * a tracing_snapshot_alloc(), or by doing it manually
543 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
545 * If the snapshot buffer is not allocated, it will stop tracing.
546 * Basically making a permanent snapshot.
548 void tracing_snapshot(void)
550 struct trace_array *tr = &global_trace;
551 struct tracer *tracer = tr->current_trace;
555 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
556 internal_trace_puts("*** snapshot is being ignored ***\n");
560 if (!tr->allocated_snapshot) {
561 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
562 internal_trace_puts("*** stopping trace here! ***\n");
567 /* Note, snapshot can not be used when the tracer uses it */
568 if (tracer->use_max_tr) {
569 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
570 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
574 local_irq_save(flags);
575 update_max_tr(tr, current, smp_processor_id());
576 local_irq_restore(flags);
578 EXPORT_SYMBOL_GPL(tracing_snapshot);
580 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
581 struct trace_buffer *size_buf, int cpu_id);
582 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
584 static int alloc_snapshot(struct trace_array *tr)
588 if (!tr->allocated_snapshot) {
590 /* allocate spare buffer */
591 ret = resize_buffer_duplicate_size(&tr->max_buffer,
592 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
596 tr->allocated_snapshot = true;
602 static void free_snapshot(struct trace_array *tr)
605 * We don't free the ring buffer. instead, resize it because
606 * The max_tr ring buffer has some state (e.g. ring->clock) and
607 * we want preserve it.
609 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
610 set_buffer_entries(&tr->max_buffer, 1);
611 tracing_reset_online_cpus(&tr->max_buffer);
612 tr->allocated_snapshot = false;
616 * tracing_alloc_snapshot - allocate snapshot buffer.
618 * This only allocates the snapshot buffer if it isn't already
619 * allocated - it doesn't also take a snapshot.
621 * This is meant to be used in cases where the snapshot buffer needs
622 * to be set up for events that can't sleep but need to be able to
623 * trigger a snapshot.
625 int tracing_alloc_snapshot(void)
627 struct trace_array *tr = &global_trace;
630 ret = alloc_snapshot(tr);
635 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
638 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
640 * This is similar to trace_snapshot(), but it will allocate the
641 * snapshot buffer if it isn't already allocated. Use this only
642 * where it is safe to sleep, as the allocation may sleep.
644 * This causes a swap between the snapshot buffer and the current live
645 * tracing buffer. You can use this to take snapshots of the live
646 * trace when some condition is triggered, but continue to trace.
648 void tracing_snapshot_alloc(void)
652 ret = tracing_alloc_snapshot();
658 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
660 void tracing_snapshot(void)
662 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
664 EXPORT_SYMBOL_GPL(tracing_snapshot);
665 int tracing_alloc_snapshot(void)
667 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
670 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
671 void tracing_snapshot_alloc(void)
676 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
677 #endif /* CONFIG_TRACER_SNAPSHOT */
679 static void tracer_tracing_off(struct trace_array *tr)
681 if (tr->trace_buffer.buffer)
682 ring_buffer_record_off(tr->trace_buffer.buffer);
684 * This flag is looked at when buffers haven't been allocated
685 * yet, or by some tracers (like irqsoff), that just want to
686 * know if the ring buffer has been disabled, but it can handle
687 * races of where it gets disabled but we still do a record.
688 * As the check is in the fast path of the tracers, it is more
689 * important to be fast than accurate.
691 tr->buffer_disabled = 1;
692 /* Make the flag seen by readers */
697 * tracing_off - turn off tracing buffers
699 * This function stops the tracing buffers from recording data.
700 * It does not disable any overhead the tracers themselves may
701 * be causing. This function simply causes all recording to
702 * the ring buffers to fail.
704 void tracing_off(void)
706 tracer_tracing_off(&global_trace);
708 EXPORT_SYMBOL_GPL(tracing_off);
710 void disable_trace_on_warning(void)
712 if (__disable_trace_on_warning)
717 * tracer_tracing_is_on - show real state of ring buffer enabled
718 * @tr : the trace array to know if ring buffer is enabled
720 * Shows real state of the ring buffer if it is enabled or not.
722 static int tracer_tracing_is_on(struct trace_array *tr)
724 if (tr->trace_buffer.buffer)
725 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
726 return !tr->buffer_disabled;
730 * tracing_is_on - show state of ring buffers enabled
732 int tracing_is_on(void)
734 return tracer_tracing_is_on(&global_trace);
736 EXPORT_SYMBOL_GPL(tracing_is_on);
738 static int __init set_buf_size(char *str)
740 unsigned long buf_size;
744 buf_size = memparse(str, &str);
745 /* nr_entries can not be zero */
748 trace_buf_size = buf_size;
751 __setup("trace_buf_size=", set_buf_size);
753 static int __init set_tracing_thresh(char *str)
755 unsigned long threshold;
760 ret = kstrtoul(str, 0, &threshold);
763 tracing_thresh = threshold * 1000;
766 __setup("tracing_thresh=", set_tracing_thresh);
768 unsigned long nsecs_to_usecs(unsigned long nsecs)
773 /* These must match the bit postions in trace_iterator_flags */
774 static const char *trace_options[] = {
807 int in_ns; /* is this clock in nanoseconds? */
809 { trace_clock_local, "local", 1 },
810 { trace_clock_global, "global", 1 },
811 { trace_clock_counter, "counter", 0 },
812 { trace_clock_jiffies, "uptime", 1 },
813 { trace_clock, "perf", 1 },
814 { ktime_get_mono_fast_ns, "mono", 1 },
819 * trace_parser_get_init - gets the buffer for trace parser
821 int trace_parser_get_init(struct trace_parser *parser, int size)
823 memset(parser, 0, sizeof(*parser));
825 parser->buffer = kmalloc(size, GFP_KERNEL);
834 * trace_parser_put - frees the buffer for trace parser
836 void trace_parser_put(struct trace_parser *parser)
838 kfree(parser->buffer);
842 * trace_get_user - reads the user input string separated by space
843 * (matched by isspace(ch))
845 * For each string found the 'struct trace_parser' is updated,
846 * and the function returns.
848 * Returns number of bytes read.
850 * See kernel/trace/trace.h for 'struct trace_parser' details.
852 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
853 size_t cnt, loff_t *ppos)
860 trace_parser_clear(parser);
862 ret = get_user(ch, ubuf++);
870 * The parser is not finished with the last write,
871 * continue reading the user input without skipping spaces.
874 /* skip white space */
875 while (cnt && isspace(ch)) {
876 ret = get_user(ch, ubuf++);
883 /* only spaces were written */
893 /* read the non-space input */
894 while (cnt && !isspace(ch)) {
895 if (parser->idx < parser->size - 1)
896 parser->buffer[parser->idx++] = ch;
901 ret = get_user(ch, ubuf++);
908 /* We either got finished input or we have to wait for another call. */
910 parser->buffer[parser->idx] = 0;
911 parser->cont = false;
912 } else if (parser->idx < parser->size - 1) {
914 parser->buffer[parser->idx++] = ch;
927 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
935 if (s->len <= s->readpos)
938 len = s->len - s->readpos;
941 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
951 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
955 if (s->len <= s->readpos)
958 len = s->len - s->readpos;
961 memcpy(buf, s->buffer + s->readpos, cnt);
967 unsigned long __read_mostly tracing_thresh;
969 #ifdef CONFIG_TRACER_MAX_TRACE
971 * Copy the new maximum trace into the separate maximum-trace
972 * structure. (this way the maximum trace is permanently saved,
973 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
976 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
978 struct trace_buffer *trace_buf = &tr->trace_buffer;
979 struct trace_buffer *max_buf = &tr->max_buffer;
980 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
981 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
984 max_buf->time_start = data->preempt_timestamp;
986 max_data->saved_latency = tr->max_latency;
987 max_data->critical_start = data->critical_start;
988 max_data->critical_end = data->critical_end;
990 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
991 max_data->pid = tsk->pid;
993 * If tsk == current, then use current_uid(), as that does not use
994 * RCU. The irq tracer can be called out of RCU scope.
997 max_data->uid = current_uid();
999 max_data->uid = task_uid(tsk);
1001 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1002 max_data->policy = tsk->policy;
1003 max_data->rt_priority = tsk->rt_priority;
1005 /* record this tasks comm */
1006 tracing_record_cmdline(tsk);
1010 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1012 * @tsk: the task with the latency
1013 * @cpu: The cpu that initiated the trace.
1015 * Flip the buffers between the @tr and the max_tr and record information
1016 * about which task was the cause of this latency.
1019 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1021 struct ring_buffer *buf;
1026 WARN_ON_ONCE(!irqs_disabled());
1028 if (!tr->allocated_snapshot) {
1029 /* Only the nop tracer should hit this when disabling */
1030 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1034 arch_spin_lock(&tr->max_lock);
1036 buf = tr->trace_buffer.buffer;
1037 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1038 tr->max_buffer.buffer = buf;
1040 __update_max_tr(tr, tsk, cpu);
1041 arch_spin_unlock(&tr->max_lock);
1045 * update_max_tr_single - only copy one trace over, and reset the rest
1047 * @tsk - task with the latency
1048 * @cpu - the cpu of the buffer to copy.
1050 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1053 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1060 WARN_ON_ONCE(!irqs_disabled());
1061 if (!tr->allocated_snapshot) {
1062 /* Only the nop tracer should hit this when disabling */
1063 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1067 arch_spin_lock(&tr->max_lock);
1069 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1071 if (ret == -EBUSY) {
1073 * We failed to swap the buffer due to a commit taking
1074 * place on this CPU. We fail to record, but we reset
1075 * the max trace buffer (no one writes directly to it)
1076 * and flag that it failed.
1078 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1079 "Failed to swap buffers due to commit in progress\n");
1082 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1084 __update_max_tr(tr, tsk, cpu);
1085 arch_spin_unlock(&tr->max_lock);
1087 #endif /* CONFIG_TRACER_MAX_TRACE */
1089 static int wait_on_pipe(struct trace_iterator *iter)
1091 /* Iterators are static, they should be filled or empty */
1092 if (trace_buffer_iter(iter, iter->cpu_file))
1095 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1098 #ifdef CONFIG_FTRACE_STARTUP_TEST
1099 static int run_tracer_selftest(struct tracer *type)
1101 struct trace_array *tr = &global_trace;
1102 struct tracer *saved_tracer = tr->current_trace;
1105 if (!type->selftest || tracing_selftest_disabled)
1109 * Run a selftest on this tracer.
1110 * Here we reset the trace buffer, and set the current
1111 * tracer to be this tracer. The tracer can then run some
1112 * internal tracing to verify that everything is in order.
1113 * If we fail, we do not register this tracer.
1115 tracing_reset_online_cpus(&tr->trace_buffer);
1117 tr->current_trace = type;
1119 #ifdef CONFIG_TRACER_MAX_TRACE
1120 if (type->use_max_tr) {
1121 /* If we expanded the buffers, make sure the max is expanded too */
1122 if (ring_buffer_expanded)
1123 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1124 RING_BUFFER_ALL_CPUS);
1125 tr->allocated_snapshot = true;
1129 /* the test is responsible for initializing and enabling */
1130 pr_info("Testing tracer %s: ", type->name);
1131 ret = type->selftest(type, tr);
1132 /* the test is responsible for resetting too */
1133 tr->current_trace = saved_tracer;
1135 printk(KERN_CONT "FAILED!\n");
1136 /* Add the warning after printing 'FAILED' */
1140 /* Only reset on passing, to avoid touching corrupted buffers */
1141 tracing_reset_online_cpus(&tr->trace_buffer);
1143 #ifdef CONFIG_TRACER_MAX_TRACE
1144 if (type->use_max_tr) {
1145 tr->allocated_snapshot = false;
1147 /* Shrink the max buffer again */
1148 if (ring_buffer_expanded)
1149 ring_buffer_resize(tr->max_buffer.buffer, 1,
1150 RING_BUFFER_ALL_CPUS);
1154 printk(KERN_CONT "PASSED\n");
1158 static inline int run_tracer_selftest(struct tracer *type)
1162 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1165 * register_tracer - register a tracer with the ftrace system.
1166 * @type - the plugin for the tracer
1168 * Register a new plugin tracer.
1170 int register_tracer(struct tracer *type)
1176 pr_info("Tracer must have a name\n");
1180 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1181 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1185 mutex_lock(&trace_types_lock);
1187 tracing_selftest_running = true;
1189 for (t = trace_types; t; t = t->next) {
1190 if (strcmp(type->name, t->name) == 0) {
1192 pr_info("Tracer %s already registered\n",
1199 if (!type->set_flag)
1200 type->set_flag = &dummy_set_flag;
1202 type->flags = &dummy_tracer_flags;
1204 if (!type->flags->opts)
1205 type->flags->opts = dummy_tracer_opt;
1207 ret = run_tracer_selftest(type);
1211 type->next = trace_types;
1215 tracing_selftest_running = false;
1216 mutex_unlock(&trace_types_lock);
1218 if (ret || !default_bootup_tracer)
1221 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1224 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1225 /* Do we want this tracer to start on bootup? */
1226 tracing_set_tracer(&global_trace, type->name);
1227 default_bootup_tracer = NULL;
1228 /* disable other selftests, since this will break it. */
1229 tracing_selftest_disabled = true;
1230 #ifdef CONFIG_FTRACE_STARTUP_TEST
1231 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1239 void tracing_reset(struct trace_buffer *buf, int cpu)
1241 struct ring_buffer *buffer = buf->buffer;
1246 ring_buffer_record_disable(buffer);
1248 /* Make sure all commits have finished */
1249 synchronize_sched();
1250 ring_buffer_reset_cpu(buffer, cpu);
1252 ring_buffer_record_enable(buffer);
1255 void tracing_reset_online_cpus(struct trace_buffer *buf)
1257 struct ring_buffer *buffer = buf->buffer;
1263 ring_buffer_record_disable(buffer);
1265 /* Make sure all commits have finished */
1266 synchronize_sched();
1268 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1270 for_each_online_cpu(cpu)
1271 ring_buffer_reset_cpu(buffer, cpu);
1273 ring_buffer_record_enable(buffer);
1276 /* Must have trace_types_lock held */
1277 void tracing_reset_all_online_cpus(void)
1279 struct trace_array *tr;
1281 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1282 tracing_reset_online_cpus(&tr->trace_buffer);
1283 #ifdef CONFIG_TRACER_MAX_TRACE
1284 tracing_reset_online_cpus(&tr->max_buffer);
1289 #define SAVED_CMDLINES_DEFAULT 128
1290 #define NO_CMDLINE_MAP UINT_MAX
1291 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1292 struct saved_cmdlines_buffer {
1293 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1294 unsigned *map_cmdline_to_pid;
1295 unsigned cmdline_num;
1297 char *saved_cmdlines;
1299 static struct saved_cmdlines_buffer *savedcmd;
1301 /* temporary disable recording */
1302 static atomic_t trace_record_cmdline_disabled __read_mostly;
1304 static inline char *get_saved_cmdlines(int idx)
1306 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1309 static inline void set_cmdline(int idx, const char *cmdline)
1311 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1314 static int allocate_cmdlines_buffer(unsigned int val,
1315 struct saved_cmdlines_buffer *s)
1317 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1319 if (!s->map_cmdline_to_pid)
1322 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1323 if (!s->saved_cmdlines) {
1324 kfree(s->map_cmdline_to_pid);
1329 s->cmdline_num = val;
1330 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1331 sizeof(s->map_pid_to_cmdline));
1332 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1333 val * sizeof(*s->map_cmdline_to_pid));
1338 static int trace_create_savedcmd(void)
1342 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1346 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1356 int is_tracing_stopped(void)
1358 return global_trace.stop_count;
1362 * tracing_start - quick start of the tracer
1364 * If tracing is enabled but was stopped by tracing_stop,
1365 * this will start the tracer back up.
1367 void tracing_start(void)
1369 struct ring_buffer *buffer;
1370 unsigned long flags;
1372 if (tracing_disabled)
1375 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1376 if (--global_trace.stop_count) {
1377 if (global_trace.stop_count < 0) {
1378 /* Someone screwed up their debugging */
1380 global_trace.stop_count = 0;
1385 /* Prevent the buffers from switching */
1386 arch_spin_lock(&global_trace.max_lock);
1388 buffer = global_trace.trace_buffer.buffer;
1390 ring_buffer_record_enable(buffer);
1392 #ifdef CONFIG_TRACER_MAX_TRACE
1393 buffer = global_trace.max_buffer.buffer;
1395 ring_buffer_record_enable(buffer);
1398 arch_spin_unlock(&global_trace.max_lock);
1401 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1404 static void tracing_start_tr(struct trace_array *tr)
1406 struct ring_buffer *buffer;
1407 unsigned long flags;
1409 if (tracing_disabled)
1412 /* If global, we need to also start the max tracer */
1413 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1414 return tracing_start();
1416 raw_spin_lock_irqsave(&tr->start_lock, flags);
1418 if (--tr->stop_count) {
1419 if (tr->stop_count < 0) {
1420 /* Someone screwed up their debugging */
1427 buffer = tr->trace_buffer.buffer;
1429 ring_buffer_record_enable(buffer);
1432 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1436 * tracing_stop - quick stop of the tracer
1438 * Light weight way to stop tracing. Use in conjunction with
1441 void tracing_stop(void)
1443 struct ring_buffer *buffer;
1444 unsigned long flags;
1446 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1447 if (global_trace.stop_count++)
1450 /* Prevent the buffers from switching */
1451 arch_spin_lock(&global_trace.max_lock);
1453 buffer = global_trace.trace_buffer.buffer;
1455 ring_buffer_record_disable(buffer);
1457 #ifdef CONFIG_TRACER_MAX_TRACE
1458 buffer = global_trace.max_buffer.buffer;
1460 ring_buffer_record_disable(buffer);
1463 arch_spin_unlock(&global_trace.max_lock);
1466 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1469 static void tracing_stop_tr(struct trace_array *tr)
1471 struct ring_buffer *buffer;
1472 unsigned long flags;
1474 /* If global, we need to also stop the max tracer */
1475 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1476 return tracing_stop();
1478 raw_spin_lock_irqsave(&tr->start_lock, flags);
1479 if (tr->stop_count++)
1482 buffer = tr->trace_buffer.buffer;
1484 ring_buffer_record_disable(buffer);
1487 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1490 void trace_stop_cmdline_recording(void);
1492 static int trace_save_cmdline(struct task_struct *tsk)
1496 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1500 * It's not the end of the world if we don't get
1501 * the lock, but we also don't want to spin
1502 * nor do we want to disable interrupts,
1503 * so if we miss here, then better luck next time.
1505 if (!arch_spin_trylock(&trace_cmdline_lock))
1508 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1509 if (idx == NO_CMDLINE_MAP) {
1510 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1513 * Check whether the cmdline buffer at idx has a pid
1514 * mapped. We are going to overwrite that entry so we
1515 * need to clear the map_pid_to_cmdline. Otherwise we
1516 * would read the new comm for the old pid.
1518 pid = savedcmd->map_cmdline_to_pid[idx];
1519 if (pid != NO_CMDLINE_MAP)
1520 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1522 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1523 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1525 savedcmd->cmdline_idx = idx;
1528 set_cmdline(idx, tsk->comm);
1530 arch_spin_unlock(&trace_cmdline_lock);
1535 static void __trace_find_cmdline(int pid, char comm[])
1540 strcpy(comm, "<idle>");
1544 if (WARN_ON_ONCE(pid < 0)) {
1545 strcpy(comm, "<XXX>");
1549 if (pid > PID_MAX_DEFAULT) {
1550 strcpy(comm, "<...>");
1554 map = savedcmd->map_pid_to_cmdline[pid];
1555 if (map != NO_CMDLINE_MAP)
1556 strcpy(comm, get_saved_cmdlines(map));
1558 strcpy(comm, "<...>");
1561 void trace_find_cmdline(int pid, char comm[])
1564 arch_spin_lock(&trace_cmdline_lock);
1566 __trace_find_cmdline(pid, comm);
1568 arch_spin_unlock(&trace_cmdline_lock);
1572 void tracing_record_cmdline(struct task_struct *tsk)
1574 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1577 if (!__this_cpu_read(trace_cmdline_save))
1580 if (trace_save_cmdline(tsk))
1581 __this_cpu_write(trace_cmdline_save, false);
1585 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1588 struct task_struct *tsk = current;
1590 entry->preempt_count = pc & 0xff;
1591 entry->pid = (tsk) ? tsk->pid : 0;
1593 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1594 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1596 TRACE_FLAG_IRQS_NOSUPPORT |
1598 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1599 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1600 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1601 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1603 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1605 struct ring_buffer_event *
1606 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1609 unsigned long flags, int pc)
1611 struct ring_buffer_event *event;
1613 event = ring_buffer_lock_reserve(buffer, len);
1614 if (event != NULL) {
1615 struct trace_entry *ent = ring_buffer_event_data(event);
1617 tracing_generic_entry_update(ent, flags, pc);
1625 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1627 __this_cpu_write(trace_cmdline_save, true);
1628 ring_buffer_unlock_commit(buffer, event);
1632 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1633 struct ring_buffer_event *event,
1634 unsigned long flags, int pc)
1636 __buffer_unlock_commit(buffer, event);
1638 ftrace_trace_stack(buffer, flags, 6, pc);
1639 ftrace_trace_userstack(buffer, flags, pc);
1642 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1643 struct ring_buffer_event *event,
1644 unsigned long flags, int pc)
1646 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1648 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1650 static struct ring_buffer *temp_buffer;
1652 struct ring_buffer_event *
1653 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1654 struct ftrace_event_file *ftrace_file,
1655 int type, unsigned long len,
1656 unsigned long flags, int pc)
1658 struct ring_buffer_event *entry;
1660 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1661 entry = trace_buffer_lock_reserve(*current_rb,
1662 type, len, flags, pc);
1664 * If tracing is off, but we have triggers enabled
1665 * we still need to look at the event data. Use the temp_buffer
1666 * to store the trace event for the tigger to use. It's recusive
1667 * safe and will not be recorded anywhere.
1669 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1670 *current_rb = temp_buffer;
1671 entry = trace_buffer_lock_reserve(*current_rb,
1672 type, len, flags, pc);
1676 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1678 struct ring_buffer_event *
1679 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1680 int type, unsigned long len,
1681 unsigned long flags, int pc)
1683 *current_rb = global_trace.trace_buffer.buffer;
1684 return trace_buffer_lock_reserve(*current_rb,
1685 type, len, flags, pc);
1687 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1689 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1690 struct ring_buffer_event *event,
1691 unsigned long flags, int pc)
1693 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1695 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1697 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1698 struct ring_buffer_event *event,
1699 unsigned long flags, int pc,
1700 struct pt_regs *regs)
1702 __buffer_unlock_commit(buffer, event);
1704 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1705 ftrace_trace_userstack(buffer, flags, pc);
1707 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1709 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1710 struct ring_buffer_event *event)
1712 ring_buffer_discard_commit(buffer, event);
1714 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1717 trace_function(struct trace_array *tr,
1718 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1721 struct ftrace_event_call *call = &event_function;
1722 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1723 struct ring_buffer_event *event;
1724 struct ftrace_entry *entry;
1726 /* If we are reading the ring buffer, don't trace */
1727 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1730 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1734 entry = ring_buffer_event_data(event);
1736 entry->parent_ip = parent_ip;
1738 if (!call_filter_check_discard(call, entry, buffer, event))
1739 __buffer_unlock_commit(buffer, event);
1742 #ifdef CONFIG_STACKTRACE
1744 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1745 struct ftrace_stack {
1746 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1749 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1750 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1752 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1753 unsigned long flags,
1754 int skip, int pc, struct pt_regs *regs)
1756 struct ftrace_event_call *call = &event_kernel_stack;
1757 struct ring_buffer_event *event;
1758 struct stack_entry *entry;
1759 struct stack_trace trace;
1761 int size = FTRACE_STACK_ENTRIES;
1763 trace.nr_entries = 0;
1767 * Since events can happen in NMIs there's no safe way to
1768 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1769 * or NMI comes in, it will just have to use the default
1770 * FTRACE_STACK_SIZE.
1772 preempt_disable_notrace();
1774 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1776 * We don't need any atomic variables, just a barrier.
1777 * If an interrupt comes in, we don't care, because it would
1778 * have exited and put the counter back to what we want.
1779 * We just need a barrier to keep gcc from moving things
1783 if (use_stack == 1) {
1784 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1785 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1788 save_stack_trace_regs(regs, &trace);
1790 save_stack_trace(&trace);
1792 if (trace.nr_entries > size)
1793 size = trace.nr_entries;
1795 /* From now on, use_stack is a boolean */
1798 size *= sizeof(unsigned long);
1800 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1801 sizeof(*entry) + size, flags, pc);
1804 entry = ring_buffer_event_data(event);
1806 memset(&entry->caller, 0, size);
1809 memcpy(&entry->caller, trace.entries,
1810 trace.nr_entries * sizeof(unsigned long));
1812 trace.max_entries = FTRACE_STACK_ENTRIES;
1813 trace.entries = entry->caller;
1815 save_stack_trace_regs(regs, &trace);
1817 save_stack_trace(&trace);
1820 entry->size = trace.nr_entries;
1822 if (!call_filter_check_discard(call, entry, buffer, event))
1823 __buffer_unlock_commit(buffer, event);
1826 /* Again, don't let gcc optimize things here */
1828 __this_cpu_dec(ftrace_stack_reserve);
1829 preempt_enable_notrace();
1833 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1834 int skip, int pc, struct pt_regs *regs)
1836 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1839 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1842 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1845 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1848 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1851 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1854 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1858 * trace_dump_stack - record a stack back trace in the trace buffer
1859 * @skip: Number of functions to skip (helper handlers)
1861 void trace_dump_stack(int skip)
1863 unsigned long flags;
1865 if (tracing_disabled || tracing_selftest_running)
1868 local_save_flags(flags);
1871 * Skip 3 more, seems to get us at the caller of
1875 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1876 flags, skip, preempt_count(), NULL);
1879 static DEFINE_PER_CPU(int, user_stack_count);
1882 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1884 struct ftrace_event_call *call = &event_user_stack;
1885 struct ring_buffer_event *event;
1886 struct userstack_entry *entry;
1887 struct stack_trace trace;
1889 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1893 * NMIs can not handle page faults, even with fix ups.
1894 * The save user stack can (and often does) fault.
1896 if (unlikely(in_nmi()))
1900 * prevent recursion, since the user stack tracing may
1901 * trigger other kernel events.
1904 if (__this_cpu_read(user_stack_count))
1907 __this_cpu_inc(user_stack_count);
1909 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1910 sizeof(*entry), flags, pc);
1912 goto out_drop_count;
1913 entry = ring_buffer_event_data(event);
1915 entry->tgid = current->tgid;
1916 memset(&entry->caller, 0, sizeof(entry->caller));
1918 trace.nr_entries = 0;
1919 trace.max_entries = FTRACE_STACK_ENTRIES;
1921 trace.entries = entry->caller;
1923 save_stack_trace_user(&trace);
1924 if (!call_filter_check_discard(call, entry, buffer, event))
1925 __buffer_unlock_commit(buffer, event);
1928 __this_cpu_dec(user_stack_count);
1934 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1936 ftrace_trace_userstack(tr, flags, preempt_count());
1940 #endif /* CONFIG_STACKTRACE */
1942 /* created for use with alloc_percpu */
1943 struct trace_buffer_struct {
1944 char buffer[TRACE_BUF_SIZE];
1947 static struct trace_buffer_struct *trace_percpu_buffer;
1948 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1949 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1950 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1953 * The buffer used is dependent on the context. There is a per cpu
1954 * buffer for normal context, softirq contex, hard irq context and
1955 * for NMI context. Thise allows for lockless recording.
1957 * Note, if the buffers failed to be allocated, then this returns NULL
1959 static char *get_trace_buf(void)
1961 struct trace_buffer_struct *percpu_buffer;
1964 * If we have allocated per cpu buffers, then we do not
1965 * need to do any locking.
1968 percpu_buffer = trace_percpu_nmi_buffer;
1970 percpu_buffer = trace_percpu_irq_buffer;
1971 else if (in_softirq())
1972 percpu_buffer = trace_percpu_sirq_buffer;
1974 percpu_buffer = trace_percpu_buffer;
1979 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1982 static int alloc_percpu_trace_buffer(void)
1984 struct trace_buffer_struct *buffers;
1985 struct trace_buffer_struct *sirq_buffers;
1986 struct trace_buffer_struct *irq_buffers;
1987 struct trace_buffer_struct *nmi_buffers;
1989 buffers = alloc_percpu(struct trace_buffer_struct);
1993 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1997 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2001 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2005 trace_percpu_buffer = buffers;
2006 trace_percpu_sirq_buffer = sirq_buffers;
2007 trace_percpu_irq_buffer = irq_buffers;
2008 trace_percpu_nmi_buffer = nmi_buffers;
2013 free_percpu(irq_buffers);
2015 free_percpu(sirq_buffers);
2017 free_percpu(buffers);
2019 WARN(1, "Could not allocate percpu trace_printk buffer");
2023 static int buffers_allocated;
2025 void trace_printk_init_buffers(void)
2027 if (buffers_allocated)
2030 if (alloc_percpu_trace_buffer())
2033 /* trace_printk() is for debug use only. Don't use it in production. */
2035 pr_warning("\n**********************************************************\n");
2036 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2037 pr_warning("** **\n");
2038 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2039 pr_warning("** **\n");
2040 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2041 pr_warning("** unsafe for produciton use. **\n");
2042 pr_warning("** **\n");
2043 pr_warning("** If you see this message and you are not debugging **\n");
2044 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2045 pr_warning("** **\n");
2046 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2047 pr_warning("**********************************************************\n");
2049 /* Expand the buffers to set size */
2050 tracing_update_buffers();
2052 buffers_allocated = 1;
2055 * trace_printk_init_buffers() can be called by modules.
2056 * If that happens, then we need to start cmdline recording
2057 * directly here. If the global_trace.buffer is already
2058 * allocated here, then this was called by module code.
2060 if (global_trace.trace_buffer.buffer)
2061 tracing_start_cmdline_record();
2064 void trace_printk_start_comm(void)
2066 /* Start tracing comms if trace printk is set */
2067 if (!buffers_allocated)
2069 tracing_start_cmdline_record();
2072 static void trace_printk_start_stop_comm(int enabled)
2074 if (!buffers_allocated)
2078 tracing_start_cmdline_record();
2080 tracing_stop_cmdline_record();
2084 * trace_vbprintk - write binary msg to tracing buffer
2087 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2089 struct ftrace_event_call *call = &event_bprint;
2090 struct ring_buffer_event *event;
2091 struct ring_buffer *buffer;
2092 struct trace_array *tr = &global_trace;
2093 struct bprint_entry *entry;
2094 unsigned long flags;
2096 int len = 0, size, pc;
2098 if (unlikely(tracing_selftest_running || tracing_disabled))
2101 /* Don't pollute graph traces with trace_vprintk internals */
2102 pause_graph_tracing();
2104 pc = preempt_count();
2105 preempt_disable_notrace();
2107 tbuffer = get_trace_buf();
2113 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2115 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2118 local_save_flags(flags);
2119 size = sizeof(*entry) + sizeof(u32) * len;
2120 buffer = tr->trace_buffer.buffer;
2121 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2125 entry = ring_buffer_event_data(event);
2129 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2130 if (!call_filter_check_discard(call, entry, buffer, event)) {
2131 __buffer_unlock_commit(buffer, event);
2132 ftrace_trace_stack(buffer, flags, 6, pc);
2136 preempt_enable_notrace();
2137 unpause_graph_tracing();
2141 EXPORT_SYMBOL_GPL(trace_vbprintk);
2144 __trace_array_vprintk(struct ring_buffer *buffer,
2145 unsigned long ip, const char *fmt, va_list args)
2147 struct ftrace_event_call *call = &event_print;
2148 struct ring_buffer_event *event;
2149 int len = 0, size, pc;
2150 struct print_entry *entry;
2151 unsigned long flags;
2154 if (tracing_disabled || tracing_selftest_running)
2157 /* Don't pollute graph traces with trace_vprintk internals */
2158 pause_graph_tracing();
2160 pc = preempt_count();
2161 preempt_disable_notrace();
2164 tbuffer = get_trace_buf();
2170 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2171 if (len > TRACE_BUF_SIZE)
2174 local_save_flags(flags);
2175 size = sizeof(*entry) + len + 1;
2176 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2180 entry = ring_buffer_event_data(event);
2183 memcpy(&entry->buf, tbuffer, len);
2184 entry->buf[len] = '\0';
2185 if (!call_filter_check_discard(call, entry, buffer, event)) {
2186 __buffer_unlock_commit(buffer, event);
2187 ftrace_trace_stack(buffer, flags, 6, pc);
2190 preempt_enable_notrace();
2191 unpause_graph_tracing();
2196 int trace_array_vprintk(struct trace_array *tr,
2197 unsigned long ip, const char *fmt, va_list args)
2199 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2202 int trace_array_printk(struct trace_array *tr,
2203 unsigned long ip, const char *fmt, ...)
2208 if (!(trace_flags & TRACE_ITER_PRINTK))
2212 ret = trace_array_vprintk(tr, ip, fmt, ap);
2217 int trace_array_printk_buf(struct ring_buffer *buffer,
2218 unsigned long ip, const char *fmt, ...)
2223 if (!(trace_flags & TRACE_ITER_PRINTK))
2227 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2232 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2234 return trace_array_vprintk(&global_trace, ip, fmt, args);
2236 EXPORT_SYMBOL_GPL(trace_vprintk);
2238 static void trace_iterator_increment(struct trace_iterator *iter)
2240 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2244 ring_buffer_read(buf_iter, NULL);
2247 static struct trace_entry *
2248 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2249 unsigned long *lost_events)
2251 struct ring_buffer_event *event;
2252 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2255 event = ring_buffer_iter_peek(buf_iter, ts);
2257 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2261 iter->ent_size = ring_buffer_event_length(event);
2262 return ring_buffer_event_data(event);
2268 static struct trace_entry *
2269 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2270 unsigned long *missing_events, u64 *ent_ts)
2272 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2273 struct trace_entry *ent, *next = NULL;
2274 unsigned long lost_events = 0, next_lost = 0;
2275 int cpu_file = iter->cpu_file;
2276 u64 next_ts = 0, ts;
2282 * If we are in a per_cpu trace file, don't bother by iterating over
2283 * all cpu and peek directly.
2285 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2286 if (ring_buffer_empty_cpu(buffer, cpu_file))
2288 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2290 *ent_cpu = cpu_file;
2295 for_each_tracing_cpu(cpu) {
2297 if (ring_buffer_empty_cpu(buffer, cpu))
2300 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2303 * Pick the entry with the smallest timestamp:
2305 if (ent && (!next || ts < next_ts)) {
2309 next_lost = lost_events;
2310 next_size = iter->ent_size;
2314 iter->ent_size = next_size;
2317 *ent_cpu = next_cpu;
2323 *missing_events = next_lost;
2328 /* Find the next real entry, without updating the iterator itself */
2329 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2330 int *ent_cpu, u64 *ent_ts)
2332 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2335 /* Find the next real entry, and increment the iterator to the next entry */
2336 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2338 iter->ent = __find_next_entry(iter, &iter->cpu,
2339 &iter->lost_events, &iter->ts);
2342 trace_iterator_increment(iter);
2344 return iter->ent ? iter : NULL;
2347 static void trace_consume(struct trace_iterator *iter)
2349 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2350 &iter->lost_events);
2353 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2355 struct trace_iterator *iter = m->private;
2359 WARN_ON_ONCE(iter->leftover);
2363 /* can't go backwards */
2368 ent = trace_find_next_entry_inc(iter);
2372 while (ent && iter->idx < i)
2373 ent = trace_find_next_entry_inc(iter);
2380 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2382 struct ring_buffer_event *event;
2383 struct ring_buffer_iter *buf_iter;
2384 unsigned long entries = 0;
2387 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2389 buf_iter = trace_buffer_iter(iter, cpu);
2393 ring_buffer_iter_reset(buf_iter);
2396 * We could have the case with the max latency tracers
2397 * that a reset never took place on a cpu. This is evident
2398 * by the timestamp being before the start of the buffer.
2400 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2401 if (ts >= iter->trace_buffer->time_start)
2404 ring_buffer_read(buf_iter, NULL);
2407 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2411 * The current tracer is copied to avoid a global locking
2414 static void *s_start(struct seq_file *m, loff_t *pos)
2416 struct trace_iterator *iter = m->private;
2417 struct trace_array *tr = iter->tr;
2418 int cpu_file = iter->cpu_file;
2424 * copy the tracer to avoid using a global lock all around.
2425 * iter->trace is a copy of current_trace, the pointer to the
2426 * name may be used instead of a strcmp(), as iter->trace->name
2427 * will point to the same string as current_trace->name.
2429 mutex_lock(&trace_types_lock);
2430 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2431 *iter->trace = *tr->current_trace;
2432 mutex_unlock(&trace_types_lock);
2434 #ifdef CONFIG_TRACER_MAX_TRACE
2435 if (iter->snapshot && iter->trace->use_max_tr)
2436 return ERR_PTR(-EBUSY);
2439 if (!iter->snapshot)
2440 atomic_inc(&trace_record_cmdline_disabled);
2442 if (*pos != iter->pos) {
2447 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2448 for_each_tracing_cpu(cpu)
2449 tracing_iter_reset(iter, cpu);
2451 tracing_iter_reset(iter, cpu_file);
2454 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2459 * If we overflowed the seq_file before, then we want
2460 * to just reuse the trace_seq buffer again.
2466 p = s_next(m, p, &l);
2470 trace_event_read_lock();
2471 trace_access_lock(cpu_file);
2475 static void s_stop(struct seq_file *m, void *p)
2477 struct trace_iterator *iter = m->private;
2479 #ifdef CONFIG_TRACER_MAX_TRACE
2480 if (iter->snapshot && iter->trace->use_max_tr)
2484 if (!iter->snapshot)
2485 atomic_dec(&trace_record_cmdline_disabled);
2487 trace_access_unlock(iter->cpu_file);
2488 trace_event_read_unlock();
2492 get_total_entries(struct trace_buffer *buf,
2493 unsigned long *total, unsigned long *entries)
2495 unsigned long count;
2501 for_each_tracing_cpu(cpu) {
2502 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2504 * If this buffer has skipped entries, then we hold all
2505 * entries for the trace and we need to ignore the
2506 * ones before the time stamp.
2508 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2509 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2510 /* total is the same as the entries */
2514 ring_buffer_overrun_cpu(buf->buffer, cpu);
2519 static void print_lat_help_header(struct seq_file *m)
2521 seq_puts(m, "# _------=> CPU# \n");
2522 seq_puts(m, "# / _-----=> irqs-off \n");
2523 seq_puts(m, "# | / _----=> need-resched \n");
2524 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2525 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2526 seq_puts(m, "# |||| / delay \n");
2527 seq_puts(m, "# cmd pid ||||| time | caller \n");
2528 seq_puts(m, "# \\ / ||||| \\ | / \n");
2531 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2533 unsigned long total;
2534 unsigned long entries;
2536 get_total_entries(buf, &total, &entries);
2537 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2538 entries, total, num_online_cpus());
2542 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2544 print_event_info(buf, m);
2545 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2546 seq_puts(m, "# | | | | |\n");
2549 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2551 print_event_info(buf, m);
2552 seq_puts(m, "# _-----=> irqs-off\n");
2553 seq_puts(m, "# / _----=> need-resched\n");
2554 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2555 seq_puts(m, "# || / _--=> preempt-depth\n");
2556 seq_puts(m, "# ||| / delay\n");
2557 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2558 seq_puts(m, "# | | | |||| | |\n");
2562 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2564 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2565 struct trace_buffer *buf = iter->trace_buffer;
2566 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2567 struct tracer *type = iter->trace;
2568 unsigned long entries;
2569 unsigned long total;
2570 const char *name = "preemption";
2574 get_total_entries(buf, &total, &entries);
2576 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2578 seq_puts(m, "# -----------------------------------"
2579 "---------------------------------\n");
2580 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2581 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2582 nsecs_to_usecs(data->saved_latency),
2586 #if defined(CONFIG_PREEMPT_NONE)
2588 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2590 #elif defined(CONFIG_PREEMPT)
2595 /* These are reserved for later use */
2598 seq_printf(m, " #P:%d)\n", num_online_cpus());
2602 seq_puts(m, "# -----------------\n");
2603 seq_printf(m, "# | task: %.16s-%d "
2604 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2605 data->comm, data->pid,
2606 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2607 data->policy, data->rt_priority);
2608 seq_puts(m, "# -----------------\n");
2610 if (data->critical_start) {
2611 seq_puts(m, "# => started at: ");
2612 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2613 trace_print_seq(m, &iter->seq);
2614 seq_puts(m, "\n# => ended at: ");
2615 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2616 trace_print_seq(m, &iter->seq);
2617 seq_puts(m, "\n#\n");
2623 static void test_cpu_buff_start(struct trace_iterator *iter)
2625 struct trace_seq *s = &iter->seq;
2627 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2630 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2633 if (cpumask_test_cpu(iter->cpu, iter->started))
2636 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2639 cpumask_set_cpu(iter->cpu, iter->started);
2641 /* Don't print started cpu buffer for the first entry of the trace */
2643 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2647 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2649 struct trace_seq *s = &iter->seq;
2650 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2651 struct trace_entry *entry;
2652 struct trace_event *event;
2656 test_cpu_buff_start(iter);
2658 event = ftrace_find_event(entry->type);
2660 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2661 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2662 if (!trace_print_lat_context(iter))
2665 if (!trace_print_context(iter))
2671 return event->funcs->trace(iter, sym_flags, event);
2673 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2676 return TRACE_TYPE_HANDLED;
2678 return TRACE_TYPE_PARTIAL_LINE;
2681 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2683 struct trace_seq *s = &iter->seq;
2684 struct trace_entry *entry;
2685 struct trace_event *event;
2689 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2690 if (!trace_seq_printf(s, "%d %d %llu ",
2691 entry->pid, iter->cpu, iter->ts))
2695 event = ftrace_find_event(entry->type);
2697 return event->funcs->raw(iter, 0, event);
2699 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2702 return TRACE_TYPE_HANDLED;
2704 return TRACE_TYPE_PARTIAL_LINE;
2707 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2709 struct trace_seq *s = &iter->seq;
2710 unsigned char newline = '\n';
2711 struct trace_entry *entry;
2712 struct trace_event *event;
2716 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2717 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2718 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2719 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2722 event = ftrace_find_event(entry->type);
2724 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2725 if (ret != TRACE_TYPE_HANDLED)
2729 SEQ_PUT_FIELD_RET(s, newline);
2731 return TRACE_TYPE_HANDLED;
2734 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2736 struct trace_seq *s = &iter->seq;
2737 struct trace_entry *entry;
2738 struct trace_event *event;
2742 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2743 SEQ_PUT_FIELD_RET(s, entry->pid);
2744 SEQ_PUT_FIELD_RET(s, iter->cpu);
2745 SEQ_PUT_FIELD_RET(s, iter->ts);
2748 event = ftrace_find_event(entry->type);
2749 return event ? event->funcs->binary(iter, 0, event) :
2753 int trace_empty(struct trace_iterator *iter)
2755 struct ring_buffer_iter *buf_iter;
2758 /* If we are looking at one CPU buffer, only check that one */
2759 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2760 cpu = iter->cpu_file;
2761 buf_iter = trace_buffer_iter(iter, cpu);
2763 if (!ring_buffer_iter_empty(buf_iter))
2766 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2772 for_each_tracing_cpu(cpu) {
2773 buf_iter = trace_buffer_iter(iter, cpu);
2775 if (!ring_buffer_iter_empty(buf_iter))
2778 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2786 /* Called with trace_event_read_lock() held. */
2787 enum print_line_t print_trace_line(struct trace_iterator *iter)
2789 enum print_line_t ret;
2791 if (iter->lost_events &&
2792 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2793 iter->cpu, iter->lost_events))
2794 return TRACE_TYPE_PARTIAL_LINE;
2796 if (iter->trace && iter->trace->print_line) {
2797 ret = iter->trace->print_line(iter);
2798 if (ret != TRACE_TYPE_UNHANDLED)
2802 if (iter->ent->type == TRACE_BPUTS &&
2803 trace_flags & TRACE_ITER_PRINTK &&
2804 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2805 return trace_print_bputs_msg_only(iter);
2807 if (iter->ent->type == TRACE_BPRINT &&
2808 trace_flags & TRACE_ITER_PRINTK &&
2809 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2810 return trace_print_bprintk_msg_only(iter);
2812 if (iter->ent->type == TRACE_PRINT &&
2813 trace_flags & TRACE_ITER_PRINTK &&
2814 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2815 return trace_print_printk_msg_only(iter);
2817 if (trace_flags & TRACE_ITER_BIN)
2818 return print_bin_fmt(iter);
2820 if (trace_flags & TRACE_ITER_HEX)
2821 return print_hex_fmt(iter);
2823 if (trace_flags & TRACE_ITER_RAW)
2824 return print_raw_fmt(iter);
2826 return print_trace_fmt(iter);
2829 void trace_latency_header(struct seq_file *m)
2831 struct trace_iterator *iter = m->private;
2833 /* print nothing if the buffers are empty */
2834 if (trace_empty(iter))
2837 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2838 print_trace_header(m, iter);
2840 if (!(trace_flags & TRACE_ITER_VERBOSE))
2841 print_lat_help_header(m);
2844 void trace_default_header(struct seq_file *m)
2846 struct trace_iterator *iter = m->private;
2848 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2851 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2852 /* print nothing if the buffers are empty */
2853 if (trace_empty(iter))
2855 print_trace_header(m, iter);
2856 if (!(trace_flags & TRACE_ITER_VERBOSE))
2857 print_lat_help_header(m);
2859 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2860 if (trace_flags & TRACE_ITER_IRQ_INFO)
2861 print_func_help_header_irq(iter->trace_buffer, m);
2863 print_func_help_header(iter->trace_buffer, m);
2868 static void test_ftrace_alive(struct seq_file *m)
2870 if (!ftrace_is_dead())
2872 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2873 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2876 #ifdef CONFIG_TRACER_MAX_TRACE
2877 static void show_snapshot_main_help(struct seq_file *m)
2879 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2880 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2881 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2882 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2883 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2884 seq_printf(m, "# is not a '0' or '1')\n");
2887 static void show_snapshot_percpu_help(struct seq_file *m)
2889 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2890 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2891 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2892 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2894 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2895 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2897 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2898 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2899 seq_printf(m, "# is not a '0' or '1')\n");
2902 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2904 if (iter->tr->allocated_snapshot)
2905 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2907 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2909 seq_printf(m, "# Snapshot commands:\n");
2910 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2911 show_snapshot_main_help(m);
2913 show_snapshot_percpu_help(m);
2916 /* Should never be called */
2917 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2920 static int s_show(struct seq_file *m, void *v)
2922 struct trace_iterator *iter = v;
2925 if (iter->ent == NULL) {
2927 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2929 test_ftrace_alive(m);
2931 if (iter->snapshot && trace_empty(iter))
2932 print_snapshot_help(m, iter);
2933 else if (iter->trace && iter->trace->print_header)
2934 iter->trace->print_header(m);
2936 trace_default_header(m);
2938 } else if (iter->leftover) {
2940 * If we filled the seq_file buffer earlier, we
2941 * want to just show it now.
2943 ret = trace_print_seq(m, &iter->seq);
2945 /* ret should this time be zero, but you never know */
2946 iter->leftover = ret;
2949 print_trace_line(iter);
2950 ret = trace_print_seq(m, &iter->seq);
2952 * If we overflow the seq_file buffer, then it will
2953 * ask us for this data again at start up.
2955 * ret is 0 if seq_file write succeeded.
2958 iter->leftover = ret;
2965 * Should be used after trace_array_get(), trace_types_lock
2966 * ensures that i_cdev was already initialized.
2968 static inline int tracing_get_cpu(struct inode *inode)
2970 if (inode->i_cdev) /* See trace_create_cpu_file() */
2971 return (long)inode->i_cdev - 1;
2972 return RING_BUFFER_ALL_CPUS;
2975 static const struct seq_operations tracer_seq_ops = {
2982 static struct trace_iterator *
2983 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2985 struct trace_array *tr = inode->i_private;
2986 struct trace_iterator *iter;
2989 if (tracing_disabled)
2990 return ERR_PTR(-ENODEV);
2992 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2994 return ERR_PTR(-ENOMEM);
2996 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2998 if (!iter->buffer_iter)
3002 * We make a copy of the current tracer to avoid concurrent
3003 * changes on it while we are reading.
3005 mutex_lock(&trace_types_lock);
3006 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3010 *iter->trace = *tr->current_trace;
3012 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3017 #ifdef CONFIG_TRACER_MAX_TRACE
3018 /* Currently only the top directory has a snapshot */
3019 if (tr->current_trace->print_max || snapshot)
3020 iter->trace_buffer = &tr->max_buffer;
3023 iter->trace_buffer = &tr->trace_buffer;
3024 iter->snapshot = snapshot;
3026 iter->cpu_file = tracing_get_cpu(inode);
3027 mutex_init(&iter->mutex);
3029 /* Notify the tracer early; before we stop tracing. */
3030 if (iter->trace && iter->trace->open)
3031 iter->trace->open(iter);
3033 /* Annotate start of buffers if we had overruns */
3034 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3035 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3037 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3038 if (trace_clocks[tr->clock_id].in_ns)
3039 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3041 /* stop the trace while dumping if we are not opening "snapshot" */
3042 if (!iter->snapshot)
3043 tracing_stop_tr(tr);
3045 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3046 for_each_tracing_cpu(cpu) {
3047 iter->buffer_iter[cpu] =
3048 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3050 ring_buffer_read_prepare_sync();
3051 for_each_tracing_cpu(cpu) {
3052 ring_buffer_read_start(iter->buffer_iter[cpu]);
3053 tracing_iter_reset(iter, cpu);
3056 cpu = iter->cpu_file;
3057 iter->buffer_iter[cpu] =
3058 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3059 ring_buffer_read_prepare_sync();
3060 ring_buffer_read_start(iter->buffer_iter[cpu]);
3061 tracing_iter_reset(iter, cpu);
3064 mutex_unlock(&trace_types_lock);
3069 mutex_unlock(&trace_types_lock);
3071 kfree(iter->buffer_iter);
3073 seq_release_private(inode, file);
3074 return ERR_PTR(-ENOMEM);
3077 int tracing_open_generic(struct inode *inode, struct file *filp)
3079 if (tracing_disabled)
3082 filp->private_data = inode->i_private;
3086 bool tracing_is_disabled(void)
3088 return (tracing_disabled) ? true: false;
3092 * Open and update trace_array ref count.
3093 * Must have the current trace_array passed to it.
3095 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3097 struct trace_array *tr = inode->i_private;
3099 if (tracing_disabled)
3102 if (trace_array_get(tr) < 0)
3105 filp->private_data = inode->i_private;
3110 static int tracing_release(struct inode *inode, struct file *file)
3112 struct trace_array *tr = inode->i_private;
3113 struct seq_file *m = file->private_data;
3114 struct trace_iterator *iter;
3117 if (!(file->f_mode & FMODE_READ)) {
3118 trace_array_put(tr);
3122 /* Writes do not use seq_file */
3124 mutex_lock(&trace_types_lock);
3126 for_each_tracing_cpu(cpu) {
3127 if (iter->buffer_iter[cpu])
3128 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3131 if (iter->trace && iter->trace->close)
3132 iter->trace->close(iter);
3134 if (!iter->snapshot)
3135 /* reenable tracing if it was previously enabled */
3136 tracing_start_tr(tr);
3138 __trace_array_put(tr);
3140 mutex_unlock(&trace_types_lock);
3142 mutex_destroy(&iter->mutex);
3143 free_cpumask_var(iter->started);
3145 kfree(iter->buffer_iter);
3146 seq_release_private(inode, file);
3151 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3153 struct trace_array *tr = inode->i_private;
3155 trace_array_put(tr);
3159 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3161 struct trace_array *tr = inode->i_private;
3163 trace_array_put(tr);
3165 return single_release(inode, file);
3168 static int tracing_open(struct inode *inode, struct file *file)
3170 struct trace_array *tr = inode->i_private;
3171 struct trace_iterator *iter;
3174 if (trace_array_get(tr) < 0)
3177 /* If this file was open for write, then erase contents */
3178 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3179 int cpu = tracing_get_cpu(inode);
3181 if (cpu == RING_BUFFER_ALL_CPUS)
3182 tracing_reset_online_cpus(&tr->trace_buffer);
3184 tracing_reset(&tr->trace_buffer, cpu);
3187 if (file->f_mode & FMODE_READ) {
3188 iter = __tracing_open(inode, file, false);
3190 ret = PTR_ERR(iter);
3191 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3192 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3196 trace_array_put(tr);
3202 * Some tracers are not suitable for instance buffers.
3203 * A tracer is always available for the global array (toplevel)
3204 * or if it explicitly states that it is.
3207 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3209 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3212 /* Find the next tracer that this trace array may use */
3213 static struct tracer *
3214 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3216 while (t && !trace_ok_for_array(t, tr))
3223 t_next(struct seq_file *m, void *v, loff_t *pos)
3225 struct trace_array *tr = m->private;
3226 struct tracer *t = v;
3231 t = get_tracer_for_array(tr, t->next);
3236 static void *t_start(struct seq_file *m, loff_t *pos)
3238 struct trace_array *tr = m->private;
3242 mutex_lock(&trace_types_lock);
3244 t = get_tracer_for_array(tr, trace_types);
3245 for (; t && l < *pos; t = t_next(m, t, &l))
3251 static void t_stop(struct seq_file *m, void *p)
3253 mutex_unlock(&trace_types_lock);
3256 static int t_show(struct seq_file *m, void *v)
3258 struct tracer *t = v;
3263 seq_printf(m, "%s", t->name);
3272 static const struct seq_operations show_traces_seq_ops = {
3279 static int show_traces_open(struct inode *inode, struct file *file)
3281 struct trace_array *tr = inode->i_private;
3285 if (tracing_disabled)
3288 ret = seq_open(file, &show_traces_seq_ops);
3292 m = file->private_data;
3299 tracing_write_stub(struct file *filp, const char __user *ubuf,
3300 size_t count, loff_t *ppos)
3305 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3309 if (file->f_mode & FMODE_READ)
3310 ret = seq_lseek(file, offset, whence);
3312 file->f_pos = ret = 0;
3317 static const struct file_operations tracing_fops = {
3318 .open = tracing_open,
3320 .write = tracing_write_stub,
3321 .llseek = tracing_lseek,
3322 .release = tracing_release,
3325 static const struct file_operations show_traces_fops = {
3326 .open = show_traces_open,
3328 .release = seq_release,
3329 .llseek = seq_lseek,
3333 * The tracer itself will not take this lock, but still we want
3334 * to provide a consistent cpumask to user-space:
3336 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3339 * Temporary storage for the character representation of the
3340 * CPU bitmask (and one more byte for the newline):
3342 static char mask_str[NR_CPUS + 1];
3345 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3346 size_t count, loff_t *ppos)
3348 struct trace_array *tr = file_inode(filp)->i_private;
3351 mutex_lock(&tracing_cpumask_update_lock);
3353 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3354 if (count - len < 2) {
3358 len += sprintf(mask_str + len, "\n");
3359 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3362 mutex_unlock(&tracing_cpumask_update_lock);
3368 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3369 size_t count, loff_t *ppos)
3371 struct trace_array *tr = file_inode(filp)->i_private;
3372 cpumask_var_t tracing_cpumask_new;
3375 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3378 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3382 mutex_lock(&tracing_cpumask_update_lock);
3384 local_irq_disable();
3385 arch_spin_lock(&tr->max_lock);
3386 for_each_tracing_cpu(cpu) {
3388 * Increase/decrease the disabled counter if we are
3389 * about to flip a bit in the cpumask:
3391 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3392 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3393 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3394 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3396 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3397 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3398 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3399 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3402 arch_spin_unlock(&tr->max_lock);
3405 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3407 mutex_unlock(&tracing_cpumask_update_lock);
3408 free_cpumask_var(tracing_cpumask_new);
3413 free_cpumask_var(tracing_cpumask_new);
3418 static const struct file_operations tracing_cpumask_fops = {
3419 .open = tracing_open_generic_tr,
3420 .read = tracing_cpumask_read,
3421 .write = tracing_cpumask_write,
3422 .release = tracing_release_generic_tr,
3423 .llseek = generic_file_llseek,
3426 static int tracing_trace_options_show(struct seq_file *m, void *v)
3428 struct tracer_opt *trace_opts;
3429 struct trace_array *tr = m->private;
3433 mutex_lock(&trace_types_lock);
3434 tracer_flags = tr->current_trace->flags->val;
3435 trace_opts = tr->current_trace->flags->opts;
3437 for (i = 0; trace_options[i]; i++) {
3438 if (trace_flags & (1 << i))
3439 seq_printf(m, "%s\n", trace_options[i]);
3441 seq_printf(m, "no%s\n", trace_options[i]);
3444 for (i = 0; trace_opts[i].name; i++) {
3445 if (tracer_flags & trace_opts[i].bit)
3446 seq_printf(m, "%s\n", trace_opts[i].name);
3448 seq_printf(m, "no%s\n", trace_opts[i].name);
3450 mutex_unlock(&trace_types_lock);
3455 static int __set_tracer_option(struct trace_array *tr,
3456 struct tracer_flags *tracer_flags,
3457 struct tracer_opt *opts, int neg)
3459 struct tracer *trace = tr->current_trace;
3462 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3467 tracer_flags->val &= ~opts->bit;
3469 tracer_flags->val |= opts->bit;
3473 /* Try to assign a tracer specific option */
3474 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3476 struct tracer *trace = tr->current_trace;
3477 struct tracer_flags *tracer_flags = trace->flags;
3478 struct tracer_opt *opts = NULL;
3481 for (i = 0; tracer_flags->opts[i].name; i++) {
3482 opts = &tracer_flags->opts[i];
3484 if (strcmp(cmp, opts->name) == 0)
3485 return __set_tracer_option(tr, trace->flags, opts, neg);
3491 /* Some tracers require overwrite to stay enabled */
3492 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3494 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3500 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3502 /* do nothing if flag is already set */
3503 if (!!(trace_flags & mask) == !!enabled)
3506 /* Give the tracer a chance to approve the change */
3507 if (tr->current_trace->flag_changed)
3508 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3512 trace_flags |= mask;
3514 trace_flags &= ~mask;
3516 if (mask == TRACE_ITER_RECORD_CMD)
3517 trace_event_enable_cmd_record(enabled);
3519 if (mask == TRACE_ITER_OVERWRITE) {
3520 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3521 #ifdef CONFIG_TRACER_MAX_TRACE
3522 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3526 if (mask == TRACE_ITER_PRINTK)
3527 trace_printk_start_stop_comm(enabled);
3532 static int trace_set_options(struct trace_array *tr, char *option)
3539 cmp = strstrip(option);
3541 if (strncmp(cmp, "no", 2) == 0) {
3546 mutex_lock(&trace_types_lock);
3548 for (i = 0; trace_options[i]; i++) {
3549 if (strcmp(cmp, trace_options[i]) == 0) {
3550 ret = set_tracer_flag(tr, 1 << i, !neg);
3555 /* If no option could be set, test the specific tracer options */
3556 if (!trace_options[i])
3557 ret = set_tracer_option(tr, cmp, neg);
3559 mutex_unlock(&trace_types_lock);
3565 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3566 size_t cnt, loff_t *ppos)
3568 struct seq_file *m = filp->private_data;
3569 struct trace_array *tr = m->private;
3573 if (cnt >= sizeof(buf))
3576 if (copy_from_user(&buf, ubuf, cnt))
3581 ret = trace_set_options(tr, buf);
3590 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3592 struct trace_array *tr = inode->i_private;
3595 if (tracing_disabled)
3598 if (trace_array_get(tr) < 0)
3601 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3603 trace_array_put(tr);
3608 static const struct file_operations tracing_iter_fops = {
3609 .open = tracing_trace_options_open,
3611 .llseek = seq_lseek,
3612 .release = tracing_single_release_tr,
3613 .write = tracing_trace_options_write,
3616 static const char readme_msg[] =
3617 "tracing mini-HOWTO:\n\n"
3618 "# echo 0 > tracing_on : quick way to disable tracing\n"
3619 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3620 " Important files:\n"
3621 " trace\t\t\t- The static contents of the buffer\n"
3622 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3623 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3624 " current_tracer\t- function and latency tracers\n"
3625 " available_tracers\t- list of configured tracers for current_tracer\n"
3626 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3627 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3628 " trace_clock\t\t-change the clock used to order events\n"
3629 " local: Per cpu clock but may not be synced across CPUs\n"
3630 " global: Synced across CPUs but slows tracing down.\n"
3631 " counter: Not a clock, but just an increment\n"
3632 " uptime: Jiffy counter from time of boot\n"
3633 " perf: Same clock that perf events use\n"
3634 #ifdef CONFIG_X86_64
3635 " x86-tsc: TSC cycle counter\n"
3637 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3638 " tracing_cpumask\t- Limit which CPUs to trace\n"
3639 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3640 "\t\t\t Remove sub-buffer with rmdir\n"
3641 " trace_options\t\t- Set format or modify how tracing happens\n"
3642 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3643 "\t\t\t option name\n"
3644 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3645 #ifdef CONFIG_DYNAMIC_FTRACE
3646 "\n available_filter_functions - list of functions that can be filtered on\n"
3647 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3648 "\t\t\t functions\n"
3649 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3650 "\t modules: Can select a group via module\n"
3651 "\t Format: :mod:<module-name>\n"
3652 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3653 "\t triggers: a command to perform when function is hit\n"
3654 "\t Format: <function>:<trigger>[:count]\n"
3655 "\t trigger: traceon, traceoff\n"
3656 "\t\t enable_event:<system>:<event>\n"
3657 "\t\t disable_event:<system>:<event>\n"
3658 #ifdef CONFIG_STACKTRACE
3661 #ifdef CONFIG_TRACER_SNAPSHOT
3666 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3667 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3668 "\t The first one will disable tracing every time do_fault is hit\n"
3669 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3670 "\t The first time do trap is hit and it disables tracing, the\n"
3671 "\t counter will decrement to 2. If tracing is already disabled,\n"
3672 "\t the counter will not decrement. It only decrements when the\n"
3673 "\t trigger did work\n"
3674 "\t To remove trigger without count:\n"
3675 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3676 "\t To remove trigger with a count:\n"
3677 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3678 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3679 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3680 "\t modules: Can select a group via module command :mod:\n"
3681 "\t Does not accept triggers\n"
3682 #endif /* CONFIG_DYNAMIC_FTRACE */
3683 #ifdef CONFIG_FUNCTION_TRACER
3684 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3687 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3688 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3689 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3691 #ifdef CONFIG_TRACER_SNAPSHOT
3692 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3693 "\t\t\t snapshot buffer. Read the contents for more\n"
3694 "\t\t\t information\n"
3696 #ifdef CONFIG_STACK_TRACER
3697 " stack_trace\t\t- Shows the max stack trace when active\n"
3698 " stack_max_size\t- Shows current max stack size that was traced\n"
3699 "\t\t\t Write into this file to reset the max size (trigger a\n"
3700 "\t\t\t new trace)\n"
3701 #ifdef CONFIG_DYNAMIC_FTRACE
3702 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3705 #endif /* CONFIG_STACK_TRACER */
3706 " events/\t\t- Directory containing all trace event subsystems:\n"
3707 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3708 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3709 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3711 " filter\t\t- If set, only events passing filter are traced\n"
3712 " events/<system>/<event>/\t- Directory containing control files for\n"
3714 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3715 " filter\t\t- If set, only events passing filter are traced\n"
3716 " trigger\t\t- If set, a command to perform when event is hit\n"
3717 "\t Format: <trigger>[:count][if <filter>]\n"
3718 "\t trigger: traceon, traceoff\n"
3719 "\t enable_event:<system>:<event>\n"
3720 "\t disable_event:<system>:<event>\n"
3721 #ifdef CONFIG_STACKTRACE
3724 #ifdef CONFIG_TRACER_SNAPSHOT
3727 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3728 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3729 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3730 "\t events/block/block_unplug/trigger\n"
3731 "\t The first disables tracing every time block_unplug is hit.\n"
3732 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3733 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3734 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3735 "\t Like function triggers, the counter is only decremented if it\n"
3736 "\t enabled or disabled tracing.\n"
3737 "\t To remove a trigger without a count:\n"
3738 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3739 "\t To remove a trigger with a count:\n"
3740 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3741 "\t Filters can be ignored when removing a trigger.\n"
3745 tracing_readme_read(struct file *filp, char __user *ubuf,
3746 size_t cnt, loff_t *ppos)
3748 return simple_read_from_buffer(ubuf, cnt, ppos,
3749 readme_msg, strlen(readme_msg));
3752 static const struct file_operations tracing_readme_fops = {
3753 .open = tracing_open_generic,
3754 .read = tracing_readme_read,
3755 .llseek = generic_file_llseek,
3758 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3760 unsigned int *ptr = v;
3762 if (*pos || m->count)
3767 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3769 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3778 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3784 arch_spin_lock(&trace_cmdline_lock);
3786 v = &savedcmd->map_cmdline_to_pid[0];
3788 v = saved_cmdlines_next(m, v, &l);
3796 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3798 arch_spin_unlock(&trace_cmdline_lock);
3802 static int saved_cmdlines_show(struct seq_file *m, void *v)
3804 char buf[TASK_COMM_LEN];
3805 unsigned int *pid = v;
3807 __trace_find_cmdline(*pid, buf);
3808 seq_printf(m, "%d %s\n", *pid, buf);
3812 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3813 .start = saved_cmdlines_start,
3814 .next = saved_cmdlines_next,
3815 .stop = saved_cmdlines_stop,
3816 .show = saved_cmdlines_show,
3819 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3821 if (tracing_disabled)
3824 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3827 static const struct file_operations tracing_saved_cmdlines_fops = {
3828 .open = tracing_saved_cmdlines_open,
3830 .llseek = seq_lseek,
3831 .release = seq_release,
3835 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3836 size_t cnt, loff_t *ppos)
3841 arch_spin_lock(&trace_cmdline_lock);
3842 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3843 arch_spin_unlock(&trace_cmdline_lock);
3845 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3848 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3850 kfree(s->saved_cmdlines);
3851 kfree(s->map_cmdline_to_pid);
3855 static int tracing_resize_saved_cmdlines(unsigned int val)
3857 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3859 s = kmalloc(sizeof(*s), GFP_KERNEL);
3863 if (allocate_cmdlines_buffer(val, s) < 0) {
3868 arch_spin_lock(&trace_cmdline_lock);
3869 savedcmd_temp = savedcmd;
3871 arch_spin_unlock(&trace_cmdline_lock);
3872 free_saved_cmdlines_buffer(savedcmd_temp);
3878 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3879 size_t cnt, loff_t *ppos)
3884 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3888 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3889 if (!val || val > PID_MAX_DEFAULT)
3892 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3901 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3902 .open = tracing_open_generic,
3903 .read = tracing_saved_cmdlines_size_read,
3904 .write = tracing_saved_cmdlines_size_write,
3908 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3909 size_t cnt, loff_t *ppos)
3911 struct trace_array *tr = filp->private_data;
3912 char buf[MAX_TRACER_SIZE+2];
3915 mutex_lock(&trace_types_lock);
3916 r = sprintf(buf, "%s\n", tr->current_trace->name);
3917 mutex_unlock(&trace_types_lock);
3919 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3922 int tracer_init(struct tracer *t, struct trace_array *tr)
3924 tracing_reset_online_cpus(&tr->trace_buffer);
3928 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3932 for_each_tracing_cpu(cpu)
3933 per_cpu_ptr(buf->data, cpu)->entries = val;
3936 #ifdef CONFIG_TRACER_MAX_TRACE
3937 /* resize @tr's buffer to the size of @size_tr's entries */
3938 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3939 struct trace_buffer *size_buf, int cpu_id)
3943 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3944 for_each_tracing_cpu(cpu) {
3945 ret = ring_buffer_resize(trace_buf->buffer,
3946 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3949 per_cpu_ptr(trace_buf->data, cpu)->entries =
3950 per_cpu_ptr(size_buf->data, cpu)->entries;
3953 ret = ring_buffer_resize(trace_buf->buffer,
3954 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3956 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3957 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3962 #endif /* CONFIG_TRACER_MAX_TRACE */
3964 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3965 unsigned long size, int cpu)
3970 * If kernel or user changes the size of the ring buffer
3971 * we use the size that was given, and we can forget about
3972 * expanding it later.
3974 ring_buffer_expanded = true;
3976 /* May be called before buffers are initialized */
3977 if (!tr->trace_buffer.buffer)
3980 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3984 #ifdef CONFIG_TRACER_MAX_TRACE
3985 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3986 !tr->current_trace->use_max_tr)
3989 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3991 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3992 &tr->trace_buffer, cpu);
3995 * AARGH! We are left with different
3996 * size max buffer!!!!
3997 * The max buffer is our "snapshot" buffer.
3998 * When a tracer needs a snapshot (one of the
3999 * latency tracers), it swaps the max buffer
4000 * with the saved snap shot. We succeeded to
4001 * update the size of the main buffer, but failed to
4002 * update the size of the max buffer. But when we tried
4003 * to reset the main buffer to the original size, we
4004 * failed there too. This is very unlikely to
4005 * happen, but if it does, warn and kill all
4009 tracing_disabled = 1;
4014 if (cpu == RING_BUFFER_ALL_CPUS)
4015 set_buffer_entries(&tr->max_buffer, size);
4017 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4020 #endif /* CONFIG_TRACER_MAX_TRACE */
4022 if (cpu == RING_BUFFER_ALL_CPUS)
4023 set_buffer_entries(&tr->trace_buffer, size);
4025 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4030 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4031 unsigned long size, int cpu_id)
4035 mutex_lock(&trace_types_lock);
4037 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4038 /* make sure, this cpu is enabled in the mask */
4039 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4045 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4050 mutex_unlock(&trace_types_lock);
4057 * tracing_update_buffers - used by tracing facility to expand ring buffers
4059 * To save on memory when the tracing is never used on a system with it
4060 * configured in. The ring buffers are set to a minimum size. But once
4061 * a user starts to use the tracing facility, then they need to grow
4062 * to their default size.
4064 * This function is to be called when a tracer is about to be used.
4066 int tracing_update_buffers(void)
4070 mutex_lock(&trace_types_lock);
4071 if (!ring_buffer_expanded)
4072 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4073 RING_BUFFER_ALL_CPUS);
4074 mutex_unlock(&trace_types_lock);
4079 struct trace_option_dentry;
4081 static struct trace_option_dentry *
4082 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4085 destroy_trace_option_files(struct trace_option_dentry *topts);
4088 * Used to clear out the tracer before deletion of an instance.
4089 * Must have trace_types_lock held.
4091 static void tracing_set_nop(struct trace_array *tr)
4093 if (tr->current_trace == &nop_trace)
4096 tr->current_trace->enabled--;
4098 if (tr->current_trace->reset)
4099 tr->current_trace->reset(tr);
4101 tr->current_trace = &nop_trace;
4104 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4106 static struct trace_option_dentry *topts;
4108 #ifdef CONFIG_TRACER_MAX_TRACE
4113 mutex_lock(&trace_types_lock);
4115 if (!ring_buffer_expanded) {
4116 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4117 RING_BUFFER_ALL_CPUS);
4123 for (t = trace_types; t; t = t->next) {
4124 if (strcmp(t->name, buf) == 0)
4131 if (t == tr->current_trace)
4134 /* Some tracers are only allowed for the top level buffer */
4135 if (!trace_ok_for_array(t, tr)) {
4140 trace_branch_disable();
4142 tr->current_trace->enabled--;
4144 if (tr->current_trace->reset)
4145 tr->current_trace->reset(tr);
4147 /* Current trace needs to be nop_trace before synchronize_sched */
4148 tr->current_trace = &nop_trace;
4150 #ifdef CONFIG_TRACER_MAX_TRACE
4151 had_max_tr = tr->allocated_snapshot;
4153 if (had_max_tr && !t->use_max_tr) {
4155 * We need to make sure that the update_max_tr sees that
4156 * current_trace changed to nop_trace to keep it from
4157 * swapping the buffers after we resize it.
4158 * The update_max_tr is called from interrupts disabled
4159 * so a synchronized_sched() is sufficient.
4161 synchronize_sched();
4165 /* Currently, only the top instance has options */
4166 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4167 destroy_trace_option_files(topts);
4168 topts = create_trace_option_files(tr, t);
4171 #ifdef CONFIG_TRACER_MAX_TRACE
4172 if (t->use_max_tr && !had_max_tr) {
4173 ret = alloc_snapshot(tr);
4180 ret = tracer_init(t, tr);
4185 tr->current_trace = t;
4186 tr->current_trace->enabled++;
4187 trace_branch_enable(tr);
4189 mutex_unlock(&trace_types_lock);
4195 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4196 size_t cnt, loff_t *ppos)
4198 struct trace_array *tr = filp->private_data;
4199 char buf[MAX_TRACER_SIZE+1];
4206 if (cnt > MAX_TRACER_SIZE)
4207 cnt = MAX_TRACER_SIZE;
4209 if (copy_from_user(&buf, ubuf, cnt))
4214 /* strip ending whitespace. */
4215 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4218 err = tracing_set_tracer(tr, buf);
4228 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4229 size_t cnt, loff_t *ppos)
4231 unsigned long *ptr = filp->private_data;
4235 r = snprintf(buf, sizeof(buf), "%ld\n",
4236 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4237 if (r > sizeof(buf))
4239 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4243 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4244 size_t cnt, loff_t *ppos)
4246 unsigned long *ptr = filp->private_data;
4250 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4259 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4261 struct trace_array *tr = inode->i_private;
4262 struct trace_iterator *iter;
4265 if (tracing_disabled)
4268 if (trace_array_get(tr) < 0)
4271 mutex_lock(&trace_types_lock);
4273 /* create a buffer to store the information to pass to userspace */
4274 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4277 __trace_array_put(tr);
4282 * We make a copy of the current tracer to avoid concurrent
4283 * changes on it while we are reading.
4285 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4290 *iter->trace = *tr->current_trace;
4292 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4297 /* trace pipe does not show start of buffer */
4298 cpumask_setall(iter->started);
4300 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4301 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4303 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4304 if (trace_clocks[tr->clock_id].in_ns)
4305 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4308 iter->trace_buffer = &tr->trace_buffer;
4309 iter->cpu_file = tracing_get_cpu(inode);
4310 mutex_init(&iter->mutex);
4311 filp->private_data = iter;
4313 if (iter->trace->pipe_open)
4314 iter->trace->pipe_open(iter);
4316 nonseekable_open(inode, filp);
4318 mutex_unlock(&trace_types_lock);
4324 __trace_array_put(tr);
4325 mutex_unlock(&trace_types_lock);
4329 static int tracing_release_pipe(struct inode *inode, struct file *file)
4331 struct trace_iterator *iter = file->private_data;
4332 struct trace_array *tr = inode->i_private;
4334 mutex_lock(&trace_types_lock);
4336 if (iter->trace->pipe_close)
4337 iter->trace->pipe_close(iter);
4339 mutex_unlock(&trace_types_lock);
4341 free_cpumask_var(iter->started);
4342 mutex_destroy(&iter->mutex);
4346 trace_array_put(tr);
4352 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4354 /* Iterators are static, they should be filled or empty */
4355 if (trace_buffer_iter(iter, iter->cpu_file))
4356 return POLLIN | POLLRDNORM;
4358 if (trace_flags & TRACE_ITER_BLOCK)
4360 * Always select as readable when in blocking mode
4362 return POLLIN | POLLRDNORM;
4364 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4369 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4371 struct trace_iterator *iter = filp->private_data;
4373 return trace_poll(iter, filp, poll_table);
4376 /* Must be called with trace_types_lock mutex held. */
4377 static int tracing_wait_pipe(struct file *filp)
4379 struct trace_iterator *iter = filp->private_data;
4382 while (trace_empty(iter)) {
4384 if ((filp->f_flags & O_NONBLOCK)) {
4389 * We block until we read something and tracing is disabled.
4390 * We still block if tracing is disabled, but we have never
4391 * read anything. This allows a user to cat this file, and
4392 * then enable tracing. But after we have read something,
4393 * we give an EOF when tracing is again disabled.
4395 * iter->pos will be 0 if we haven't read anything.
4397 if (!tracing_is_on() && iter->pos)
4400 mutex_unlock(&iter->mutex);
4402 ret = wait_on_pipe(iter);
4404 mutex_lock(&iter->mutex);
4409 if (signal_pending(current))
4420 tracing_read_pipe(struct file *filp, char __user *ubuf,
4421 size_t cnt, loff_t *ppos)
4423 struct trace_iterator *iter = filp->private_data;
4424 struct trace_array *tr = iter->tr;
4427 /* return any leftover data */
4428 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4432 trace_seq_init(&iter->seq);
4434 /* copy the tracer to avoid using a global lock all around */
4435 mutex_lock(&trace_types_lock);
4436 if (unlikely(iter->trace->name != tr->current_trace->name))
4437 *iter->trace = *tr->current_trace;
4438 mutex_unlock(&trace_types_lock);
4441 * Avoid more than one consumer on a single file descriptor
4442 * This is just a matter of traces coherency, the ring buffer itself
4445 mutex_lock(&iter->mutex);
4446 if (iter->trace->read) {
4447 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4453 sret = tracing_wait_pipe(filp);
4457 /* stop when tracing is finished */
4458 if (trace_empty(iter)) {
4463 if (cnt >= PAGE_SIZE)
4464 cnt = PAGE_SIZE - 1;
4466 /* reset all but tr, trace, and overruns */
4467 memset(&iter->seq, 0,
4468 sizeof(struct trace_iterator) -
4469 offsetof(struct trace_iterator, seq));
4470 cpumask_clear(iter->started);
4473 trace_event_read_lock();
4474 trace_access_lock(iter->cpu_file);
4475 while (trace_find_next_entry_inc(iter) != NULL) {
4476 enum print_line_t ret;
4477 int len = iter->seq.len;
4479 ret = print_trace_line(iter);
4480 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4481 /* don't print partial lines */
4482 iter->seq.len = len;
4485 if (ret != TRACE_TYPE_NO_CONSUME)
4486 trace_consume(iter);
4488 if (iter->seq.len >= cnt)
4492 * Setting the full flag means we reached the trace_seq buffer
4493 * size and we should leave by partial output condition above.
4494 * One of the trace_seq_* functions is not used properly.
4496 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4499 trace_access_unlock(iter->cpu_file);
4500 trace_event_read_unlock();
4502 /* Now copy what we have to the user */
4503 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4504 if (iter->seq.readpos >= iter->seq.len)
4505 trace_seq_init(&iter->seq);
4508 * If there was nothing to send to user, in spite of consuming trace
4509 * entries, go back to wait for more entries.
4515 mutex_unlock(&iter->mutex);
4520 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4523 __free_page(spd->pages[idx]);
4526 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4528 .confirm = generic_pipe_buf_confirm,
4529 .release = generic_pipe_buf_release,
4530 .steal = generic_pipe_buf_steal,
4531 .get = generic_pipe_buf_get,
4535 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4540 /* Seq buffer is page-sized, exactly what we need. */
4542 count = iter->seq.len;
4543 ret = print_trace_line(iter);
4544 count = iter->seq.len - count;
4547 iter->seq.len -= count;
4550 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4551 iter->seq.len -= count;
4555 if (ret != TRACE_TYPE_NO_CONSUME)
4556 trace_consume(iter);
4558 if (!trace_find_next_entry_inc(iter)) {
4568 static ssize_t tracing_splice_read_pipe(struct file *filp,
4570 struct pipe_inode_info *pipe,
4574 struct page *pages_def[PIPE_DEF_BUFFERS];
4575 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4576 struct trace_iterator *iter = filp->private_data;
4577 struct splice_pipe_desc spd = {
4579 .partial = partial_def,
4580 .nr_pages = 0, /* This gets updated below. */
4581 .nr_pages_max = PIPE_DEF_BUFFERS,
4583 .ops = &tracing_pipe_buf_ops,
4584 .spd_release = tracing_spd_release_pipe,
4586 struct trace_array *tr = iter->tr;
4591 if (splice_grow_spd(pipe, &spd))
4594 /* copy the tracer to avoid using a global lock all around */
4595 mutex_lock(&trace_types_lock);
4596 if (unlikely(iter->trace->name != tr->current_trace->name))
4597 *iter->trace = *tr->current_trace;
4598 mutex_unlock(&trace_types_lock);
4600 mutex_lock(&iter->mutex);
4602 if (iter->trace->splice_read) {
4603 ret = iter->trace->splice_read(iter, filp,
4604 ppos, pipe, len, flags);
4609 ret = tracing_wait_pipe(filp);
4613 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4618 trace_event_read_lock();
4619 trace_access_lock(iter->cpu_file);
4621 /* Fill as many pages as possible. */
4622 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4623 spd.pages[i] = alloc_page(GFP_KERNEL);
4627 rem = tracing_fill_pipe_page(rem, iter);
4629 /* Copy the data into the page, so we can start over. */
4630 ret = trace_seq_to_buffer(&iter->seq,
4631 page_address(spd.pages[i]),
4634 __free_page(spd.pages[i]);
4637 spd.partial[i].offset = 0;
4638 spd.partial[i].len = iter->seq.len;
4640 trace_seq_init(&iter->seq);
4643 trace_access_unlock(iter->cpu_file);
4644 trace_event_read_unlock();
4645 mutex_unlock(&iter->mutex);
4649 ret = splice_to_pipe(pipe, &spd);
4651 splice_shrink_spd(&spd);
4655 mutex_unlock(&iter->mutex);
4660 tracing_entries_read(struct file *filp, char __user *ubuf,
4661 size_t cnt, loff_t *ppos)
4663 struct inode *inode = file_inode(filp);
4664 struct trace_array *tr = inode->i_private;
4665 int cpu = tracing_get_cpu(inode);
4670 mutex_lock(&trace_types_lock);
4672 if (cpu == RING_BUFFER_ALL_CPUS) {
4673 int cpu, buf_size_same;
4678 /* check if all cpu sizes are same */
4679 for_each_tracing_cpu(cpu) {
4680 /* fill in the size from first enabled cpu */
4682 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4683 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4689 if (buf_size_same) {
4690 if (!ring_buffer_expanded)
4691 r = sprintf(buf, "%lu (expanded: %lu)\n",
4693 trace_buf_size >> 10);
4695 r = sprintf(buf, "%lu\n", size >> 10);
4697 r = sprintf(buf, "X\n");
4699 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4701 mutex_unlock(&trace_types_lock);
4703 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4708 tracing_entries_write(struct file *filp, const char __user *ubuf,
4709 size_t cnt, loff_t *ppos)
4711 struct inode *inode = file_inode(filp);
4712 struct trace_array *tr = inode->i_private;
4716 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4720 /* must have at least 1 entry */
4724 /* value is in KB */
4726 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4736 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4737 size_t cnt, loff_t *ppos)
4739 struct trace_array *tr = filp->private_data;
4742 unsigned long size = 0, expanded_size = 0;
4744 mutex_lock(&trace_types_lock);
4745 for_each_tracing_cpu(cpu) {
4746 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4747 if (!ring_buffer_expanded)
4748 expanded_size += trace_buf_size >> 10;
4750 if (ring_buffer_expanded)
4751 r = sprintf(buf, "%lu\n", size);
4753 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4754 mutex_unlock(&trace_types_lock);
4756 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4760 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4761 size_t cnt, loff_t *ppos)
4764 * There is no need to read what the user has written, this function
4765 * is just to make sure that there is no error when "echo" is used
4774 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4776 struct trace_array *tr = inode->i_private;
4778 /* disable tracing ? */
4779 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4780 tracer_tracing_off(tr);
4781 /* resize the ring buffer to 0 */
4782 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4784 trace_array_put(tr);
4790 tracing_mark_write(struct file *filp, const char __user *ubuf,
4791 size_t cnt, loff_t *fpos)
4793 unsigned long addr = (unsigned long)ubuf;
4794 struct trace_array *tr = filp->private_data;
4795 struct ring_buffer_event *event;
4796 struct ring_buffer *buffer;
4797 struct print_entry *entry;
4798 unsigned long irq_flags;
4799 struct page *pages[2];
4809 if (tracing_disabled)
4812 if (!(trace_flags & TRACE_ITER_MARKERS))
4815 if (cnt > TRACE_BUF_SIZE)
4816 cnt = TRACE_BUF_SIZE;
4819 * Userspace is injecting traces into the kernel trace buffer.
4820 * We want to be as non intrusive as possible.
4821 * To do so, we do not want to allocate any special buffers
4822 * or take any locks, but instead write the userspace data
4823 * straight into the ring buffer.
4825 * First we need to pin the userspace buffer into memory,
4826 * which, most likely it is, because it just referenced it.
4827 * But there's no guarantee that it is. By using get_user_pages_fast()
4828 * and kmap_atomic/kunmap_atomic() we can get access to the
4829 * pages directly. We then write the data directly into the
4832 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4834 /* check if we cross pages */
4835 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4838 offset = addr & (PAGE_SIZE - 1);
4841 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4842 if (ret < nr_pages) {
4844 put_page(pages[ret]);
4849 for (i = 0; i < nr_pages; i++)
4850 map_page[i] = kmap_atomic(pages[i]);
4852 local_save_flags(irq_flags);
4853 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4854 buffer = tr->trace_buffer.buffer;
4855 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4856 irq_flags, preempt_count());
4858 /* Ring buffer disabled, return as if not open for write */
4863 entry = ring_buffer_event_data(event);
4864 entry->ip = _THIS_IP_;
4866 if (nr_pages == 2) {
4867 len = PAGE_SIZE - offset;
4868 memcpy(&entry->buf, map_page[0] + offset, len);
4869 memcpy(&entry->buf[len], map_page[1], cnt - len);
4871 memcpy(&entry->buf, map_page[0] + offset, cnt);
4873 if (entry->buf[cnt - 1] != '\n') {
4874 entry->buf[cnt] = '\n';
4875 entry->buf[cnt + 1] = '\0';
4877 entry->buf[cnt] = '\0';
4879 __buffer_unlock_commit(buffer, event);
4886 for (i = 0; i < nr_pages; i++){
4887 kunmap_atomic(map_page[i]);
4894 static int tracing_clock_show(struct seq_file *m, void *v)
4896 struct trace_array *tr = m->private;
4899 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4901 "%s%s%s%s", i ? " " : "",
4902 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4903 i == tr->clock_id ? "]" : "");
4909 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4913 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4914 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4917 if (i == ARRAY_SIZE(trace_clocks))
4920 mutex_lock(&trace_types_lock);
4924 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4927 * New clock may not be consistent with the previous clock.
4928 * Reset the buffer so that it doesn't have incomparable timestamps.
4930 tracing_reset_online_cpus(&tr->trace_buffer);
4932 #ifdef CONFIG_TRACER_MAX_TRACE
4933 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4934 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4935 tracing_reset_online_cpus(&tr->max_buffer);
4938 mutex_unlock(&trace_types_lock);
4943 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4944 size_t cnt, loff_t *fpos)
4946 struct seq_file *m = filp->private_data;
4947 struct trace_array *tr = m->private;
4949 const char *clockstr;
4952 if (cnt >= sizeof(buf))
4955 if (copy_from_user(&buf, ubuf, cnt))
4960 clockstr = strstrip(buf);
4962 ret = tracing_set_clock(tr, clockstr);
4971 static int tracing_clock_open(struct inode *inode, struct file *file)
4973 struct trace_array *tr = inode->i_private;
4976 if (tracing_disabled)
4979 if (trace_array_get(tr))
4982 ret = single_open(file, tracing_clock_show, inode->i_private);
4984 trace_array_put(tr);
4989 struct ftrace_buffer_info {
4990 struct trace_iterator iter;
4995 #ifdef CONFIG_TRACER_SNAPSHOT
4996 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4998 struct trace_array *tr = inode->i_private;
4999 struct trace_iterator *iter;
5003 if (trace_array_get(tr) < 0)
5006 if (file->f_mode & FMODE_READ) {
5007 iter = __tracing_open(inode, file, true);
5009 ret = PTR_ERR(iter);
5011 /* Writes still need the seq_file to hold the private data */
5013 m = kzalloc(sizeof(*m), GFP_KERNEL);
5016 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5024 iter->trace_buffer = &tr->max_buffer;
5025 iter->cpu_file = tracing_get_cpu(inode);
5027 file->private_data = m;
5031 trace_array_put(tr);
5037 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5040 struct seq_file *m = filp->private_data;
5041 struct trace_iterator *iter = m->private;
5042 struct trace_array *tr = iter->tr;
5046 ret = tracing_update_buffers();
5050 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5054 mutex_lock(&trace_types_lock);
5056 if (tr->current_trace->use_max_tr) {
5063 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5067 if (tr->allocated_snapshot)
5071 /* Only allow per-cpu swap if the ring buffer supports it */
5072 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5073 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5078 if (!tr->allocated_snapshot) {
5079 ret = alloc_snapshot(tr);
5083 local_irq_disable();
5084 /* Now, we're going to swap */
5085 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5086 update_max_tr(tr, current, smp_processor_id());
5088 update_max_tr_single(tr, current, iter->cpu_file);
5092 if (tr->allocated_snapshot) {
5093 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5094 tracing_reset_online_cpus(&tr->max_buffer);
5096 tracing_reset(&tr->max_buffer, iter->cpu_file);
5106 mutex_unlock(&trace_types_lock);
5110 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5112 struct seq_file *m = file->private_data;
5115 ret = tracing_release(inode, file);
5117 if (file->f_mode & FMODE_READ)
5120 /* If write only, the seq_file is just a stub */
5128 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5129 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5130 size_t count, loff_t *ppos);
5131 static int tracing_buffers_release(struct inode *inode, struct file *file);
5132 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5133 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5135 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5137 struct ftrace_buffer_info *info;
5140 ret = tracing_buffers_open(inode, filp);
5144 info = filp->private_data;
5146 if (info->iter.trace->use_max_tr) {
5147 tracing_buffers_release(inode, filp);
5151 info->iter.snapshot = true;
5152 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5157 #endif /* CONFIG_TRACER_SNAPSHOT */
5160 static const struct file_operations tracing_max_lat_fops = {
5161 .open = tracing_open_generic,
5162 .read = tracing_max_lat_read,
5163 .write = tracing_max_lat_write,
5164 .llseek = generic_file_llseek,
5167 static const struct file_operations set_tracer_fops = {
5168 .open = tracing_open_generic,
5169 .read = tracing_set_trace_read,
5170 .write = tracing_set_trace_write,
5171 .llseek = generic_file_llseek,
5174 static const struct file_operations tracing_pipe_fops = {
5175 .open = tracing_open_pipe,
5176 .poll = tracing_poll_pipe,
5177 .read = tracing_read_pipe,
5178 .splice_read = tracing_splice_read_pipe,
5179 .release = tracing_release_pipe,
5180 .llseek = no_llseek,
5183 static const struct file_operations tracing_entries_fops = {
5184 .open = tracing_open_generic_tr,
5185 .read = tracing_entries_read,
5186 .write = tracing_entries_write,
5187 .llseek = generic_file_llseek,
5188 .release = tracing_release_generic_tr,
5191 static const struct file_operations tracing_total_entries_fops = {
5192 .open = tracing_open_generic_tr,
5193 .read = tracing_total_entries_read,
5194 .llseek = generic_file_llseek,
5195 .release = tracing_release_generic_tr,
5198 static const struct file_operations tracing_free_buffer_fops = {
5199 .open = tracing_open_generic_tr,
5200 .write = tracing_free_buffer_write,
5201 .release = tracing_free_buffer_release,
5204 static const struct file_operations tracing_mark_fops = {
5205 .open = tracing_open_generic_tr,
5206 .write = tracing_mark_write,
5207 .llseek = generic_file_llseek,
5208 .release = tracing_release_generic_tr,
5211 static const struct file_operations trace_clock_fops = {
5212 .open = tracing_clock_open,
5214 .llseek = seq_lseek,
5215 .release = tracing_single_release_tr,
5216 .write = tracing_clock_write,
5219 #ifdef CONFIG_TRACER_SNAPSHOT
5220 static const struct file_operations snapshot_fops = {
5221 .open = tracing_snapshot_open,
5223 .write = tracing_snapshot_write,
5224 .llseek = tracing_lseek,
5225 .release = tracing_snapshot_release,
5228 static const struct file_operations snapshot_raw_fops = {
5229 .open = snapshot_raw_open,
5230 .read = tracing_buffers_read,
5231 .release = tracing_buffers_release,
5232 .splice_read = tracing_buffers_splice_read,
5233 .llseek = no_llseek,
5236 #endif /* CONFIG_TRACER_SNAPSHOT */
5238 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5240 struct trace_array *tr = inode->i_private;
5241 struct ftrace_buffer_info *info;
5244 if (tracing_disabled)
5247 if (trace_array_get(tr) < 0)
5250 info = kzalloc(sizeof(*info), GFP_KERNEL);
5252 trace_array_put(tr);
5256 mutex_lock(&trace_types_lock);
5259 info->iter.cpu_file = tracing_get_cpu(inode);
5260 info->iter.trace = tr->current_trace;
5261 info->iter.trace_buffer = &tr->trace_buffer;
5263 /* Force reading ring buffer for first read */
5264 info->read = (unsigned int)-1;
5266 filp->private_data = info;
5268 mutex_unlock(&trace_types_lock);
5270 ret = nonseekable_open(inode, filp);
5272 trace_array_put(tr);
5278 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5280 struct ftrace_buffer_info *info = filp->private_data;
5281 struct trace_iterator *iter = &info->iter;
5283 return trace_poll(iter, filp, poll_table);
5287 tracing_buffers_read(struct file *filp, char __user *ubuf,
5288 size_t count, loff_t *ppos)
5290 struct ftrace_buffer_info *info = filp->private_data;
5291 struct trace_iterator *iter = &info->iter;
5298 mutex_lock(&trace_types_lock);
5300 #ifdef CONFIG_TRACER_MAX_TRACE
5301 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5308 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5314 /* Do we have previous read data to read? */
5315 if (info->read < PAGE_SIZE)
5319 trace_access_lock(iter->cpu_file);
5320 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5324 trace_access_unlock(iter->cpu_file);
5327 if (trace_empty(iter)) {
5328 if ((filp->f_flags & O_NONBLOCK)) {
5332 mutex_unlock(&trace_types_lock);
5333 ret = wait_on_pipe(iter);
5334 mutex_lock(&trace_types_lock);
5339 if (signal_pending(current)) {
5351 size = PAGE_SIZE - info->read;
5355 ret = copy_to_user(ubuf, info->spare + info->read, size);
5366 mutex_unlock(&trace_types_lock);
5371 static int tracing_buffers_release(struct inode *inode, struct file *file)
5373 struct ftrace_buffer_info *info = file->private_data;
5374 struct trace_iterator *iter = &info->iter;
5376 mutex_lock(&trace_types_lock);
5378 __trace_array_put(iter->tr);
5381 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5384 mutex_unlock(&trace_types_lock);
5390 struct ring_buffer *buffer;
5395 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5396 struct pipe_buffer *buf)
5398 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5403 ring_buffer_free_read_page(ref->buffer, ref->page);
5408 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5409 struct pipe_buffer *buf)
5411 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5416 /* Pipe buffer operations for a buffer. */
5417 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5419 .confirm = generic_pipe_buf_confirm,
5420 .release = buffer_pipe_buf_release,
5421 .steal = generic_pipe_buf_steal,
5422 .get = buffer_pipe_buf_get,
5426 * Callback from splice_to_pipe(), if we need to release some pages
5427 * at the end of the spd in case we error'ed out in filling the pipe.
5429 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5431 struct buffer_ref *ref =
5432 (struct buffer_ref *)spd->partial[i].private;
5437 ring_buffer_free_read_page(ref->buffer, ref->page);
5439 spd->partial[i].private = 0;
5443 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5444 struct pipe_inode_info *pipe, size_t len,
5447 struct ftrace_buffer_info *info = file->private_data;
5448 struct trace_iterator *iter = &info->iter;
5449 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5450 struct page *pages_def[PIPE_DEF_BUFFERS];
5451 struct splice_pipe_desc spd = {
5453 .partial = partial_def,
5454 .nr_pages_max = PIPE_DEF_BUFFERS,
5456 .ops = &buffer_pipe_buf_ops,
5457 .spd_release = buffer_spd_release,
5459 struct buffer_ref *ref;
5460 int entries, size, i;
5463 mutex_lock(&trace_types_lock);
5465 #ifdef CONFIG_TRACER_MAX_TRACE
5466 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5472 if (splice_grow_spd(pipe, &spd)) {
5477 if (*ppos & (PAGE_SIZE - 1)) {
5482 if (len & (PAGE_SIZE - 1)) {
5483 if (len < PAGE_SIZE) {
5491 trace_access_lock(iter->cpu_file);
5492 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5494 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5498 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5503 ref->buffer = iter->trace_buffer->buffer;
5504 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5510 r = ring_buffer_read_page(ref->buffer, &ref->page,
5511 len, iter->cpu_file, 1);
5513 ring_buffer_free_read_page(ref->buffer, ref->page);
5519 * zero out any left over data, this is going to
5522 size = ring_buffer_page_len(ref->page);
5523 if (size < PAGE_SIZE)
5524 memset(ref->page + size, 0, PAGE_SIZE - size);
5526 page = virt_to_page(ref->page);
5528 spd.pages[i] = page;
5529 spd.partial[i].len = PAGE_SIZE;
5530 spd.partial[i].offset = 0;
5531 spd.partial[i].private = (unsigned long)ref;
5535 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5538 trace_access_unlock(iter->cpu_file);
5541 /* did we read anything? */
5542 if (!spd.nr_pages) {
5543 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5547 mutex_unlock(&trace_types_lock);
5548 ret = wait_on_pipe(iter);
5549 mutex_lock(&trace_types_lock);
5552 if (signal_pending(current)) {
5559 ret = splice_to_pipe(pipe, &spd);
5560 splice_shrink_spd(&spd);
5562 mutex_unlock(&trace_types_lock);
5567 static const struct file_operations tracing_buffers_fops = {
5568 .open = tracing_buffers_open,
5569 .read = tracing_buffers_read,
5570 .poll = tracing_buffers_poll,
5571 .release = tracing_buffers_release,
5572 .splice_read = tracing_buffers_splice_read,
5573 .llseek = no_llseek,
5577 tracing_stats_read(struct file *filp, char __user *ubuf,
5578 size_t count, loff_t *ppos)
5580 struct inode *inode = file_inode(filp);
5581 struct trace_array *tr = inode->i_private;
5582 struct trace_buffer *trace_buf = &tr->trace_buffer;
5583 int cpu = tracing_get_cpu(inode);
5584 struct trace_seq *s;
5586 unsigned long long t;
5587 unsigned long usec_rem;
5589 s = kmalloc(sizeof(*s), GFP_KERNEL);
5595 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5596 trace_seq_printf(s, "entries: %ld\n", cnt);
5598 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5599 trace_seq_printf(s, "overrun: %ld\n", cnt);
5601 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5602 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5604 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5605 trace_seq_printf(s, "bytes: %ld\n", cnt);
5607 if (trace_clocks[tr->clock_id].in_ns) {
5608 /* local or global for trace_clock */
5609 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5610 usec_rem = do_div(t, USEC_PER_SEC);
5611 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5614 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5615 usec_rem = do_div(t, USEC_PER_SEC);
5616 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5618 /* counter or tsc mode for trace_clock */
5619 trace_seq_printf(s, "oldest event ts: %llu\n",
5620 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5622 trace_seq_printf(s, "now ts: %llu\n",
5623 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5626 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5627 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5629 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5630 trace_seq_printf(s, "read events: %ld\n", cnt);
5632 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5639 static const struct file_operations tracing_stats_fops = {
5640 .open = tracing_open_generic_tr,
5641 .read = tracing_stats_read,
5642 .llseek = generic_file_llseek,
5643 .release = tracing_release_generic_tr,
5646 #ifdef CONFIG_DYNAMIC_FTRACE
5648 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5654 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5655 size_t cnt, loff_t *ppos)
5657 static char ftrace_dyn_info_buffer[1024];
5658 static DEFINE_MUTEX(dyn_info_mutex);
5659 unsigned long *p = filp->private_data;
5660 char *buf = ftrace_dyn_info_buffer;
5661 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5664 mutex_lock(&dyn_info_mutex);
5665 r = sprintf(buf, "%ld ", *p);
5667 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5670 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5672 mutex_unlock(&dyn_info_mutex);
5677 static const struct file_operations tracing_dyn_info_fops = {
5678 .open = tracing_open_generic,
5679 .read = tracing_read_dyn_info,
5680 .llseek = generic_file_llseek,
5682 #endif /* CONFIG_DYNAMIC_FTRACE */
5684 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5686 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5692 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5694 unsigned long *count = (long *)data;
5706 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5707 struct ftrace_probe_ops *ops, void *data)
5709 long count = (long)data;
5711 seq_printf(m, "%ps:", (void *)ip);
5713 seq_printf(m, "snapshot");
5716 seq_printf(m, ":unlimited\n");
5718 seq_printf(m, ":count=%ld\n", count);
5723 static struct ftrace_probe_ops snapshot_probe_ops = {
5724 .func = ftrace_snapshot,
5725 .print = ftrace_snapshot_print,
5728 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5729 .func = ftrace_count_snapshot,
5730 .print = ftrace_snapshot_print,
5734 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5735 char *glob, char *cmd, char *param, int enable)
5737 struct ftrace_probe_ops *ops;
5738 void *count = (void *)-1;
5742 /* hash funcs only work with set_ftrace_filter */
5746 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5748 if (glob[0] == '!') {
5749 unregister_ftrace_function_probe_func(glob+1, ops);
5756 number = strsep(¶m, ":");
5758 if (!strlen(number))
5762 * We use the callback data field (which is a pointer)
5765 ret = kstrtoul(number, 0, (unsigned long *)&count);
5770 ret = register_ftrace_function_probe(glob, ops, count);
5773 alloc_snapshot(&global_trace);
5775 return ret < 0 ? ret : 0;
5778 static struct ftrace_func_command ftrace_snapshot_cmd = {
5780 .func = ftrace_trace_snapshot_callback,
5783 static __init int register_snapshot_cmd(void)
5785 return register_ftrace_command(&ftrace_snapshot_cmd);
5788 static inline __init int register_snapshot_cmd(void) { return 0; }
5789 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5791 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5796 if (!debugfs_initialized())
5799 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5800 tr->dir = debugfs_create_dir("tracing", NULL);
5803 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5808 struct dentry *tracing_init_dentry(void)
5810 return tracing_init_dentry_tr(&global_trace);
5813 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5815 struct dentry *d_tracer;
5818 return tr->percpu_dir;
5820 d_tracer = tracing_init_dentry_tr(tr);
5824 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5826 WARN_ONCE(!tr->percpu_dir,
5827 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5829 return tr->percpu_dir;
5832 static struct dentry *
5833 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5834 void *data, long cpu, const struct file_operations *fops)
5836 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5838 if (ret) /* See tracing_get_cpu() */
5839 ret->d_inode->i_cdev = (void *)(cpu + 1);
5844 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5846 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5847 struct dentry *d_cpu;
5848 char cpu_dir[30]; /* 30 characters should be more than enough */
5853 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5854 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5856 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5860 /* per cpu trace_pipe */
5861 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5862 tr, cpu, &tracing_pipe_fops);
5865 trace_create_cpu_file("trace", 0644, d_cpu,
5866 tr, cpu, &tracing_fops);
5868 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5869 tr, cpu, &tracing_buffers_fops);
5871 trace_create_cpu_file("stats", 0444, d_cpu,
5872 tr, cpu, &tracing_stats_fops);
5874 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5875 tr, cpu, &tracing_entries_fops);
5877 #ifdef CONFIG_TRACER_SNAPSHOT
5878 trace_create_cpu_file("snapshot", 0644, d_cpu,
5879 tr, cpu, &snapshot_fops);
5881 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5882 tr, cpu, &snapshot_raw_fops);
5886 #ifdef CONFIG_FTRACE_SELFTEST
5887 /* Let selftest have access to static functions in this file */
5888 #include "trace_selftest.c"
5891 struct trace_option_dentry {
5892 struct tracer_opt *opt;
5893 struct tracer_flags *flags;
5894 struct trace_array *tr;
5895 struct dentry *entry;
5899 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5902 struct trace_option_dentry *topt = filp->private_data;
5905 if (topt->flags->val & topt->opt->bit)
5910 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5914 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5917 struct trace_option_dentry *topt = filp->private_data;
5921 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5925 if (val != 0 && val != 1)
5928 if (!!(topt->flags->val & topt->opt->bit) != val) {
5929 mutex_lock(&trace_types_lock);
5930 ret = __set_tracer_option(topt->tr, topt->flags,
5932 mutex_unlock(&trace_types_lock);
5943 static const struct file_operations trace_options_fops = {
5944 .open = tracing_open_generic,
5945 .read = trace_options_read,
5946 .write = trace_options_write,
5947 .llseek = generic_file_llseek,
5951 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5954 long index = (long)filp->private_data;
5957 if (trace_flags & (1 << index))
5962 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5966 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5969 struct trace_array *tr = &global_trace;
5970 long index = (long)filp->private_data;
5974 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5978 if (val != 0 && val != 1)
5981 mutex_lock(&trace_types_lock);
5982 ret = set_tracer_flag(tr, 1 << index, val);
5983 mutex_unlock(&trace_types_lock);
5993 static const struct file_operations trace_options_core_fops = {
5994 .open = tracing_open_generic,
5995 .read = trace_options_core_read,
5996 .write = trace_options_core_write,
5997 .llseek = generic_file_llseek,
6000 struct dentry *trace_create_file(const char *name,
6002 struct dentry *parent,
6004 const struct file_operations *fops)
6008 ret = debugfs_create_file(name, mode, parent, data, fops);
6010 pr_warning("Could not create debugfs '%s' entry\n", name);
6016 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6018 struct dentry *d_tracer;
6023 d_tracer = tracing_init_dentry_tr(tr);
6027 tr->options = debugfs_create_dir("options", d_tracer);
6029 pr_warning("Could not create debugfs directory 'options'\n");
6037 create_trace_option_file(struct trace_array *tr,
6038 struct trace_option_dentry *topt,
6039 struct tracer_flags *flags,
6040 struct tracer_opt *opt)
6042 struct dentry *t_options;
6044 t_options = trace_options_init_dentry(tr);
6048 topt->flags = flags;
6052 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6053 &trace_options_fops);
6057 static struct trace_option_dentry *
6058 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6060 struct trace_option_dentry *topts;
6061 struct tracer_flags *flags;
6062 struct tracer_opt *opts;
6068 flags = tracer->flags;
6070 if (!flags || !flags->opts)
6075 for (cnt = 0; opts[cnt].name; cnt++)
6078 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6082 for (cnt = 0; opts[cnt].name; cnt++)
6083 create_trace_option_file(tr, &topts[cnt], flags,
6090 destroy_trace_option_files(struct trace_option_dentry *topts)
6097 for (cnt = 0; topts[cnt].opt; cnt++) {
6098 if (topts[cnt].entry)
6099 debugfs_remove(topts[cnt].entry);
6105 static struct dentry *
6106 create_trace_option_core_file(struct trace_array *tr,
6107 const char *option, long index)
6109 struct dentry *t_options;
6111 t_options = trace_options_init_dentry(tr);
6115 return trace_create_file(option, 0644, t_options, (void *)index,
6116 &trace_options_core_fops);
6119 static __init void create_trace_options_dir(struct trace_array *tr)
6121 struct dentry *t_options;
6124 t_options = trace_options_init_dentry(tr);
6128 for (i = 0; trace_options[i]; i++)
6129 create_trace_option_core_file(tr, trace_options[i], i);
6133 rb_simple_read(struct file *filp, char __user *ubuf,
6134 size_t cnt, loff_t *ppos)
6136 struct trace_array *tr = filp->private_data;
6140 r = tracer_tracing_is_on(tr);
6141 r = sprintf(buf, "%d\n", r);
6143 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6147 rb_simple_write(struct file *filp, const char __user *ubuf,
6148 size_t cnt, loff_t *ppos)
6150 struct trace_array *tr = filp->private_data;
6151 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6155 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6160 mutex_lock(&trace_types_lock);
6162 tracer_tracing_on(tr);
6163 if (tr->current_trace->start)
6164 tr->current_trace->start(tr);
6166 tracer_tracing_off(tr);
6167 if (tr->current_trace->stop)
6168 tr->current_trace->stop(tr);
6170 mutex_unlock(&trace_types_lock);
6178 static const struct file_operations rb_simple_fops = {
6179 .open = tracing_open_generic_tr,
6180 .read = rb_simple_read,
6181 .write = rb_simple_write,
6182 .release = tracing_release_generic_tr,
6183 .llseek = default_llseek,
6186 struct dentry *trace_instance_dir;
6189 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6192 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6194 enum ring_buffer_flags rb_flags;
6196 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6200 buf->buffer = ring_buffer_alloc(size, rb_flags);
6204 buf->data = alloc_percpu(struct trace_array_cpu);
6206 ring_buffer_free(buf->buffer);
6210 /* Allocate the first page for all buffers */
6211 set_buffer_entries(&tr->trace_buffer,
6212 ring_buffer_size(tr->trace_buffer.buffer, 0));
6217 static int allocate_trace_buffers(struct trace_array *tr, int size)
6221 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6225 #ifdef CONFIG_TRACER_MAX_TRACE
6226 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6227 allocate_snapshot ? size : 1);
6229 ring_buffer_free(tr->trace_buffer.buffer);
6230 free_percpu(tr->trace_buffer.data);
6233 tr->allocated_snapshot = allocate_snapshot;
6236 * Only the top level trace array gets its snapshot allocated
6237 * from the kernel command line.
6239 allocate_snapshot = false;
6244 static void free_trace_buffer(struct trace_buffer *buf)
6247 ring_buffer_free(buf->buffer);
6249 free_percpu(buf->data);
6254 static void free_trace_buffers(struct trace_array *tr)
6259 free_trace_buffer(&tr->trace_buffer);
6261 #ifdef CONFIG_TRACER_MAX_TRACE
6262 free_trace_buffer(&tr->max_buffer);
6266 static int new_instance_create(const char *name)
6268 struct trace_array *tr;
6271 mutex_lock(&trace_types_lock);
6274 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6275 if (tr->name && strcmp(tr->name, name) == 0)
6280 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6284 tr->name = kstrdup(name, GFP_KERNEL);
6288 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6291 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6293 raw_spin_lock_init(&tr->start_lock);
6295 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6297 tr->current_trace = &nop_trace;
6299 INIT_LIST_HEAD(&tr->systems);
6300 INIT_LIST_HEAD(&tr->events);
6302 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6305 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6309 ret = event_trace_add_tracer(tr->dir, tr);
6311 debugfs_remove_recursive(tr->dir);
6315 init_tracer_debugfs(tr, tr->dir);
6317 list_add(&tr->list, &ftrace_trace_arrays);
6319 mutex_unlock(&trace_types_lock);
6324 free_trace_buffers(tr);
6325 free_cpumask_var(tr->tracing_cpumask);
6330 mutex_unlock(&trace_types_lock);
6336 static int instance_delete(const char *name)
6338 struct trace_array *tr;
6342 mutex_lock(&trace_types_lock);
6345 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6346 if (tr->name && strcmp(tr->name, name) == 0) {
6358 list_del(&tr->list);
6360 tracing_set_nop(tr);
6361 event_trace_del_tracer(tr);
6362 ftrace_destroy_function_files(tr);
6363 debugfs_remove_recursive(tr->dir);
6364 free_trace_buffers(tr);
6372 mutex_unlock(&trace_types_lock);
6377 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6379 struct dentry *parent;
6382 /* Paranoid: Make sure the parent is the "instances" directory */
6383 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6384 if (WARN_ON_ONCE(parent != trace_instance_dir))
6388 * The inode mutex is locked, but debugfs_create_dir() will also
6389 * take the mutex. As the instances directory can not be destroyed
6390 * or changed in any other way, it is safe to unlock it, and
6391 * let the dentry try. If two users try to make the same dir at
6392 * the same time, then the new_instance_create() will determine the
6395 mutex_unlock(&inode->i_mutex);
6397 ret = new_instance_create(dentry->d_iname);
6399 mutex_lock(&inode->i_mutex);
6404 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6406 struct dentry *parent;
6409 /* Paranoid: Make sure the parent is the "instances" directory */
6410 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6411 if (WARN_ON_ONCE(parent != trace_instance_dir))
6414 /* The caller did a dget() on dentry */
6415 mutex_unlock(&dentry->d_inode->i_mutex);
6418 * The inode mutex is locked, but debugfs_create_dir() will also
6419 * take the mutex. As the instances directory can not be destroyed
6420 * or changed in any other way, it is safe to unlock it, and
6421 * let the dentry try. If two users try to make the same dir at
6422 * the same time, then the instance_delete() will determine the
6425 mutex_unlock(&inode->i_mutex);
6427 ret = instance_delete(dentry->d_iname);
6429 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6430 mutex_lock(&dentry->d_inode->i_mutex);
6435 static const struct inode_operations instance_dir_inode_operations = {
6436 .lookup = simple_lookup,
6437 .mkdir = instance_mkdir,
6438 .rmdir = instance_rmdir,
6441 static __init void create_trace_instances(struct dentry *d_tracer)
6443 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6444 if (WARN_ON(!trace_instance_dir))
6447 /* Hijack the dir inode operations, to allow mkdir */
6448 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6452 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6456 trace_create_file("available_tracers", 0444, d_tracer,
6457 tr, &show_traces_fops);
6459 trace_create_file("current_tracer", 0644, d_tracer,
6460 tr, &set_tracer_fops);
6462 trace_create_file("tracing_cpumask", 0644, d_tracer,
6463 tr, &tracing_cpumask_fops);
6465 trace_create_file("trace_options", 0644, d_tracer,
6466 tr, &tracing_iter_fops);
6468 trace_create_file("trace", 0644, d_tracer,
6471 trace_create_file("trace_pipe", 0444, d_tracer,
6472 tr, &tracing_pipe_fops);
6474 trace_create_file("buffer_size_kb", 0644, d_tracer,
6475 tr, &tracing_entries_fops);
6477 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6478 tr, &tracing_total_entries_fops);
6480 trace_create_file("free_buffer", 0200, d_tracer,
6481 tr, &tracing_free_buffer_fops);
6483 trace_create_file("trace_marker", 0220, d_tracer,
6484 tr, &tracing_mark_fops);
6486 trace_create_file("trace_clock", 0644, d_tracer, tr,
6489 trace_create_file("tracing_on", 0644, d_tracer,
6490 tr, &rb_simple_fops);
6492 #ifdef CONFIG_TRACER_MAX_TRACE
6493 trace_create_file("tracing_max_latency", 0644, d_tracer,
6494 &tr->max_latency, &tracing_max_lat_fops);
6497 if (ftrace_create_function_files(tr, d_tracer))
6498 WARN(1, "Could not allocate function filter files");
6500 #ifdef CONFIG_TRACER_SNAPSHOT
6501 trace_create_file("snapshot", 0644, d_tracer,
6502 tr, &snapshot_fops);
6505 for_each_tracing_cpu(cpu)
6506 tracing_init_debugfs_percpu(tr, cpu);
6510 static __init int tracer_init_debugfs(void)
6512 struct dentry *d_tracer;
6514 trace_access_lock_init();
6516 d_tracer = tracing_init_dentry();
6520 init_tracer_debugfs(&global_trace, d_tracer);
6522 trace_create_file("tracing_thresh", 0644, d_tracer,
6523 &tracing_thresh, &tracing_max_lat_fops);
6525 trace_create_file("README", 0444, d_tracer,
6526 NULL, &tracing_readme_fops);
6528 trace_create_file("saved_cmdlines", 0444, d_tracer,
6529 NULL, &tracing_saved_cmdlines_fops);
6531 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6532 NULL, &tracing_saved_cmdlines_size_fops);
6534 #ifdef CONFIG_DYNAMIC_FTRACE
6535 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6536 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6539 create_trace_instances(d_tracer);
6541 create_trace_options_dir(&global_trace);
6546 static int trace_panic_handler(struct notifier_block *this,
6547 unsigned long event, void *unused)
6549 if (ftrace_dump_on_oops)
6550 ftrace_dump(ftrace_dump_on_oops);
6554 static struct notifier_block trace_panic_notifier = {
6555 .notifier_call = trace_panic_handler,
6557 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6560 static int trace_die_handler(struct notifier_block *self,
6566 if (ftrace_dump_on_oops)
6567 ftrace_dump(ftrace_dump_on_oops);
6575 static struct notifier_block trace_die_notifier = {
6576 .notifier_call = trace_die_handler,
6581 * printk is set to max of 1024, we really don't need it that big.
6582 * Nothing should be printing 1000 characters anyway.
6584 #define TRACE_MAX_PRINT 1000
6587 * Define here KERN_TRACE so that we have one place to modify
6588 * it if we decide to change what log level the ftrace dump
6591 #define KERN_TRACE KERN_EMERG
6594 trace_printk_seq(struct trace_seq *s)
6596 /* Probably should print a warning here. */
6597 if (s->len >= TRACE_MAX_PRINT)
6598 s->len = TRACE_MAX_PRINT;
6600 /* should be zero ended, but we are paranoid. */
6601 s->buffer[s->len] = 0;
6603 printk(KERN_TRACE "%s", s->buffer);
6608 void trace_init_global_iter(struct trace_iterator *iter)
6610 iter->tr = &global_trace;
6611 iter->trace = iter->tr->current_trace;
6612 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6613 iter->trace_buffer = &global_trace.trace_buffer;
6615 if (iter->trace && iter->trace->open)
6616 iter->trace->open(iter);
6618 /* Annotate start of buffers if we had overruns */
6619 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6620 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6622 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6623 if (trace_clocks[iter->tr->clock_id].in_ns)
6624 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6627 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6629 /* use static because iter can be a bit big for the stack */
6630 static struct trace_iterator iter;
6631 static atomic_t dump_running;
6632 unsigned int old_userobj;
6633 unsigned long flags;
6636 /* Only allow one dump user at a time. */
6637 if (atomic_inc_return(&dump_running) != 1) {
6638 atomic_dec(&dump_running);
6643 * Always turn off tracing when we dump.
6644 * We don't need to show trace output of what happens
6645 * between multiple crashes.
6647 * If the user does a sysrq-z, then they can re-enable
6648 * tracing with echo 1 > tracing_on.
6652 local_irq_save(flags);
6654 /* Simulate the iterator */
6655 trace_init_global_iter(&iter);
6657 for_each_tracing_cpu(cpu) {
6658 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6661 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6663 /* don't look at user memory in panic mode */
6664 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6666 switch (oops_dump_mode) {
6668 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6671 iter.cpu_file = raw_smp_processor_id();
6676 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6677 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6680 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6682 /* Did function tracer already get disabled? */
6683 if (ftrace_is_dead()) {
6684 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6685 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6689 * We need to stop all tracing on all CPUS to read the
6690 * the next buffer. This is a bit expensive, but is
6691 * not done often. We fill all what we can read,
6692 * and then release the locks again.
6695 while (!trace_empty(&iter)) {
6698 printk(KERN_TRACE "---------------------------------\n");
6702 /* reset all but tr, trace, and overruns */
6703 memset(&iter.seq, 0,
6704 sizeof(struct trace_iterator) -
6705 offsetof(struct trace_iterator, seq));
6706 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6709 if (trace_find_next_entry_inc(&iter) != NULL) {
6712 ret = print_trace_line(&iter);
6713 if (ret != TRACE_TYPE_NO_CONSUME)
6714 trace_consume(&iter);
6716 touch_nmi_watchdog();
6718 trace_printk_seq(&iter.seq);
6722 printk(KERN_TRACE " (ftrace buffer empty)\n");
6724 printk(KERN_TRACE "---------------------------------\n");
6727 trace_flags |= old_userobj;
6729 for_each_tracing_cpu(cpu) {
6730 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6732 atomic_dec(&dump_running);
6733 local_irq_restore(flags);
6735 EXPORT_SYMBOL_GPL(ftrace_dump);
6737 __init static int tracer_alloc_buffers(void)
6743 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6746 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6747 goto out_free_buffer_mask;
6749 /* Only allocate trace_printk buffers if a trace_printk exists */
6750 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6751 /* Must be called before global_trace.buffer is allocated */
6752 trace_printk_init_buffers();
6754 /* To save memory, keep the ring buffer size to its minimum */
6755 if (ring_buffer_expanded)
6756 ring_buf_size = trace_buf_size;
6760 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6761 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6763 raw_spin_lock_init(&global_trace.start_lock);
6765 /* Used for event triggers */
6766 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6768 goto out_free_cpumask;
6770 if (trace_create_savedcmd() < 0)
6771 goto out_free_temp_buffer;
6773 /* TODO: make the number of buffers hot pluggable with CPUS */
6774 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6775 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6777 goto out_free_savedcmd;
6780 if (global_trace.buffer_disabled)
6783 if (trace_boot_clock) {
6784 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6786 pr_warning("Trace clock %s not defined, going back to default\n",
6791 * register_tracer() might reference current_trace, so it
6792 * needs to be set before we register anything. This is
6793 * just a bootstrap of current_trace anyway.
6795 global_trace.current_trace = &nop_trace;
6797 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6799 ftrace_init_global_array_ops(&global_trace);
6801 register_tracer(&nop_trace);
6803 /* All seems OK, enable tracing */
6804 tracing_disabled = 0;
6806 atomic_notifier_chain_register(&panic_notifier_list,
6807 &trace_panic_notifier);
6809 register_die_notifier(&trace_die_notifier);
6811 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6813 INIT_LIST_HEAD(&global_trace.systems);
6814 INIT_LIST_HEAD(&global_trace.events);
6815 list_add(&global_trace.list, &ftrace_trace_arrays);
6817 while (trace_boot_options) {
6820 option = strsep(&trace_boot_options, ",");
6821 trace_set_options(&global_trace, option);
6824 register_snapshot_cmd();
6829 free_saved_cmdlines_buffer(savedcmd);
6830 out_free_temp_buffer:
6831 ring_buffer_free(temp_buffer);
6833 free_cpumask_var(global_trace.tracing_cpumask);
6834 out_free_buffer_mask:
6835 free_cpumask_var(tracing_buffer_mask);
6840 __init static int clear_boot_tracer(void)
6843 * The default tracer at boot buffer is an init section.
6844 * This function is called in lateinit. If we did not
6845 * find the boot tracer, then clear it out, to prevent
6846 * later registration from accessing the buffer that is
6847 * about to be freed.
6849 if (!default_bootup_tracer)
6852 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6853 default_bootup_tracer);
6854 default_bootup_tracer = NULL;
6859 early_initcall(tracer_alloc_buffers);
6860 fs_initcall(tracer_init_debugfs);
6861 late_initcall(clear_boot_tracer);