2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
78 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled = 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head {
125 unsigned long length;
128 union trace_enum_map_item;
130 struct trace_enum_map_tail {
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item *next;
136 const char *end; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item {
149 struct trace_enum_map map;
150 struct trace_enum_map_head head;
151 struct trace_enum_map_tail tail;
154 static union trace_enum_map_item *trace_enum_maps;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161 static char *default_bootup_tracer;
163 static bool allocate_snapshot;
165 static int __init set_cmdline_ftrace(char *str)
167 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168 default_bootup_tracer = bootup_tracer_buf;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded = true;
173 __setup("ftrace=", set_cmdline_ftrace);
175 static int __init set_ftrace_dump_on_oops(char *str)
177 if (*str++ != '=' || !*str) {
178 ftrace_dump_on_oops = DUMP_ALL;
182 if (!strcmp("orig_cpu", str)) {
183 ftrace_dump_on_oops = DUMP_ORIG;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
191 static int __init stop_trace_on_warning(char *str)
193 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194 __disable_trace_on_warning = 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning);
199 static int __init boot_alloc_snapshot(char *str)
201 allocate_snapshot = true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded = true;
206 __setup("alloc_snapshot", boot_alloc_snapshot);
209 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
211 static int __init set_trace_boot_options(char *str)
213 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
216 __setup("trace_options=", set_trace_boot_options);
218 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219 static char *trace_boot_clock __initdata;
221 static int __init set_trace_boot_clock(char *str)
223 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224 trace_boot_clock = trace_boot_clock_buf;
227 __setup("trace_clock=", set_trace_boot_clock);
229 static int __init set_tracepoint_printk(char *str)
231 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
232 tracepoint_printk = 1;
235 __setup("tp_printk", set_tracepoint_printk);
237 unsigned long long ns2usecs(cycle_t nsec)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace = {
273 .trace_flags = TRACE_DEFAULT_FLAGS,
276 LIST_HEAD(ftrace_trace_arrays);
278 int trace_array_get(struct trace_array *this_tr)
280 struct trace_array *tr;
283 mutex_lock(&trace_types_lock);
284 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
291 mutex_unlock(&trace_types_lock);
296 static void __trace_array_put(struct trace_array *this_tr)
298 WARN_ON(!this_tr->ref);
302 void trace_array_put(struct trace_array *this_tr)
304 mutex_lock(&trace_types_lock);
305 __trace_array_put(this_tr);
306 mutex_unlock(&trace_types_lock);
309 int call_filter_check_discard(struct trace_event_call *call, void *rec,
310 struct ring_buffer *buffer,
311 struct ring_buffer_event *event)
313 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
314 !filter_match_preds(call->filter, rec)) {
315 __trace_event_discard_commit(buffer, event);
322 void trace_free_pid_list(struct trace_pid_list *pid_list)
324 vfree(pid_list->pids);
329 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
330 * @filtered_pids: The list of pids to check
331 * @search_pid: The PID to find in @filtered_pids
333 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
336 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
339 * If pid_max changed after filtered_pids was created, we
340 * by default ignore all pids greater than the previous pid_max.
342 if (search_pid >= filtered_pids->pid_max)
345 return test_bit(search_pid, filtered_pids->pids);
349 * trace_ignore_this_task - should a task be ignored for tracing
350 * @filtered_pids: The list of pids to check
351 * @task: The task that should be ignored if not filtered
353 * Checks if @task should be traced or not from @filtered_pids.
354 * Returns true if @task should *NOT* be traced.
355 * Returns false if @task should be traced.
358 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
361 * Return false, because if filtered_pids does not exist,
362 * all pids are good to trace.
367 return !trace_find_filtered_pid(filtered_pids, task->pid);
371 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
372 * @pid_list: The list to modify
373 * @self: The current task for fork or NULL for exit
374 * @task: The task to add or remove
376 * If adding a task, if @self is defined, the task is only added if @self
377 * is also included in @pid_list. This happens on fork and tasks should
378 * only be added when the parent is listed. If @self is NULL, then the
379 * @task pid will be removed from the list, which would happen on exit
382 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
383 struct task_struct *self,
384 struct task_struct *task)
389 /* For forks, we only add if the forking task is listed */
391 if (!trace_find_filtered_pid(pid_list, self->pid))
395 /* Sorry, but we don't support pid_max changing after setting */
396 if (task->pid >= pid_list->pid_max)
399 /* "self" is set for forks, and NULL for exits */
401 set_bit(task->pid, pid_list->pids);
403 clear_bit(task->pid, pid_list->pids);
407 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
408 * @pid_list: The pid list to show
409 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
410 * @pos: The position of the file
412 * This is used by the seq_file "next" operation to iterate the pids
413 * listed in a trace_pid_list structure.
415 * Returns the pid+1 as we want to display pid of zero, but NULL would
416 * stop the iteration.
418 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
420 unsigned long pid = (unsigned long)v;
424 /* pid already is +1 of the actual prevous bit */
425 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
427 /* Return pid + 1 to allow zero to be represented */
428 if (pid < pid_list->pid_max)
429 return (void *)(pid + 1);
435 * trace_pid_start - Used for seq_file to start reading pid lists
436 * @pid_list: The pid list to show
437 * @pos: The position of the file
439 * This is used by seq_file "start" operation to start the iteration
442 * Returns the pid+1 as we want to display pid of zero, but NULL would
443 * stop the iteration.
445 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
450 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
451 if (pid >= pid_list->pid_max)
454 /* Return pid + 1 so that zero can be the exit value */
455 for (pid++; pid && l < *pos;
456 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
462 * trace_pid_show - show the current pid in seq_file processing
463 * @m: The seq_file structure to write into
464 * @v: A void pointer of the pid (+1) value to display
466 * Can be directly used by seq_file operations to display the current
469 int trace_pid_show(struct seq_file *m, void *v)
471 unsigned long pid = (unsigned long)v - 1;
473 seq_printf(m, "%lu\n", pid);
477 /* 128 should be much more than enough */
478 #define PID_BUF_SIZE 127
480 int trace_pid_write(struct trace_pid_list *filtered_pids,
481 struct trace_pid_list **new_pid_list,
482 const char __user *ubuf, size_t cnt)
484 struct trace_pid_list *pid_list;
485 struct trace_parser parser;
493 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
497 * Always recreate a new array. The write is an all or nothing
498 * operation. Always create a new array when adding new pids by
499 * the user. If the operation fails, then the current list is
502 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
506 pid_list->pid_max = READ_ONCE(pid_max);
508 /* Only truncating will shrink pid_max */
509 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
510 pid_list->pid_max = filtered_pids->pid_max;
512 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
513 if (!pid_list->pids) {
519 /* copy the current bits to the new max */
520 pid = find_first_bit(filtered_pids->pids,
521 filtered_pids->pid_max);
522 while (pid < filtered_pids->pid_max) {
523 set_bit(pid, pid_list->pids);
524 pid = find_next_bit(filtered_pids->pids,
525 filtered_pids->pid_max,
535 ret = trace_get_user(&parser, ubuf, cnt, &pos);
536 if (ret < 0 || !trace_parser_loaded(&parser))
543 parser.buffer[parser.idx] = 0;
546 if (kstrtoul(parser.buffer, 0, &val))
548 if (val >= pid_list->pid_max)
553 set_bit(pid, pid_list->pids);
556 trace_parser_clear(&parser);
559 trace_parser_put(&parser);
562 trace_free_pid_list(pid_list);
567 /* Cleared the list of pids */
568 trace_free_pid_list(pid_list);
573 *new_pid_list = pid_list;
578 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
582 /* Early boot up does not have a buffer yet */
584 return trace_clock_local();
586 ts = ring_buffer_time_stamp(buf->buffer, cpu);
587 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
592 cycle_t ftrace_now(int cpu)
594 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
598 * tracing_is_enabled - Show if global_trace has been disabled
600 * Shows if the global trace has been enabled or not. It uses the
601 * mirror flag "buffer_disabled" to be used in fast paths such as for
602 * the irqsoff tracer. But it may be inaccurate due to races. If you
603 * need to know the accurate state, use tracing_is_on() which is a little
604 * slower, but accurate.
606 int tracing_is_enabled(void)
609 * For quick access (irqsoff uses this in fast path), just
610 * return the mirror variable of the state of the ring buffer.
611 * It's a little racy, but we don't really care.
614 return !global_trace.buffer_disabled;
618 * trace_buf_size is the size in bytes that is allocated
619 * for a buffer. Note, the number of bytes is always rounded
622 * This number is purposely set to a low number of 16384.
623 * If the dump on oops happens, it will be much appreciated
624 * to not have to wait for all that output. Anyway this can be
625 * boot time and run time configurable.
627 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
629 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
631 /* trace_types holds a link list of available tracers. */
632 static struct tracer *trace_types __read_mostly;
635 * trace_types_lock is used to protect the trace_types list.
637 DEFINE_MUTEX(trace_types_lock);
640 * serialize the access of the ring buffer
642 * ring buffer serializes readers, but it is low level protection.
643 * The validity of the events (which returns by ring_buffer_peek() ..etc)
644 * are not protected by ring buffer.
646 * The content of events may become garbage if we allow other process consumes
647 * these events concurrently:
648 * A) the page of the consumed events may become a normal page
649 * (not reader page) in ring buffer, and this page will be rewrited
650 * by events producer.
651 * B) The page of the consumed events may become a page for splice_read,
652 * and this page will be returned to system.
654 * These primitives allow multi process access to different cpu ring buffer
657 * These primitives don't distinguish read-only and read-consume access.
658 * Multi read-only access are also serialized.
662 static DECLARE_RWSEM(all_cpu_access_lock);
663 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
665 static inline void trace_access_lock(int cpu)
667 if (cpu == RING_BUFFER_ALL_CPUS) {
668 /* gain it for accessing the whole ring buffer. */
669 down_write(&all_cpu_access_lock);
671 /* gain it for accessing a cpu ring buffer. */
673 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
674 down_read(&all_cpu_access_lock);
676 /* Secondly block other access to this @cpu ring buffer. */
677 mutex_lock(&per_cpu(cpu_access_lock, cpu));
681 static inline void trace_access_unlock(int cpu)
683 if (cpu == RING_BUFFER_ALL_CPUS) {
684 up_write(&all_cpu_access_lock);
686 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
687 up_read(&all_cpu_access_lock);
691 static inline void trace_access_lock_init(void)
695 for_each_possible_cpu(cpu)
696 mutex_init(&per_cpu(cpu_access_lock, cpu));
701 static DEFINE_MUTEX(access_lock);
703 static inline void trace_access_lock(int cpu)
706 mutex_lock(&access_lock);
709 static inline void trace_access_unlock(int cpu)
712 mutex_unlock(&access_lock);
715 static inline void trace_access_lock_init(void)
721 #ifdef CONFIG_STACKTRACE
722 static void __ftrace_trace_stack(struct ring_buffer *buffer,
724 int skip, int pc, struct pt_regs *regs);
725 static inline void ftrace_trace_stack(struct trace_array *tr,
726 struct ring_buffer *buffer,
728 int skip, int pc, struct pt_regs *regs);
731 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
733 int skip, int pc, struct pt_regs *regs)
736 static inline void ftrace_trace_stack(struct trace_array *tr,
737 struct ring_buffer *buffer,
739 int skip, int pc, struct pt_regs *regs)
745 static void tracer_tracing_on(struct trace_array *tr)
747 if (tr->trace_buffer.buffer)
748 ring_buffer_record_on(tr->trace_buffer.buffer);
750 * This flag is looked at when buffers haven't been allocated
751 * yet, or by some tracers (like irqsoff), that just want to
752 * know if the ring buffer has been disabled, but it can handle
753 * races of where it gets disabled but we still do a record.
754 * As the check is in the fast path of the tracers, it is more
755 * important to be fast than accurate.
757 tr->buffer_disabled = 0;
758 /* Make the flag seen by readers */
763 * tracing_on - enable tracing buffers
765 * This function enables tracing buffers that may have been
766 * disabled with tracing_off.
768 void tracing_on(void)
770 tracer_tracing_on(&global_trace);
772 EXPORT_SYMBOL_GPL(tracing_on);
775 * __trace_puts - write a constant string into the trace buffer.
776 * @ip: The address of the caller
777 * @str: The constant string to write
778 * @size: The size of the string.
780 int __trace_puts(unsigned long ip, const char *str, int size)
782 struct ring_buffer_event *event;
783 struct ring_buffer *buffer;
784 struct print_entry *entry;
785 unsigned long irq_flags;
789 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
792 pc = preempt_count();
794 if (unlikely(tracing_selftest_running || tracing_disabled))
797 alloc = sizeof(*entry) + size + 2; /* possible \n added */
799 local_save_flags(irq_flags);
800 buffer = global_trace.trace_buffer.buffer;
801 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
806 entry = ring_buffer_event_data(event);
809 memcpy(&entry->buf, str, size);
811 /* Add a newline if necessary */
812 if (entry->buf[size - 1] != '\n') {
813 entry->buf[size] = '\n';
814 entry->buf[size + 1] = '\0';
816 entry->buf[size] = '\0';
818 __buffer_unlock_commit(buffer, event);
819 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
823 EXPORT_SYMBOL_GPL(__trace_puts);
826 * __trace_bputs - write the pointer to a constant string into trace buffer
827 * @ip: The address of the caller
828 * @str: The constant string to write to the buffer to
830 int __trace_bputs(unsigned long ip, const char *str)
832 struct ring_buffer_event *event;
833 struct ring_buffer *buffer;
834 struct bputs_entry *entry;
835 unsigned long irq_flags;
836 int size = sizeof(struct bputs_entry);
839 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
842 pc = preempt_count();
844 if (unlikely(tracing_selftest_running || tracing_disabled))
847 local_save_flags(irq_flags);
848 buffer = global_trace.trace_buffer.buffer;
849 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
854 entry = ring_buffer_event_data(event);
858 __buffer_unlock_commit(buffer, event);
859 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
863 EXPORT_SYMBOL_GPL(__trace_bputs);
865 #ifdef CONFIG_TRACER_SNAPSHOT
867 * trace_snapshot - take a snapshot of the current buffer.
869 * This causes a swap between the snapshot buffer and the current live
870 * tracing buffer. You can use this to take snapshots of the live
871 * trace when some condition is triggered, but continue to trace.
873 * Note, make sure to allocate the snapshot with either
874 * a tracing_snapshot_alloc(), or by doing it manually
875 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
877 * If the snapshot buffer is not allocated, it will stop tracing.
878 * Basically making a permanent snapshot.
880 void tracing_snapshot(void)
882 struct trace_array *tr = &global_trace;
883 struct tracer *tracer = tr->current_trace;
887 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
888 internal_trace_puts("*** snapshot is being ignored ***\n");
892 if (!tr->allocated_snapshot) {
893 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
894 internal_trace_puts("*** stopping trace here! ***\n");
899 /* Note, snapshot can not be used when the tracer uses it */
900 if (tracer->use_max_tr) {
901 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
902 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
906 local_irq_save(flags);
907 update_max_tr(tr, current, smp_processor_id());
908 local_irq_restore(flags);
910 EXPORT_SYMBOL_GPL(tracing_snapshot);
912 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
913 struct trace_buffer *size_buf, int cpu_id);
914 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
916 static int alloc_snapshot(struct trace_array *tr)
920 if (!tr->allocated_snapshot) {
922 /* allocate spare buffer */
923 ret = resize_buffer_duplicate_size(&tr->max_buffer,
924 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
928 tr->allocated_snapshot = true;
934 static void free_snapshot(struct trace_array *tr)
937 * We don't free the ring buffer. instead, resize it because
938 * The max_tr ring buffer has some state (e.g. ring->clock) and
939 * we want preserve it.
941 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
942 set_buffer_entries(&tr->max_buffer, 1);
943 tracing_reset_online_cpus(&tr->max_buffer);
944 tr->allocated_snapshot = false;
948 * tracing_alloc_snapshot - allocate snapshot buffer.
950 * This only allocates the snapshot buffer if it isn't already
951 * allocated - it doesn't also take a snapshot.
953 * This is meant to be used in cases where the snapshot buffer needs
954 * to be set up for events that can't sleep but need to be able to
955 * trigger a snapshot.
957 int tracing_alloc_snapshot(void)
959 struct trace_array *tr = &global_trace;
962 ret = alloc_snapshot(tr);
967 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
970 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
972 * This is similar to trace_snapshot(), but it will allocate the
973 * snapshot buffer if it isn't already allocated. Use this only
974 * where it is safe to sleep, as the allocation may sleep.
976 * This causes a swap between the snapshot buffer and the current live
977 * tracing buffer. You can use this to take snapshots of the live
978 * trace when some condition is triggered, but continue to trace.
980 void tracing_snapshot_alloc(void)
984 ret = tracing_alloc_snapshot();
990 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
992 void tracing_snapshot(void)
994 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
996 EXPORT_SYMBOL_GPL(tracing_snapshot);
997 int tracing_alloc_snapshot(void)
999 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1002 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1003 void tracing_snapshot_alloc(void)
1008 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1009 #endif /* CONFIG_TRACER_SNAPSHOT */
1011 static void tracer_tracing_off(struct trace_array *tr)
1013 if (tr->trace_buffer.buffer)
1014 ring_buffer_record_off(tr->trace_buffer.buffer);
1016 * This flag is looked at when buffers haven't been allocated
1017 * yet, or by some tracers (like irqsoff), that just want to
1018 * know if the ring buffer has been disabled, but it can handle
1019 * races of where it gets disabled but we still do a record.
1020 * As the check is in the fast path of the tracers, it is more
1021 * important to be fast than accurate.
1023 tr->buffer_disabled = 1;
1024 /* Make the flag seen by readers */
1029 * tracing_off - turn off tracing buffers
1031 * This function stops the tracing buffers from recording data.
1032 * It does not disable any overhead the tracers themselves may
1033 * be causing. This function simply causes all recording to
1034 * the ring buffers to fail.
1036 void tracing_off(void)
1038 tracer_tracing_off(&global_trace);
1040 EXPORT_SYMBOL_GPL(tracing_off);
1042 void disable_trace_on_warning(void)
1044 if (__disable_trace_on_warning)
1049 * tracer_tracing_is_on - show real state of ring buffer enabled
1050 * @tr : the trace array to know if ring buffer is enabled
1052 * Shows real state of the ring buffer if it is enabled or not.
1054 static int tracer_tracing_is_on(struct trace_array *tr)
1056 if (tr->trace_buffer.buffer)
1057 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1058 return !tr->buffer_disabled;
1062 * tracing_is_on - show state of ring buffers enabled
1064 int tracing_is_on(void)
1066 return tracer_tracing_is_on(&global_trace);
1068 EXPORT_SYMBOL_GPL(tracing_is_on);
1070 static int __init set_buf_size(char *str)
1072 unsigned long buf_size;
1076 buf_size = memparse(str, &str);
1077 /* nr_entries can not be zero */
1080 trace_buf_size = buf_size;
1083 __setup("trace_buf_size=", set_buf_size);
1085 static int __init set_tracing_thresh(char *str)
1087 unsigned long threshold;
1092 ret = kstrtoul(str, 0, &threshold);
1095 tracing_thresh = threshold * 1000;
1098 __setup("tracing_thresh=", set_tracing_thresh);
1100 unsigned long nsecs_to_usecs(unsigned long nsecs)
1102 return nsecs / 1000;
1106 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1107 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1108 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1109 * of strings in the order that the enums were defined.
1114 /* These must match the bit postions in trace_iterator_flags */
1115 static const char *trace_options[] = {
1123 int in_ns; /* is this clock in nanoseconds? */
1124 } trace_clocks[] = {
1125 { trace_clock_local, "local", 1 },
1126 { trace_clock_global, "global", 1 },
1127 { trace_clock_counter, "counter", 0 },
1128 { trace_clock_jiffies, "uptime", 0 },
1129 { trace_clock, "perf", 1 },
1130 { ktime_get_mono_fast_ns, "mono", 1 },
1131 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1136 * trace_parser_get_init - gets the buffer for trace parser
1138 int trace_parser_get_init(struct trace_parser *parser, int size)
1140 memset(parser, 0, sizeof(*parser));
1142 parser->buffer = kmalloc(size, GFP_KERNEL);
1143 if (!parser->buffer)
1146 parser->size = size;
1151 * trace_parser_put - frees the buffer for trace parser
1153 void trace_parser_put(struct trace_parser *parser)
1155 kfree(parser->buffer);
1159 * trace_get_user - reads the user input string separated by space
1160 * (matched by isspace(ch))
1162 * For each string found the 'struct trace_parser' is updated,
1163 * and the function returns.
1165 * Returns number of bytes read.
1167 * See kernel/trace/trace.h for 'struct trace_parser' details.
1169 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1170 size_t cnt, loff_t *ppos)
1177 trace_parser_clear(parser);
1179 ret = get_user(ch, ubuf++);
1187 * The parser is not finished with the last write,
1188 * continue reading the user input without skipping spaces.
1190 if (!parser->cont) {
1191 /* skip white space */
1192 while (cnt && isspace(ch)) {
1193 ret = get_user(ch, ubuf++);
1200 /* only spaces were written */
1210 /* read the non-space input */
1211 while (cnt && !isspace(ch)) {
1212 if (parser->idx < parser->size - 1)
1213 parser->buffer[parser->idx++] = ch;
1218 ret = get_user(ch, ubuf++);
1225 /* We either got finished input or we have to wait for another call. */
1227 parser->buffer[parser->idx] = 0;
1228 parser->cont = false;
1229 } else if (parser->idx < parser->size - 1) {
1230 parser->cont = true;
1231 parser->buffer[parser->idx++] = ch;
1244 /* TODO add a seq_buf_to_buffer() */
1245 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1249 if (trace_seq_used(s) <= s->seq.readpos)
1252 len = trace_seq_used(s) - s->seq.readpos;
1255 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1257 s->seq.readpos += cnt;
1261 unsigned long __read_mostly tracing_thresh;
1263 #ifdef CONFIG_TRACER_MAX_TRACE
1265 * Copy the new maximum trace into the separate maximum-trace
1266 * structure. (this way the maximum trace is permanently saved,
1267 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1270 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1272 struct trace_buffer *trace_buf = &tr->trace_buffer;
1273 struct trace_buffer *max_buf = &tr->max_buffer;
1274 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1275 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1278 max_buf->time_start = data->preempt_timestamp;
1280 max_data->saved_latency = tr->max_latency;
1281 max_data->critical_start = data->critical_start;
1282 max_data->critical_end = data->critical_end;
1284 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1285 max_data->pid = tsk->pid;
1287 * If tsk == current, then use current_uid(), as that does not use
1288 * RCU. The irq tracer can be called out of RCU scope.
1291 max_data->uid = current_uid();
1293 max_data->uid = task_uid(tsk);
1295 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1296 max_data->policy = tsk->policy;
1297 max_data->rt_priority = tsk->rt_priority;
1299 /* record this tasks comm */
1300 tracing_record_cmdline(tsk);
1304 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1306 * @tsk: the task with the latency
1307 * @cpu: The cpu that initiated the trace.
1309 * Flip the buffers between the @tr and the max_tr and record information
1310 * about which task was the cause of this latency.
1313 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1315 struct ring_buffer *buf;
1320 WARN_ON_ONCE(!irqs_disabled());
1322 if (!tr->allocated_snapshot) {
1323 /* Only the nop tracer should hit this when disabling */
1324 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1328 arch_spin_lock(&tr->max_lock);
1330 buf = tr->trace_buffer.buffer;
1331 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1332 tr->max_buffer.buffer = buf;
1334 __update_max_tr(tr, tsk, cpu);
1335 arch_spin_unlock(&tr->max_lock);
1339 * update_max_tr_single - only copy one trace over, and reset the rest
1341 * @tsk - task with the latency
1342 * @cpu - the cpu of the buffer to copy.
1344 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1347 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1354 WARN_ON_ONCE(!irqs_disabled());
1355 if (!tr->allocated_snapshot) {
1356 /* Only the nop tracer should hit this when disabling */
1357 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1361 arch_spin_lock(&tr->max_lock);
1363 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1365 if (ret == -EBUSY) {
1367 * We failed to swap the buffer due to a commit taking
1368 * place on this CPU. We fail to record, but we reset
1369 * the max trace buffer (no one writes directly to it)
1370 * and flag that it failed.
1372 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1373 "Failed to swap buffers due to commit in progress\n");
1376 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1378 __update_max_tr(tr, tsk, cpu);
1379 arch_spin_unlock(&tr->max_lock);
1381 #endif /* CONFIG_TRACER_MAX_TRACE */
1383 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1385 /* Iterators are static, they should be filled or empty */
1386 if (trace_buffer_iter(iter, iter->cpu_file))
1389 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1393 #ifdef CONFIG_FTRACE_STARTUP_TEST
1394 static int run_tracer_selftest(struct tracer *type)
1396 struct trace_array *tr = &global_trace;
1397 struct tracer *saved_tracer = tr->current_trace;
1400 if (!type->selftest || tracing_selftest_disabled)
1404 * Run a selftest on this tracer.
1405 * Here we reset the trace buffer, and set the current
1406 * tracer to be this tracer. The tracer can then run some
1407 * internal tracing to verify that everything is in order.
1408 * If we fail, we do not register this tracer.
1410 tracing_reset_online_cpus(&tr->trace_buffer);
1412 tr->current_trace = type;
1414 #ifdef CONFIG_TRACER_MAX_TRACE
1415 if (type->use_max_tr) {
1416 /* If we expanded the buffers, make sure the max is expanded too */
1417 if (ring_buffer_expanded)
1418 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1419 RING_BUFFER_ALL_CPUS);
1420 tr->allocated_snapshot = true;
1424 /* the test is responsible for initializing and enabling */
1425 pr_info("Testing tracer %s: ", type->name);
1426 ret = type->selftest(type, tr);
1427 /* the test is responsible for resetting too */
1428 tr->current_trace = saved_tracer;
1430 printk(KERN_CONT "FAILED!\n");
1431 /* Add the warning after printing 'FAILED' */
1435 /* Only reset on passing, to avoid touching corrupted buffers */
1436 tracing_reset_online_cpus(&tr->trace_buffer);
1438 #ifdef CONFIG_TRACER_MAX_TRACE
1439 if (type->use_max_tr) {
1440 tr->allocated_snapshot = false;
1442 /* Shrink the max buffer again */
1443 if (ring_buffer_expanded)
1444 ring_buffer_resize(tr->max_buffer.buffer, 1,
1445 RING_BUFFER_ALL_CPUS);
1449 printk(KERN_CONT "PASSED\n");
1453 static inline int run_tracer_selftest(struct tracer *type)
1457 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1459 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1461 static void __init apply_trace_boot_options(void);
1464 * register_tracer - register a tracer with the ftrace system.
1465 * @type - the plugin for the tracer
1467 * Register a new plugin tracer.
1469 int __init register_tracer(struct tracer *type)
1475 pr_info("Tracer must have a name\n");
1479 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1480 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1484 mutex_lock(&trace_types_lock);
1486 tracing_selftest_running = true;
1488 for (t = trace_types; t; t = t->next) {
1489 if (strcmp(type->name, t->name) == 0) {
1491 pr_info("Tracer %s already registered\n",
1498 if (!type->set_flag)
1499 type->set_flag = &dummy_set_flag;
1501 /*allocate a dummy tracer_flags*/
1502 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1507 type->flags->val = 0;
1508 type->flags->opts = dummy_tracer_opt;
1510 if (!type->flags->opts)
1511 type->flags->opts = dummy_tracer_opt;
1513 /* store the tracer for __set_tracer_option */
1514 type->flags->trace = type;
1516 ret = run_tracer_selftest(type);
1520 type->next = trace_types;
1522 add_tracer_options(&global_trace, type);
1525 tracing_selftest_running = false;
1526 mutex_unlock(&trace_types_lock);
1528 if (ret || !default_bootup_tracer)
1531 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1534 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1535 /* Do we want this tracer to start on bootup? */
1536 tracing_set_tracer(&global_trace, type->name);
1537 default_bootup_tracer = NULL;
1539 apply_trace_boot_options();
1541 /* disable other selftests, since this will break it. */
1542 tracing_selftest_disabled = true;
1543 #ifdef CONFIG_FTRACE_STARTUP_TEST
1544 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1552 void tracing_reset(struct trace_buffer *buf, int cpu)
1554 struct ring_buffer *buffer = buf->buffer;
1559 ring_buffer_record_disable(buffer);
1561 /* Make sure all commits have finished */
1562 synchronize_sched();
1563 ring_buffer_reset_cpu(buffer, cpu);
1565 ring_buffer_record_enable(buffer);
1568 void tracing_reset_online_cpus(struct trace_buffer *buf)
1570 struct ring_buffer *buffer = buf->buffer;
1576 ring_buffer_record_disable(buffer);
1578 /* Make sure all commits have finished */
1579 synchronize_sched();
1581 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1583 for_each_online_cpu(cpu)
1584 ring_buffer_reset_cpu(buffer, cpu);
1586 ring_buffer_record_enable(buffer);
1589 /* Must have trace_types_lock held */
1590 void tracing_reset_all_online_cpus(void)
1592 struct trace_array *tr;
1594 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1595 tracing_reset_online_cpus(&tr->trace_buffer);
1596 #ifdef CONFIG_TRACER_MAX_TRACE
1597 tracing_reset_online_cpus(&tr->max_buffer);
1602 #define SAVED_CMDLINES_DEFAULT 128
1603 #define NO_CMDLINE_MAP UINT_MAX
1604 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1605 struct saved_cmdlines_buffer {
1606 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1607 unsigned *map_cmdline_to_pid;
1608 unsigned cmdline_num;
1610 char *saved_cmdlines;
1612 static struct saved_cmdlines_buffer *savedcmd;
1614 /* temporary disable recording */
1615 static atomic_t trace_record_cmdline_disabled __read_mostly;
1617 static inline char *get_saved_cmdlines(int idx)
1619 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1622 static inline void set_cmdline(int idx, const char *cmdline)
1624 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1627 static int allocate_cmdlines_buffer(unsigned int val,
1628 struct saved_cmdlines_buffer *s)
1630 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1632 if (!s->map_cmdline_to_pid)
1635 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1636 if (!s->saved_cmdlines) {
1637 kfree(s->map_cmdline_to_pid);
1642 s->cmdline_num = val;
1643 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1644 sizeof(s->map_pid_to_cmdline));
1645 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1646 val * sizeof(*s->map_cmdline_to_pid));
1651 static int trace_create_savedcmd(void)
1655 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1659 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1669 int is_tracing_stopped(void)
1671 return global_trace.stop_count;
1675 * tracing_start - quick start of the tracer
1677 * If tracing is enabled but was stopped by tracing_stop,
1678 * this will start the tracer back up.
1680 void tracing_start(void)
1682 struct ring_buffer *buffer;
1683 unsigned long flags;
1685 if (tracing_disabled)
1688 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1689 if (--global_trace.stop_count) {
1690 if (global_trace.stop_count < 0) {
1691 /* Someone screwed up their debugging */
1693 global_trace.stop_count = 0;
1698 /* Prevent the buffers from switching */
1699 arch_spin_lock(&global_trace.max_lock);
1701 buffer = global_trace.trace_buffer.buffer;
1703 ring_buffer_record_enable(buffer);
1705 #ifdef CONFIG_TRACER_MAX_TRACE
1706 buffer = global_trace.max_buffer.buffer;
1708 ring_buffer_record_enable(buffer);
1711 arch_spin_unlock(&global_trace.max_lock);
1714 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1717 static void tracing_start_tr(struct trace_array *tr)
1719 struct ring_buffer *buffer;
1720 unsigned long flags;
1722 if (tracing_disabled)
1725 /* If global, we need to also start the max tracer */
1726 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1727 return tracing_start();
1729 raw_spin_lock_irqsave(&tr->start_lock, flags);
1731 if (--tr->stop_count) {
1732 if (tr->stop_count < 0) {
1733 /* Someone screwed up their debugging */
1740 buffer = tr->trace_buffer.buffer;
1742 ring_buffer_record_enable(buffer);
1745 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1749 * tracing_stop - quick stop of the tracer
1751 * Light weight way to stop tracing. Use in conjunction with
1754 void tracing_stop(void)
1756 struct ring_buffer *buffer;
1757 unsigned long flags;
1759 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1760 if (global_trace.stop_count++)
1763 /* Prevent the buffers from switching */
1764 arch_spin_lock(&global_trace.max_lock);
1766 buffer = global_trace.trace_buffer.buffer;
1768 ring_buffer_record_disable(buffer);
1770 #ifdef CONFIG_TRACER_MAX_TRACE
1771 buffer = global_trace.max_buffer.buffer;
1773 ring_buffer_record_disable(buffer);
1776 arch_spin_unlock(&global_trace.max_lock);
1779 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1782 static void tracing_stop_tr(struct trace_array *tr)
1784 struct ring_buffer *buffer;
1785 unsigned long flags;
1787 /* If global, we need to also stop the max tracer */
1788 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1789 return tracing_stop();
1791 raw_spin_lock_irqsave(&tr->start_lock, flags);
1792 if (tr->stop_count++)
1795 buffer = tr->trace_buffer.buffer;
1797 ring_buffer_record_disable(buffer);
1800 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1803 void trace_stop_cmdline_recording(void);
1805 static int trace_save_cmdline(struct task_struct *tsk)
1809 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1813 * It's not the end of the world if we don't get
1814 * the lock, but we also don't want to spin
1815 * nor do we want to disable interrupts,
1816 * so if we miss here, then better luck next time.
1818 if (!arch_spin_trylock(&trace_cmdline_lock))
1821 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1822 if (idx == NO_CMDLINE_MAP) {
1823 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1826 * Check whether the cmdline buffer at idx has a pid
1827 * mapped. We are going to overwrite that entry so we
1828 * need to clear the map_pid_to_cmdline. Otherwise we
1829 * would read the new comm for the old pid.
1831 pid = savedcmd->map_cmdline_to_pid[idx];
1832 if (pid != NO_CMDLINE_MAP)
1833 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1835 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1836 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1838 savedcmd->cmdline_idx = idx;
1841 set_cmdline(idx, tsk->comm);
1843 arch_spin_unlock(&trace_cmdline_lock);
1848 static void __trace_find_cmdline(int pid, char comm[])
1853 strcpy(comm, "<idle>");
1857 if (WARN_ON_ONCE(pid < 0)) {
1858 strcpy(comm, "<XXX>");
1862 if (pid > PID_MAX_DEFAULT) {
1863 strcpy(comm, "<...>");
1867 map = savedcmd->map_pid_to_cmdline[pid];
1868 if (map != NO_CMDLINE_MAP)
1869 strcpy(comm, get_saved_cmdlines(map));
1871 strcpy(comm, "<...>");
1874 void trace_find_cmdline(int pid, char comm[])
1877 arch_spin_lock(&trace_cmdline_lock);
1879 __trace_find_cmdline(pid, comm);
1881 arch_spin_unlock(&trace_cmdline_lock);
1885 void tracing_record_cmdline(struct task_struct *tsk)
1887 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1890 if (!__this_cpu_read(trace_cmdline_save))
1893 if (trace_save_cmdline(tsk))
1894 __this_cpu_write(trace_cmdline_save, false);
1898 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1901 struct task_struct *tsk = current;
1903 entry->preempt_count = pc & 0xff;
1904 entry->pid = (tsk) ? tsk->pid : 0;
1906 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1907 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1909 TRACE_FLAG_IRQS_NOSUPPORT |
1911 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
1912 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1913 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1914 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1915 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1917 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1919 static __always_inline void
1920 trace_event_setup(struct ring_buffer_event *event,
1921 int type, unsigned long flags, int pc)
1923 struct trace_entry *ent = ring_buffer_event_data(event);
1925 tracing_generic_entry_update(ent, flags, pc);
1929 struct ring_buffer_event *
1930 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1933 unsigned long flags, int pc)
1935 struct ring_buffer_event *event;
1937 event = ring_buffer_lock_reserve(buffer, len);
1939 trace_event_setup(event, type, flags, pc);
1944 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1945 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1946 static int trace_buffered_event_ref;
1949 * trace_buffered_event_enable - enable buffering events
1951 * When events are being filtered, it is quicker to use a temporary
1952 * buffer to write the event data into if there's a likely chance
1953 * that it will not be committed. The discard of the ring buffer
1954 * is not as fast as committing, and is much slower than copying
1957 * When an event is to be filtered, allocate per cpu buffers to
1958 * write the event data into, and if the event is filtered and discarded
1959 * it is simply dropped, otherwise, the entire data is to be committed
1962 void trace_buffered_event_enable(void)
1964 struct ring_buffer_event *event;
1968 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1970 if (trace_buffered_event_ref++)
1973 for_each_tracing_cpu(cpu) {
1974 page = alloc_pages_node(cpu_to_node(cpu),
1975 GFP_KERNEL | __GFP_NORETRY, 0);
1979 event = page_address(page);
1980 memset(event, 0, sizeof(*event));
1982 per_cpu(trace_buffered_event, cpu) = event;
1985 if (cpu == smp_processor_id() &&
1986 this_cpu_read(trace_buffered_event) !=
1987 per_cpu(trace_buffered_event, cpu))
1994 trace_buffered_event_disable();
1997 static void enable_trace_buffered_event(void *data)
1999 /* Probably not needed, but do it anyway */
2001 this_cpu_dec(trace_buffered_event_cnt);
2004 static void disable_trace_buffered_event(void *data)
2006 this_cpu_inc(trace_buffered_event_cnt);
2010 * trace_buffered_event_disable - disable buffering events
2012 * When a filter is removed, it is faster to not use the buffered
2013 * events, and to commit directly into the ring buffer. Free up
2014 * the temp buffers when there are no more users. This requires
2015 * special synchronization with current events.
2017 void trace_buffered_event_disable(void)
2021 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2023 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2026 if (--trace_buffered_event_ref)
2030 /* For each CPU, set the buffer as used. */
2031 smp_call_function_many(tracing_buffer_mask,
2032 disable_trace_buffered_event, NULL, 1);
2035 /* Wait for all current users to finish */
2036 synchronize_sched();
2038 for_each_tracing_cpu(cpu) {
2039 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2040 per_cpu(trace_buffered_event, cpu) = NULL;
2043 * Make sure trace_buffered_event is NULL before clearing
2044 * trace_buffered_event_cnt.
2049 /* Do the work on each cpu */
2050 smp_call_function_many(tracing_buffer_mask,
2051 enable_trace_buffered_event, NULL, 1);
2056 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
2058 __this_cpu_write(trace_cmdline_save, true);
2060 /* If this is the temp buffer, we need to commit fully */
2061 if (this_cpu_read(trace_buffered_event) == event) {
2062 /* Length is in event->array[0] */
2063 ring_buffer_write(buffer, event->array[0], &event->array[1]);
2064 /* Release the temp buffer */
2065 this_cpu_dec(trace_buffered_event_cnt);
2067 ring_buffer_unlock_commit(buffer, event);
2070 static struct ring_buffer *temp_buffer;
2072 struct ring_buffer_event *
2073 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2074 struct trace_event_file *trace_file,
2075 int type, unsigned long len,
2076 unsigned long flags, int pc)
2078 struct ring_buffer_event *entry;
2081 *current_rb = trace_file->tr->trace_buffer.buffer;
2083 if ((trace_file->flags &
2084 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2085 (entry = this_cpu_read(trace_buffered_event))) {
2086 /* Try to use the per cpu buffer first */
2087 val = this_cpu_inc_return(trace_buffered_event_cnt);
2089 trace_event_setup(entry, type, flags, pc);
2090 entry->array[0] = len;
2093 this_cpu_dec(trace_buffered_event_cnt);
2096 entry = trace_buffer_lock_reserve(*current_rb,
2097 type, len, flags, pc);
2099 * If tracing is off, but we have triggers enabled
2100 * we still need to look at the event data. Use the temp_buffer
2101 * to store the trace event for the tigger to use. It's recusive
2102 * safe and will not be recorded anywhere.
2104 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2105 *current_rb = temp_buffer;
2106 entry = trace_buffer_lock_reserve(*current_rb,
2107 type, len, flags, pc);
2111 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2113 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2114 struct ring_buffer *buffer,
2115 struct ring_buffer_event *event,
2116 unsigned long flags, int pc,
2117 struct pt_regs *regs)
2119 __buffer_unlock_commit(buffer, event);
2121 ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
2122 ftrace_trace_userstack(buffer, flags, pc);
2126 trace_function(struct trace_array *tr,
2127 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2130 struct trace_event_call *call = &event_function;
2131 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2132 struct ring_buffer_event *event;
2133 struct ftrace_entry *entry;
2135 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2139 entry = ring_buffer_event_data(event);
2141 entry->parent_ip = parent_ip;
2143 if (!call_filter_check_discard(call, entry, buffer, event))
2144 __buffer_unlock_commit(buffer, event);
2147 #ifdef CONFIG_STACKTRACE
2149 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2150 struct ftrace_stack {
2151 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2154 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2155 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2157 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2158 unsigned long flags,
2159 int skip, int pc, struct pt_regs *regs)
2161 struct trace_event_call *call = &event_kernel_stack;
2162 struct ring_buffer_event *event;
2163 struct stack_entry *entry;
2164 struct stack_trace trace;
2166 int size = FTRACE_STACK_ENTRIES;
2168 trace.nr_entries = 0;
2172 * Since events can happen in NMIs there's no safe way to
2173 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2174 * or NMI comes in, it will just have to use the default
2175 * FTRACE_STACK_SIZE.
2177 preempt_disable_notrace();
2179 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2181 * We don't need any atomic variables, just a barrier.
2182 * If an interrupt comes in, we don't care, because it would
2183 * have exited and put the counter back to what we want.
2184 * We just need a barrier to keep gcc from moving things
2188 if (use_stack == 1) {
2189 trace.entries = this_cpu_ptr(ftrace_stack.calls);
2190 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2193 save_stack_trace_regs(regs, &trace);
2195 save_stack_trace(&trace);
2197 if (trace.nr_entries > size)
2198 size = trace.nr_entries;
2200 /* From now on, use_stack is a boolean */
2203 size *= sizeof(unsigned long);
2205 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
2206 sizeof(*entry) + size, flags, pc);
2209 entry = ring_buffer_event_data(event);
2211 memset(&entry->caller, 0, size);
2214 memcpy(&entry->caller, trace.entries,
2215 trace.nr_entries * sizeof(unsigned long));
2217 trace.max_entries = FTRACE_STACK_ENTRIES;
2218 trace.entries = entry->caller;
2220 save_stack_trace_regs(regs, &trace);
2222 save_stack_trace(&trace);
2225 entry->size = trace.nr_entries;
2227 if (!call_filter_check_discard(call, entry, buffer, event))
2228 __buffer_unlock_commit(buffer, event);
2231 /* Again, don't let gcc optimize things here */
2233 __this_cpu_dec(ftrace_stack_reserve);
2234 preempt_enable_notrace();
2238 static inline void ftrace_trace_stack(struct trace_array *tr,
2239 struct ring_buffer *buffer,
2240 unsigned long flags,
2241 int skip, int pc, struct pt_regs *regs)
2243 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2246 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2249 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2252 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
2256 * trace_dump_stack - record a stack back trace in the trace buffer
2257 * @skip: Number of functions to skip (helper handlers)
2259 void trace_dump_stack(int skip)
2261 unsigned long flags;
2263 if (tracing_disabled || tracing_selftest_running)
2266 local_save_flags(flags);
2269 * Skip 3 more, seems to get us at the caller of
2273 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2274 flags, skip, preempt_count(), NULL);
2277 static DEFINE_PER_CPU(int, user_stack_count);
2280 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2282 struct trace_event_call *call = &event_user_stack;
2283 struct ring_buffer_event *event;
2284 struct userstack_entry *entry;
2285 struct stack_trace trace;
2287 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2291 * NMIs can not handle page faults, even with fix ups.
2292 * The save user stack can (and often does) fault.
2294 if (unlikely(in_nmi()))
2298 * prevent recursion, since the user stack tracing may
2299 * trigger other kernel events.
2302 if (__this_cpu_read(user_stack_count))
2305 __this_cpu_inc(user_stack_count);
2307 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2308 sizeof(*entry), flags, pc);
2310 goto out_drop_count;
2311 entry = ring_buffer_event_data(event);
2313 entry->tgid = current->tgid;
2314 memset(&entry->caller, 0, sizeof(entry->caller));
2316 trace.nr_entries = 0;
2317 trace.max_entries = FTRACE_STACK_ENTRIES;
2319 trace.entries = entry->caller;
2321 save_stack_trace_user(&trace);
2322 if (!call_filter_check_discard(call, entry, buffer, event))
2323 __buffer_unlock_commit(buffer, event);
2326 __this_cpu_dec(user_stack_count);
2332 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2334 ftrace_trace_userstack(tr, flags, preempt_count());
2338 #endif /* CONFIG_STACKTRACE */
2340 /* created for use with alloc_percpu */
2341 struct trace_buffer_struct {
2342 char buffer[TRACE_BUF_SIZE];
2345 static struct trace_buffer_struct *trace_percpu_buffer;
2346 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
2347 static struct trace_buffer_struct *trace_percpu_irq_buffer;
2348 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
2351 * The buffer used is dependent on the context. There is a per cpu
2352 * buffer for normal context, softirq contex, hard irq context and
2353 * for NMI context. Thise allows for lockless recording.
2355 * Note, if the buffers failed to be allocated, then this returns NULL
2357 static char *get_trace_buf(void)
2359 struct trace_buffer_struct *percpu_buffer;
2362 * If we have allocated per cpu buffers, then we do not
2363 * need to do any locking.
2366 percpu_buffer = trace_percpu_nmi_buffer;
2368 percpu_buffer = trace_percpu_irq_buffer;
2369 else if (in_softirq())
2370 percpu_buffer = trace_percpu_sirq_buffer;
2372 percpu_buffer = trace_percpu_buffer;
2377 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2380 static int alloc_percpu_trace_buffer(void)
2382 struct trace_buffer_struct *buffers;
2383 struct trace_buffer_struct *sirq_buffers;
2384 struct trace_buffer_struct *irq_buffers;
2385 struct trace_buffer_struct *nmi_buffers;
2387 buffers = alloc_percpu(struct trace_buffer_struct);
2391 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2395 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2399 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2403 trace_percpu_buffer = buffers;
2404 trace_percpu_sirq_buffer = sirq_buffers;
2405 trace_percpu_irq_buffer = irq_buffers;
2406 trace_percpu_nmi_buffer = nmi_buffers;
2411 free_percpu(irq_buffers);
2413 free_percpu(sirq_buffers);
2415 free_percpu(buffers);
2417 WARN(1, "Could not allocate percpu trace_printk buffer");
2421 static int buffers_allocated;
2423 void trace_printk_init_buffers(void)
2425 if (buffers_allocated)
2428 if (alloc_percpu_trace_buffer())
2431 /* trace_printk() is for debug use only. Don't use it in production. */
2434 pr_warn("**********************************************************\n");
2435 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2437 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2439 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2440 pr_warn("** unsafe for production use. **\n");
2442 pr_warn("** If you see this message and you are not debugging **\n");
2443 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2445 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2446 pr_warn("**********************************************************\n");
2448 /* Expand the buffers to set size */
2449 tracing_update_buffers();
2451 buffers_allocated = 1;
2454 * trace_printk_init_buffers() can be called by modules.
2455 * If that happens, then we need to start cmdline recording
2456 * directly here. If the global_trace.buffer is already
2457 * allocated here, then this was called by module code.
2459 if (global_trace.trace_buffer.buffer)
2460 tracing_start_cmdline_record();
2463 void trace_printk_start_comm(void)
2465 /* Start tracing comms if trace printk is set */
2466 if (!buffers_allocated)
2468 tracing_start_cmdline_record();
2471 static void trace_printk_start_stop_comm(int enabled)
2473 if (!buffers_allocated)
2477 tracing_start_cmdline_record();
2479 tracing_stop_cmdline_record();
2483 * trace_vbprintk - write binary msg to tracing buffer
2486 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2488 struct trace_event_call *call = &event_bprint;
2489 struct ring_buffer_event *event;
2490 struct ring_buffer *buffer;
2491 struct trace_array *tr = &global_trace;
2492 struct bprint_entry *entry;
2493 unsigned long flags;
2495 int len = 0, size, pc;
2497 if (unlikely(tracing_selftest_running || tracing_disabled))
2500 /* Don't pollute graph traces with trace_vprintk internals */
2501 pause_graph_tracing();
2503 pc = preempt_count();
2504 preempt_disable_notrace();
2506 tbuffer = get_trace_buf();
2512 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2514 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2517 local_save_flags(flags);
2518 size = sizeof(*entry) + sizeof(u32) * len;
2519 buffer = tr->trace_buffer.buffer;
2520 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2524 entry = ring_buffer_event_data(event);
2528 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2529 if (!call_filter_check_discard(call, entry, buffer, event)) {
2530 __buffer_unlock_commit(buffer, event);
2531 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2535 preempt_enable_notrace();
2536 unpause_graph_tracing();
2540 EXPORT_SYMBOL_GPL(trace_vbprintk);
2543 __trace_array_vprintk(struct ring_buffer *buffer,
2544 unsigned long ip, const char *fmt, va_list args)
2546 struct trace_event_call *call = &event_print;
2547 struct ring_buffer_event *event;
2548 int len = 0, size, pc;
2549 struct print_entry *entry;
2550 unsigned long flags;
2553 if (tracing_disabled || tracing_selftest_running)
2556 /* Don't pollute graph traces with trace_vprintk internals */
2557 pause_graph_tracing();
2559 pc = preempt_count();
2560 preempt_disable_notrace();
2563 tbuffer = get_trace_buf();
2569 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2571 local_save_flags(flags);
2572 size = sizeof(*entry) + len + 1;
2573 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2577 entry = ring_buffer_event_data(event);
2580 memcpy(&entry->buf, tbuffer, len + 1);
2581 if (!call_filter_check_discard(call, entry, buffer, event)) {
2582 __buffer_unlock_commit(buffer, event);
2583 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2586 preempt_enable_notrace();
2587 unpause_graph_tracing();
2592 int trace_array_vprintk(struct trace_array *tr,
2593 unsigned long ip, const char *fmt, va_list args)
2595 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2598 int trace_array_printk(struct trace_array *tr,
2599 unsigned long ip, const char *fmt, ...)
2604 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2608 ret = trace_array_vprintk(tr, ip, fmt, ap);
2613 int trace_array_printk_buf(struct ring_buffer *buffer,
2614 unsigned long ip, const char *fmt, ...)
2619 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2623 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2628 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2630 return trace_array_vprintk(&global_trace, ip, fmt, args);
2632 EXPORT_SYMBOL_GPL(trace_vprintk);
2634 static void trace_iterator_increment(struct trace_iterator *iter)
2636 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2640 ring_buffer_read(buf_iter, NULL);
2643 static struct trace_entry *
2644 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2645 unsigned long *lost_events)
2647 struct ring_buffer_event *event;
2648 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2651 event = ring_buffer_iter_peek(buf_iter, ts);
2653 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2657 iter->ent_size = ring_buffer_event_length(event);
2658 return ring_buffer_event_data(event);
2664 static struct trace_entry *
2665 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2666 unsigned long *missing_events, u64 *ent_ts)
2668 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2669 struct trace_entry *ent, *next = NULL;
2670 unsigned long lost_events = 0, next_lost = 0;
2671 int cpu_file = iter->cpu_file;
2672 u64 next_ts = 0, ts;
2678 * If we are in a per_cpu trace file, don't bother by iterating over
2679 * all cpu and peek directly.
2681 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2682 if (ring_buffer_empty_cpu(buffer, cpu_file))
2684 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2686 *ent_cpu = cpu_file;
2691 for_each_tracing_cpu(cpu) {
2693 if (ring_buffer_empty_cpu(buffer, cpu))
2696 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2699 * Pick the entry with the smallest timestamp:
2701 if (ent && (!next || ts < next_ts)) {
2705 next_lost = lost_events;
2706 next_size = iter->ent_size;
2710 iter->ent_size = next_size;
2713 *ent_cpu = next_cpu;
2719 *missing_events = next_lost;
2724 /* Find the next real entry, without updating the iterator itself */
2725 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2726 int *ent_cpu, u64 *ent_ts)
2728 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2731 /* Find the next real entry, and increment the iterator to the next entry */
2732 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2734 iter->ent = __find_next_entry(iter, &iter->cpu,
2735 &iter->lost_events, &iter->ts);
2738 trace_iterator_increment(iter);
2740 return iter->ent ? iter : NULL;
2743 static void trace_consume(struct trace_iterator *iter)
2745 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2746 &iter->lost_events);
2749 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2751 struct trace_iterator *iter = m->private;
2755 WARN_ON_ONCE(iter->leftover);
2759 /* can't go backwards */
2764 ent = trace_find_next_entry_inc(iter);
2768 while (ent && iter->idx < i)
2769 ent = trace_find_next_entry_inc(iter);
2776 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2778 struct ring_buffer_event *event;
2779 struct ring_buffer_iter *buf_iter;
2780 unsigned long entries = 0;
2783 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2785 buf_iter = trace_buffer_iter(iter, cpu);
2789 ring_buffer_iter_reset(buf_iter);
2792 * We could have the case with the max latency tracers
2793 * that a reset never took place on a cpu. This is evident
2794 * by the timestamp being before the start of the buffer.
2796 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2797 if (ts >= iter->trace_buffer->time_start)
2800 ring_buffer_read(buf_iter, NULL);
2803 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2807 * The current tracer is copied to avoid a global locking
2810 static void *s_start(struct seq_file *m, loff_t *pos)
2812 struct trace_iterator *iter = m->private;
2813 struct trace_array *tr = iter->tr;
2814 int cpu_file = iter->cpu_file;
2820 * copy the tracer to avoid using a global lock all around.
2821 * iter->trace is a copy of current_trace, the pointer to the
2822 * name may be used instead of a strcmp(), as iter->trace->name
2823 * will point to the same string as current_trace->name.
2825 mutex_lock(&trace_types_lock);
2826 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2827 *iter->trace = *tr->current_trace;
2828 mutex_unlock(&trace_types_lock);
2830 #ifdef CONFIG_TRACER_MAX_TRACE
2831 if (iter->snapshot && iter->trace->use_max_tr)
2832 return ERR_PTR(-EBUSY);
2835 if (!iter->snapshot)
2836 atomic_inc(&trace_record_cmdline_disabled);
2838 if (*pos != iter->pos) {
2843 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2844 for_each_tracing_cpu(cpu)
2845 tracing_iter_reset(iter, cpu);
2847 tracing_iter_reset(iter, cpu_file);
2850 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2855 * If we overflowed the seq_file before, then we want
2856 * to just reuse the trace_seq buffer again.
2862 p = s_next(m, p, &l);
2866 trace_event_read_lock();
2867 trace_access_lock(cpu_file);
2871 static void s_stop(struct seq_file *m, void *p)
2873 struct trace_iterator *iter = m->private;
2875 #ifdef CONFIG_TRACER_MAX_TRACE
2876 if (iter->snapshot && iter->trace->use_max_tr)
2880 if (!iter->snapshot)
2881 atomic_dec(&trace_record_cmdline_disabled);
2883 trace_access_unlock(iter->cpu_file);
2884 trace_event_read_unlock();
2888 get_total_entries(struct trace_buffer *buf,
2889 unsigned long *total, unsigned long *entries)
2891 unsigned long count;
2897 for_each_tracing_cpu(cpu) {
2898 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2900 * If this buffer has skipped entries, then we hold all
2901 * entries for the trace and we need to ignore the
2902 * ones before the time stamp.
2904 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2905 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2906 /* total is the same as the entries */
2910 ring_buffer_overrun_cpu(buf->buffer, cpu);
2915 static void print_lat_help_header(struct seq_file *m)
2917 seq_puts(m, "# _------=> CPU# \n"
2918 "# / _-----=> irqs-off \n"
2919 "# | / _----=> need-resched \n"
2920 "# || / _---=> hardirq/softirq \n"
2921 "# ||| / _--=> preempt-depth \n"
2923 "# cmd pid ||||| time | caller \n"
2924 "# \\ / ||||| \\ | / \n");
2927 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2929 unsigned long total;
2930 unsigned long entries;
2932 get_total_entries(buf, &total, &entries);
2933 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2934 entries, total, num_online_cpus());
2938 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2940 print_event_info(buf, m);
2941 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2945 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2947 print_event_info(buf, m);
2948 seq_puts(m, "# _-----=> irqs-off\n"
2949 "# / _----=> need-resched\n"
2950 "# | / _---=> hardirq/softirq\n"
2951 "# || / _--=> preempt-depth\n"
2953 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2954 "# | | | |||| | |\n");
2958 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2960 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2961 struct trace_buffer *buf = iter->trace_buffer;
2962 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2963 struct tracer *type = iter->trace;
2964 unsigned long entries;
2965 unsigned long total;
2966 const char *name = "preemption";
2970 get_total_entries(buf, &total, &entries);
2972 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2974 seq_puts(m, "# -----------------------------------"
2975 "---------------------------------\n");
2976 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2977 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2978 nsecs_to_usecs(data->saved_latency),
2982 #if defined(CONFIG_PREEMPT_NONE)
2984 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2986 #elif defined(CONFIG_PREEMPT)
2991 /* These are reserved for later use */
2994 seq_printf(m, " #P:%d)\n", num_online_cpus());
2998 seq_puts(m, "# -----------------\n");
2999 seq_printf(m, "# | task: %.16s-%d "
3000 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3001 data->comm, data->pid,
3002 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3003 data->policy, data->rt_priority);
3004 seq_puts(m, "# -----------------\n");
3006 if (data->critical_start) {
3007 seq_puts(m, "# => started at: ");
3008 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3009 trace_print_seq(m, &iter->seq);
3010 seq_puts(m, "\n# => ended at: ");
3011 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3012 trace_print_seq(m, &iter->seq);
3013 seq_puts(m, "\n#\n");
3019 static void test_cpu_buff_start(struct trace_iterator *iter)
3021 struct trace_seq *s = &iter->seq;
3022 struct trace_array *tr = iter->tr;
3024 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3027 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3030 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
3033 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3037 cpumask_set_cpu(iter->cpu, iter->started);
3039 /* Don't print started cpu buffer for the first entry of the trace */
3041 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3045 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3047 struct trace_array *tr = iter->tr;
3048 struct trace_seq *s = &iter->seq;
3049 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3050 struct trace_entry *entry;
3051 struct trace_event *event;
3055 test_cpu_buff_start(iter);
3057 event = ftrace_find_event(entry->type);
3059 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3060 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3061 trace_print_lat_context(iter);
3063 trace_print_context(iter);
3066 if (trace_seq_has_overflowed(s))
3067 return TRACE_TYPE_PARTIAL_LINE;
3070 return event->funcs->trace(iter, sym_flags, event);
3072 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3074 return trace_handle_return(s);
3077 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3079 struct trace_array *tr = iter->tr;
3080 struct trace_seq *s = &iter->seq;
3081 struct trace_entry *entry;
3082 struct trace_event *event;
3086 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3087 trace_seq_printf(s, "%d %d %llu ",
3088 entry->pid, iter->cpu, iter->ts);
3090 if (trace_seq_has_overflowed(s))
3091 return TRACE_TYPE_PARTIAL_LINE;
3093 event = ftrace_find_event(entry->type);
3095 return event->funcs->raw(iter, 0, event);
3097 trace_seq_printf(s, "%d ?\n", entry->type);
3099 return trace_handle_return(s);
3102 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3104 struct trace_array *tr = iter->tr;
3105 struct trace_seq *s = &iter->seq;
3106 unsigned char newline = '\n';
3107 struct trace_entry *entry;
3108 struct trace_event *event;
3112 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3113 SEQ_PUT_HEX_FIELD(s, entry->pid);
3114 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3115 SEQ_PUT_HEX_FIELD(s, iter->ts);
3116 if (trace_seq_has_overflowed(s))
3117 return TRACE_TYPE_PARTIAL_LINE;
3120 event = ftrace_find_event(entry->type);
3122 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3123 if (ret != TRACE_TYPE_HANDLED)
3127 SEQ_PUT_FIELD(s, newline);
3129 return trace_handle_return(s);
3132 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3134 struct trace_array *tr = iter->tr;
3135 struct trace_seq *s = &iter->seq;
3136 struct trace_entry *entry;
3137 struct trace_event *event;
3141 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3142 SEQ_PUT_FIELD(s, entry->pid);
3143 SEQ_PUT_FIELD(s, iter->cpu);
3144 SEQ_PUT_FIELD(s, iter->ts);
3145 if (trace_seq_has_overflowed(s))
3146 return TRACE_TYPE_PARTIAL_LINE;
3149 event = ftrace_find_event(entry->type);
3150 return event ? event->funcs->binary(iter, 0, event) :
3154 int trace_empty(struct trace_iterator *iter)
3156 struct ring_buffer_iter *buf_iter;
3159 /* If we are looking at one CPU buffer, only check that one */
3160 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3161 cpu = iter->cpu_file;
3162 buf_iter = trace_buffer_iter(iter, cpu);
3164 if (!ring_buffer_iter_empty(buf_iter))
3167 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3173 for_each_tracing_cpu(cpu) {
3174 buf_iter = trace_buffer_iter(iter, cpu);
3176 if (!ring_buffer_iter_empty(buf_iter))
3179 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3187 /* Called with trace_event_read_lock() held. */
3188 enum print_line_t print_trace_line(struct trace_iterator *iter)
3190 struct trace_array *tr = iter->tr;
3191 unsigned long trace_flags = tr->trace_flags;
3192 enum print_line_t ret;
3194 if (iter->lost_events) {
3195 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3196 iter->cpu, iter->lost_events);
3197 if (trace_seq_has_overflowed(&iter->seq))
3198 return TRACE_TYPE_PARTIAL_LINE;
3201 if (iter->trace && iter->trace->print_line) {
3202 ret = iter->trace->print_line(iter);
3203 if (ret != TRACE_TYPE_UNHANDLED)
3207 if (iter->ent->type == TRACE_BPUTS &&
3208 trace_flags & TRACE_ITER_PRINTK &&
3209 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3210 return trace_print_bputs_msg_only(iter);
3212 if (iter->ent->type == TRACE_BPRINT &&
3213 trace_flags & TRACE_ITER_PRINTK &&
3214 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3215 return trace_print_bprintk_msg_only(iter);
3217 if (iter->ent->type == TRACE_PRINT &&
3218 trace_flags & TRACE_ITER_PRINTK &&
3219 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3220 return trace_print_printk_msg_only(iter);
3222 if (trace_flags & TRACE_ITER_BIN)
3223 return print_bin_fmt(iter);
3225 if (trace_flags & TRACE_ITER_HEX)
3226 return print_hex_fmt(iter);
3228 if (trace_flags & TRACE_ITER_RAW)
3229 return print_raw_fmt(iter);
3231 return print_trace_fmt(iter);
3234 void trace_latency_header(struct seq_file *m)
3236 struct trace_iterator *iter = m->private;
3237 struct trace_array *tr = iter->tr;
3239 /* print nothing if the buffers are empty */
3240 if (trace_empty(iter))
3243 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3244 print_trace_header(m, iter);
3246 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3247 print_lat_help_header(m);
3250 void trace_default_header(struct seq_file *m)
3252 struct trace_iterator *iter = m->private;
3253 struct trace_array *tr = iter->tr;
3254 unsigned long trace_flags = tr->trace_flags;
3256 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3259 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3260 /* print nothing if the buffers are empty */
3261 if (trace_empty(iter))
3263 print_trace_header(m, iter);
3264 if (!(trace_flags & TRACE_ITER_VERBOSE))
3265 print_lat_help_header(m);
3267 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3268 if (trace_flags & TRACE_ITER_IRQ_INFO)
3269 print_func_help_header_irq(iter->trace_buffer, m);
3271 print_func_help_header(iter->trace_buffer, m);
3276 static void test_ftrace_alive(struct seq_file *m)
3278 if (!ftrace_is_dead())
3280 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3281 "# MAY BE MISSING FUNCTION EVENTS\n");
3284 #ifdef CONFIG_TRACER_MAX_TRACE
3285 static void show_snapshot_main_help(struct seq_file *m)
3287 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3288 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3289 "# Takes a snapshot of the main buffer.\n"
3290 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3291 "# (Doesn't have to be '2' works with any number that\n"
3292 "# is not a '0' or '1')\n");
3295 static void show_snapshot_percpu_help(struct seq_file *m)
3297 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3298 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3299 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3300 "# Takes a snapshot of the main buffer for this cpu.\n");
3302 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3303 "# Must use main snapshot file to allocate.\n");
3305 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3306 "# (Doesn't have to be '2' works with any number that\n"
3307 "# is not a '0' or '1')\n");
3310 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3312 if (iter->tr->allocated_snapshot)
3313 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3315 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3317 seq_puts(m, "# Snapshot commands:\n");
3318 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3319 show_snapshot_main_help(m);
3321 show_snapshot_percpu_help(m);
3324 /* Should never be called */
3325 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3328 static int s_show(struct seq_file *m, void *v)
3330 struct trace_iterator *iter = v;
3333 if (iter->ent == NULL) {
3335 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3337 test_ftrace_alive(m);
3339 if (iter->snapshot && trace_empty(iter))
3340 print_snapshot_help(m, iter);
3341 else if (iter->trace && iter->trace->print_header)
3342 iter->trace->print_header(m);
3344 trace_default_header(m);
3346 } else if (iter->leftover) {
3348 * If we filled the seq_file buffer earlier, we
3349 * want to just show it now.
3351 ret = trace_print_seq(m, &iter->seq);
3353 /* ret should this time be zero, but you never know */
3354 iter->leftover = ret;
3357 print_trace_line(iter);
3358 ret = trace_print_seq(m, &iter->seq);
3360 * If we overflow the seq_file buffer, then it will
3361 * ask us for this data again at start up.
3363 * ret is 0 if seq_file write succeeded.
3366 iter->leftover = ret;
3373 * Should be used after trace_array_get(), trace_types_lock
3374 * ensures that i_cdev was already initialized.
3376 static inline int tracing_get_cpu(struct inode *inode)
3378 if (inode->i_cdev) /* See trace_create_cpu_file() */
3379 return (long)inode->i_cdev - 1;
3380 return RING_BUFFER_ALL_CPUS;
3383 static const struct seq_operations tracer_seq_ops = {
3390 static struct trace_iterator *
3391 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3393 struct trace_array *tr = inode->i_private;
3394 struct trace_iterator *iter;
3397 if (tracing_disabled)
3398 return ERR_PTR(-ENODEV);
3400 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3402 return ERR_PTR(-ENOMEM);
3404 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3406 if (!iter->buffer_iter)
3410 * We make a copy of the current tracer to avoid concurrent
3411 * changes on it while we are reading.
3413 mutex_lock(&trace_types_lock);
3414 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3418 *iter->trace = *tr->current_trace;
3420 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3425 #ifdef CONFIG_TRACER_MAX_TRACE
3426 /* Currently only the top directory has a snapshot */
3427 if (tr->current_trace->print_max || snapshot)
3428 iter->trace_buffer = &tr->max_buffer;
3431 iter->trace_buffer = &tr->trace_buffer;
3432 iter->snapshot = snapshot;
3434 iter->cpu_file = tracing_get_cpu(inode);
3435 mutex_init(&iter->mutex);
3437 /* Notify the tracer early; before we stop tracing. */
3438 if (iter->trace && iter->trace->open)
3439 iter->trace->open(iter);
3441 /* Annotate start of buffers if we had overruns */
3442 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3443 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3445 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3446 if (trace_clocks[tr->clock_id].in_ns)
3447 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3449 /* stop the trace while dumping if we are not opening "snapshot" */
3450 if (!iter->snapshot)
3451 tracing_stop_tr(tr);
3453 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3454 for_each_tracing_cpu(cpu) {
3455 iter->buffer_iter[cpu] =
3456 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3458 ring_buffer_read_prepare_sync();
3459 for_each_tracing_cpu(cpu) {
3460 ring_buffer_read_start(iter->buffer_iter[cpu]);
3461 tracing_iter_reset(iter, cpu);
3464 cpu = iter->cpu_file;
3465 iter->buffer_iter[cpu] =
3466 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3467 ring_buffer_read_prepare_sync();
3468 ring_buffer_read_start(iter->buffer_iter[cpu]);
3469 tracing_iter_reset(iter, cpu);
3472 mutex_unlock(&trace_types_lock);
3477 mutex_unlock(&trace_types_lock);
3479 kfree(iter->buffer_iter);
3481 seq_release_private(inode, file);
3482 return ERR_PTR(-ENOMEM);
3485 int tracing_open_generic(struct inode *inode, struct file *filp)
3487 if (tracing_disabled)
3490 filp->private_data = inode->i_private;
3494 bool tracing_is_disabled(void)
3496 return (tracing_disabled) ? true: false;
3500 * Open and update trace_array ref count.
3501 * Must have the current trace_array passed to it.
3503 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3505 struct trace_array *tr = inode->i_private;
3507 if (tracing_disabled)
3510 if (trace_array_get(tr) < 0)
3513 filp->private_data = inode->i_private;
3518 static int tracing_release(struct inode *inode, struct file *file)
3520 struct trace_array *tr = inode->i_private;
3521 struct seq_file *m = file->private_data;
3522 struct trace_iterator *iter;
3525 if (!(file->f_mode & FMODE_READ)) {
3526 trace_array_put(tr);
3530 /* Writes do not use seq_file */
3532 mutex_lock(&trace_types_lock);
3534 for_each_tracing_cpu(cpu) {
3535 if (iter->buffer_iter[cpu])
3536 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3539 if (iter->trace && iter->trace->close)
3540 iter->trace->close(iter);
3542 if (!iter->snapshot)
3543 /* reenable tracing if it was previously enabled */
3544 tracing_start_tr(tr);
3546 __trace_array_put(tr);
3548 mutex_unlock(&trace_types_lock);
3550 mutex_destroy(&iter->mutex);
3551 free_cpumask_var(iter->started);
3553 kfree(iter->buffer_iter);
3554 seq_release_private(inode, file);
3559 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3561 struct trace_array *tr = inode->i_private;
3563 trace_array_put(tr);
3567 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3569 struct trace_array *tr = inode->i_private;
3571 trace_array_put(tr);
3573 return single_release(inode, file);
3576 static int tracing_open(struct inode *inode, struct file *file)
3578 struct trace_array *tr = inode->i_private;
3579 struct trace_iterator *iter;
3582 if (trace_array_get(tr) < 0)
3585 /* If this file was open for write, then erase contents */
3586 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3587 int cpu = tracing_get_cpu(inode);
3589 if (cpu == RING_BUFFER_ALL_CPUS)
3590 tracing_reset_online_cpus(&tr->trace_buffer);
3592 tracing_reset(&tr->trace_buffer, cpu);
3595 if (file->f_mode & FMODE_READ) {
3596 iter = __tracing_open(inode, file, false);
3598 ret = PTR_ERR(iter);
3599 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3600 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3604 trace_array_put(tr);
3610 * Some tracers are not suitable for instance buffers.
3611 * A tracer is always available for the global array (toplevel)
3612 * or if it explicitly states that it is.
3615 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3617 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3620 /* Find the next tracer that this trace array may use */
3621 static struct tracer *
3622 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3624 while (t && !trace_ok_for_array(t, tr))
3631 t_next(struct seq_file *m, void *v, loff_t *pos)
3633 struct trace_array *tr = m->private;
3634 struct tracer *t = v;
3639 t = get_tracer_for_array(tr, t->next);
3644 static void *t_start(struct seq_file *m, loff_t *pos)
3646 struct trace_array *tr = m->private;
3650 mutex_lock(&trace_types_lock);
3652 t = get_tracer_for_array(tr, trace_types);
3653 for (; t && l < *pos; t = t_next(m, t, &l))
3659 static void t_stop(struct seq_file *m, void *p)
3661 mutex_unlock(&trace_types_lock);
3664 static int t_show(struct seq_file *m, void *v)
3666 struct tracer *t = v;
3671 seq_puts(m, t->name);
3680 static const struct seq_operations show_traces_seq_ops = {
3687 static int show_traces_open(struct inode *inode, struct file *file)
3689 struct trace_array *tr = inode->i_private;
3693 if (tracing_disabled)
3696 ret = seq_open(file, &show_traces_seq_ops);
3700 m = file->private_data;
3707 tracing_write_stub(struct file *filp, const char __user *ubuf,
3708 size_t count, loff_t *ppos)
3713 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3717 if (file->f_mode & FMODE_READ)
3718 ret = seq_lseek(file, offset, whence);
3720 file->f_pos = ret = 0;
3725 static const struct file_operations tracing_fops = {
3726 .open = tracing_open,
3728 .write = tracing_write_stub,
3729 .llseek = tracing_lseek,
3730 .release = tracing_release,
3733 static const struct file_operations show_traces_fops = {
3734 .open = show_traces_open,
3736 .release = seq_release,
3737 .llseek = seq_lseek,
3741 * The tracer itself will not take this lock, but still we want
3742 * to provide a consistent cpumask to user-space:
3744 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3747 * Temporary storage for the character representation of the
3748 * CPU bitmask (and one more byte for the newline):
3750 static char mask_str[NR_CPUS + 1];
3753 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3754 size_t count, loff_t *ppos)
3756 struct trace_array *tr = file_inode(filp)->i_private;
3759 mutex_lock(&tracing_cpumask_update_lock);
3761 len = snprintf(mask_str, count, "%*pb\n",
3762 cpumask_pr_args(tr->tracing_cpumask));
3767 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3770 mutex_unlock(&tracing_cpumask_update_lock);
3776 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3777 size_t count, loff_t *ppos)
3779 struct trace_array *tr = file_inode(filp)->i_private;
3780 cpumask_var_t tracing_cpumask_new;
3783 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3786 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3790 mutex_lock(&tracing_cpumask_update_lock);
3792 local_irq_disable();
3793 arch_spin_lock(&tr->max_lock);
3794 for_each_tracing_cpu(cpu) {
3796 * Increase/decrease the disabled counter if we are
3797 * about to flip a bit in the cpumask:
3799 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3800 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3801 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3802 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3804 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3805 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3806 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3807 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3810 arch_spin_unlock(&tr->max_lock);
3813 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3815 mutex_unlock(&tracing_cpumask_update_lock);
3816 free_cpumask_var(tracing_cpumask_new);
3821 free_cpumask_var(tracing_cpumask_new);
3826 static const struct file_operations tracing_cpumask_fops = {
3827 .open = tracing_open_generic_tr,
3828 .read = tracing_cpumask_read,
3829 .write = tracing_cpumask_write,
3830 .release = tracing_release_generic_tr,
3831 .llseek = generic_file_llseek,
3834 static int tracing_trace_options_show(struct seq_file *m, void *v)
3836 struct tracer_opt *trace_opts;
3837 struct trace_array *tr = m->private;
3841 mutex_lock(&trace_types_lock);
3842 tracer_flags = tr->current_trace->flags->val;
3843 trace_opts = tr->current_trace->flags->opts;
3845 for (i = 0; trace_options[i]; i++) {
3846 if (tr->trace_flags & (1 << i))
3847 seq_printf(m, "%s\n", trace_options[i]);
3849 seq_printf(m, "no%s\n", trace_options[i]);
3852 for (i = 0; trace_opts[i].name; i++) {
3853 if (tracer_flags & trace_opts[i].bit)
3854 seq_printf(m, "%s\n", trace_opts[i].name);
3856 seq_printf(m, "no%s\n", trace_opts[i].name);
3858 mutex_unlock(&trace_types_lock);
3863 static int __set_tracer_option(struct trace_array *tr,
3864 struct tracer_flags *tracer_flags,
3865 struct tracer_opt *opts, int neg)
3867 struct tracer *trace = tracer_flags->trace;
3870 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3875 tracer_flags->val &= ~opts->bit;
3877 tracer_flags->val |= opts->bit;
3881 /* Try to assign a tracer specific option */
3882 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3884 struct tracer *trace = tr->current_trace;
3885 struct tracer_flags *tracer_flags = trace->flags;
3886 struct tracer_opt *opts = NULL;
3889 for (i = 0; tracer_flags->opts[i].name; i++) {
3890 opts = &tracer_flags->opts[i];
3892 if (strcmp(cmp, opts->name) == 0)
3893 return __set_tracer_option(tr, trace->flags, opts, neg);
3899 /* Some tracers require overwrite to stay enabled */
3900 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3902 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3908 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3910 /* do nothing if flag is already set */
3911 if (!!(tr->trace_flags & mask) == !!enabled)
3914 /* Give the tracer a chance to approve the change */
3915 if (tr->current_trace->flag_changed)
3916 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3920 tr->trace_flags |= mask;
3922 tr->trace_flags &= ~mask;
3924 if (mask == TRACE_ITER_RECORD_CMD)
3925 trace_event_enable_cmd_record(enabled);
3927 if (mask == TRACE_ITER_EVENT_FORK)
3928 trace_event_follow_fork(tr, enabled);
3930 if (mask == TRACE_ITER_OVERWRITE) {
3931 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3932 #ifdef CONFIG_TRACER_MAX_TRACE
3933 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3937 if (mask == TRACE_ITER_PRINTK) {
3938 trace_printk_start_stop_comm(enabled);
3939 trace_printk_control(enabled);
3945 static int trace_set_options(struct trace_array *tr, char *option)
3951 size_t orig_len = strlen(option);
3953 cmp = strstrip(option);
3955 if (strncmp(cmp, "no", 2) == 0) {
3960 mutex_lock(&trace_types_lock);
3962 for (i = 0; trace_options[i]; i++) {
3963 if (strcmp(cmp, trace_options[i]) == 0) {
3964 ret = set_tracer_flag(tr, 1 << i, !neg);
3969 /* If no option could be set, test the specific tracer options */
3970 if (!trace_options[i])
3971 ret = set_tracer_option(tr, cmp, neg);
3973 mutex_unlock(&trace_types_lock);
3976 * If the first trailing whitespace is replaced with '\0' by strstrip,
3977 * turn it back into a space.
3979 if (orig_len > strlen(option))
3980 option[strlen(option)] = ' ';
3985 static void __init apply_trace_boot_options(void)
3987 char *buf = trace_boot_options_buf;
3991 option = strsep(&buf, ",");
3997 trace_set_options(&global_trace, option);
3999 /* Put back the comma to allow this to be called again */
4006 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4007 size_t cnt, loff_t *ppos)
4009 struct seq_file *m = filp->private_data;
4010 struct trace_array *tr = m->private;
4014 if (cnt >= sizeof(buf))
4017 if (copy_from_user(buf, ubuf, cnt))
4022 ret = trace_set_options(tr, buf);
4031 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4033 struct trace_array *tr = inode->i_private;
4036 if (tracing_disabled)
4039 if (trace_array_get(tr) < 0)
4042 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4044 trace_array_put(tr);
4049 static const struct file_operations tracing_iter_fops = {
4050 .open = tracing_trace_options_open,
4052 .llseek = seq_lseek,
4053 .release = tracing_single_release_tr,
4054 .write = tracing_trace_options_write,
4057 static const char readme_msg[] =
4058 "tracing mini-HOWTO:\n\n"
4059 "# echo 0 > tracing_on : quick way to disable tracing\n"
4060 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4061 " Important files:\n"
4062 " trace\t\t\t- The static contents of the buffer\n"
4063 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4064 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4065 " current_tracer\t- function and latency tracers\n"
4066 " available_tracers\t- list of configured tracers for current_tracer\n"
4067 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4068 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4069 " trace_clock\t\t-change the clock used to order events\n"
4070 " local: Per cpu clock but may not be synced across CPUs\n"
4071 " global: Synced across CPUs but slows tracing down.\n"
4072 " counter: Not a clock, but just an increment\n"
4073 " uptime: Jiffy counter from time of boot\n"
4074 " perf: Same clock that perf events use\n"
4075 #ifdef CONFIG_X86_64
4076 " x86-tsc: TSC cycle counter\n"
4078 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4079 " tracing_cpumask\t- Limit which CPUs to trace\n"
4080 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4081 "\t\t\t Remove sub-buffer with rmdir\n"
4082 " trace_options\t\t- Set format or modify how tracing happens\n"
4083 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4084 "\t\t\t option name\n"
4085 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4086 #ifdef CONFIG_DYNAMIC_FTRACE
4087 "\n available_filter_functions - list of functions that can be filtered on\n"
4088 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4089 "\t\t\t functions\n"
4090 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4091 "\t modules: Can select a group via module\n"
4092 "\t Format: :mod:<module-name>\n"
4093 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4094 "\t triggers: a command to perform when function is hit\n"
4095 "\t Format: <function>:<trigger>[:count]\n"
4096 "\t trigger: traceon, traceoff\n"
4097 "\t\t enable_event:<system>:<event>\n"
4098 "\t\t disable_event:<system>:<event>\n"
4099 #ifdef CONFIG_STACKTRACE
4102 #ifdef CONFIG_TRACER_SNAPSHOT
4107 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4108 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4109 "\t The first one will disable tracing every time do_fault is hit\n"
4110 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4111 "\t The first time do trap is hit and it disables tracing, the\n"
4112 "\t counter will decrement to 2. If tracing is already disabled,\n"
4113 "\t the counter will not decrement. It only decrements when the\n"
4114 "\t trigger did work\n"
4115 "\t To remove trigger without count:\n"
4116 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4117 "\t To remove trigger with a count:\n"
4118 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4119 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4120 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4121 "\t modules: Can select a group via module command :mod:\n"
4122 "\t Does not accept triggers\n"
4123 #endif /* CONFIG_DYNAMIC_FTRACE */
4124 #ifdef CONFIG_FUNCTION_TRACER
4125 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4128 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4129 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4130 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4131 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4133 #ifdef CONFIG_TRACER_SNAPSHOT
4134 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4135 "\t\t\t snapshot buffer. Read the contents for more\n"
4136 "\t\t\t information\n"
4138 #ifdef CONFIG_STACK_TRACER
4139 " stack_trace\t\t- Shows the max stack trace when active\n"
4140 " stack_max_size\t- Shows current max stack size that was traced\n"
4141 "\t\t\t Write into this file to reset the max size (trigger a\n"
4142 "\t\t\t new trace)\n"
4143 #ifdef CONFIG_DYNAMIC_FTRACE
4144 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4147 #endif /* CONFIG_STACK_TRACER */
4148 " events/\t\t- Directory containing all trace event subsystems:\n"
4149 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4150 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4151 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4153 " filter\t\t- If set, only events passing filter are traced\n"
4154 " events/<system>/<event>/\t- Directory containing control files for\n"
4156 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4157 " filter\t\t- If set, only events passing filter are traced\n"
4158 " trigger\t\t- If set, a command to perform when event is hit\n"
4159 "\t Format: <trigger>[:count][if <filter>]\n"
4160 "\t trigger: traceon, traceoff\n"
4161 "\t enable_event:<system>:<event>\n"
4162 "\t disable_event:<system>:<event>\n"
4163 #ifdef CONFIG_HIST_TRIGGERS
4164 "\t enable_hist:<system>:<event>\n"
4165 "\t disable_hist:<system>:<event>\n"
4167 #ifdef CONFIG_STACKTRACE
4170 #ifdef CONFIG_TRACER_SNAPSHOT
4173 #ifdef CONFIG_HIST_TRIGGERS
4174 "\t\t hist (see below)\n"
4176 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4177 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4178 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4179 "\t events/block/block_unplug/trigger\n"
4180 "\t The first disables tracing every time block_unplug is hit.\n"
4181 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4182 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4183 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4184 "\t Like function triggers, the counter is only decremented if it\n"
4185 "\t enabled or disabled tracing.\n"
4186 "\t To remove a trigger without a count:\n"
4187 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4188 "\t To remove a trigger with a count:\n"
4189 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4190 "\t Filters can be ignored when removing a trigger.\n"
4191 #ifdef CONFIG_HIST_TRIGGERS
4192 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4193 "\t Format: hist:keys=<field1[,field2,...]>\n"
4194 "\t [:values=<field1[,field2,...]>]\n"
4195 "\t [:sort=<field1[,field2,...]>]\n"
4196 "\t [:size=#entries]\n"
4197 "\t [:pause][:continue][:clear]\n"
4198 "\t [:name=histname1]\n"
4199 "\t [if <filter>]\n\n"
4200 "\t When a matching event is hit, an entry is added to a hash\n"
4201 "\t table using the key(s) and value(s) named, and the value of a\n"
4202 "\t sum called 'hitcount' is incremented. Keys and values\n"
4203 "\t correspond to fields in the event's format description. Keys\n"
4204 "\t can be any field, or the special string 'stacktrace'.\n"
4205 "\t Compound keys consisting of up to two fields can be specified\n"
4206 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4207 "\t fields. Sort keys consisting of up to two fields can be\n"
4208 "\t specified using the 'sort' keyword. The sort direction can\n"
4209 "\t be modified by appending '.descending' or '.ascending' to a\n"
4210 "\t sort field. The 'size' parameter can be used to specify more\n"
4211 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4212 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4213 "\t its histogram data will be shared with other triggers of the\n"
4214 "\t same name, and trigger hits will update this common data.\n\n"
4215 "\t Reading the 'hist' file for the event will dump the hash\n"
4216 "\t table in its entirety to stdout. If there are multiple hist\n"
4217 "\t triggers attached to an event, there will be a table for each\n"
4218 "\t trigger in the output. The table displayed for a named\n"
4219 "\t trigger will be the same as any other instance having the\n"
4220 "\t same name. The default format used to display a given field\n"
4221 "\t can be modified by appending any of the following modifiers\n"
4222 "\t to the field name, as applicable:\n\n"
4223 "\t .hex display a number as a hex value\n"
4224 "\t .sym display an address as a symbol\n"
4225 "\t .sym-offset display an address as a symbol and offset\n"
4226 "\t .execname display a common_pid as a program name\n"
4227 "\t .syscall display a syscall id as a syscall name\n\n"
4228 "\t .log2 display log2 value rather than raw number\n\n"
4229 "\t The 'pause' parameter can be used to pause an existing hist\n"
4230 "\t trigger or to start a hist trigger but not log any events\n"
4231 "\t until told to do so. 'continue' can be used to start or\n"
4232 "\t restart a paused hist trigger.\n\n"
4233 "\t The 'clear' parameter will clear the contents of a running\n"
4234 "\t hist trigger and leave its current paused/active state\n"
4236 "\t The enable_hist and disable_hist triggers can be used to\n"
4237 "\t have one event conditionally start and stop another event's\n"
4238 "\t already-attached hist trigger. The syntax is analagous to\n"
4239 "\t the enable_event and disable_event triggers.\n"
4244 tracing_readme_read(struct file *filp, char __user *ubuf,
4245 size_t cnt, loff_t *ppos)
4247 return simple_read_from_buffer(ubuf, cnt, ppos,
4248 readme_msg, strlen(readme_msg));
4251 static const struct file_operations tracing_readme_fops = {
4252 .open = tracing_open_generic,
4253 .read = tracing_readme_read,
4254 .llseek = generic_file_llseek,
4257 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4259 unsigned int *ptr = v;
4261 if (*pos || m->count)
4266 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4268 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4277 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4283 arch_spin_lock(&trace_cmdline_lock);
4285 v = &savedcmd->map_cmdline_to_pid[0];
4287 v = saved_cmdlines_next(m, v, &l);
4295 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4297 arch_spin_unlock(&trace_cmdline_lock);
4301 static int saved_cmdlines_show(struct seq_file *m, void *v)
4303 char buf[TASK_COMM_LEN];
4304 unsigned int *pid = v;
4306 __trace_find_cmdline(*pid, buf);
4307 seq_printf(m, "%d %s\n", *pid, buf);
4311 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4312 .start = saved_cmdlines_start,
4313 .next = saved_cmdlines_next,
4314 .stop = saved_cmdlines_stop,
4315 .show = saved_cmdlines_show,
4318 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4320 if (tracing_disabled)
4323 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4326 static const struct file_operations tracing_saved_cmdlines_fops = {
4327 .open = tracing_saved_cmdlines_open,
4329 .llseek = seq_lseek,
4330 .release = seq_release,
4334 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4335 size_t cnt, loff_t *ppos)
4340 arch_spin_lock(&trace_cmdline_lock);
4341 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4342 arch_spin_unlock(&trace_cmdline_lock);
4344 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4347 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4349 kfree(s->saved_cmdlines);
4350 kfree(s->map_cmdline_to_pid);
4354 static int tracing_resize_saved_cmdlines(unsigned int val)
4356 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4358 s = kmalloc(sizeof(*s), GFP_KERNEL);
4362 if (allocate_cmdlines_buffer(val, s) < 0) {
4367 arch_spin_lock(&trace_cmdline_lock);
4368 savedcmd_temp = savedcmd;
4370 arch_spin_unlock(&trace_cmdline_lock);
4371 free_saved_cmdlines_buffer(savedcmd_temp);
4377 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4378 size_t cnt, loff_t *ppos)
4383 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4387 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4388 if (!val || val > PID_MAX_DEFAULT)
4391 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4400 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4401 .open = tracing_open_generic,
4402 .read = tracing_saved_cmdlines_size_read,
4403 .write = tracing_saved_cmdlines_size_write,
4406 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4407 static union trace_enum_map_item *
4408 update_enum_map(union trace_enum_map_item *ptr)
4410 if (!ptr->map.enum_string) {
4411 if (ptr->tail.next) {
4412 ptr = ptr->tail.next;
4413 /* Set ptr to the next real item (skip head) */
4421 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4423 union trace_enum_map_item *ptr = v;
4426 * Paranoid! If ptr points to end, we don't want to increment past it.
4427 * This really should never happen.
4429 ptr = update_enum_map(ptr);
4430 if (WARN_ON_ONCE(!ptr))
4437 ptr = update_enum_map(ptr);
4442 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4444 union trace_enum_map_item *v;
4447 mutex_lock(&trace_enum_mutex);
4449 v = trace_enum_maps;
4453 while (v && l < *pos) {
4454 v = enum_map_next(m, v, &l);
4460 static void enum_map_stop(struct seq_file *m, void *v)
4462 mutex_unlock(&trace_enum_mutex);
4465 static int enum_map_show(struct seq_file *m, void *v)
4467 union trace_enum_map_item *ptr = v;
4469 seq_printf(m, "%s %ld (%s)\n",
4470 ptr->map.enum_string, ptr->map.enum_value,
4476 static const struct seq_operations tracing_enum_map_seq_ops = {
4477 .start = enum_map_start,
4478 .next = enum_map_next,
4479 .stop = enum_map_stop,
4480 .show = enum_map_show,
4483 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4485 if (tracing_disabled)
4488 return seq_open(filp, &tracing_enum_map_seq_ops);
4491 static const struct file_operations tracing_enum_map_fops = {
4492 .open = tracing_enum_map_open,
4494 .llseek = seq_lseek,
4495 .release = seq_release,
4498 static inline union trace_enum_map_item *
4499 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4501 /* Return tail of array given the head */
4502 return ptr + ptr->head.length + 1;
4506 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4509 struct trace_enum_map **stop;
4510 struct trace_enum_map **map;
4511 union trace_enum_map_item *map_array;
4512 union trace_enum_map_item *ptr;
4517 * The trace_enum_maps contains the map plus a head and tail item,
4518 * where the head holds the module and length of array, and the
4519 * tail holds a pointer to the next list.
4521 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4523 pr_warn("Unable to allocate trace enum mapping\n");
4527 mutex_lock(&trace_enum_mutex);
4529 if (!trace_enum_maps)
4530 trace_enum_maps = map_array;
4532 ptr = trace_enum_maps;
4534 ptr = trace_enum_jmp_to_tail(ptr);
4535 if (!ptr->tail.next)
4537 ptr = ptr->tail.next;
4540 ptr->tail.next = map_array;
4542 map_array->head.mod = mod;
4543 map_array->head.length = len;
4546 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4547 map_array->map = **map;
4550 memset(map_array, 0, sizeof(*map_array));
4552 mutex_unlock(&trace_enum_mutex);
4555 static void trace_create_enum_file(struct dentry *d_tracer)
4557 trace_create_file("enum_map", 0444, d_tracer,
4558 NULL, &tracing_enum_map_fops);
4561 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4562 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4563 static inline void trace_insert_enum_map_file(struct module *mod,
4564 struct trace_enum_map **start, int len) { }
4565 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4567 static void trace_insert_enum_map(struct module *mod,
4568 struct trace_enum_map **start, int len)
4570 struct trace_enum_map **map;
4577 trace_event_enum_update(map, len);
4579 trace_insert_enum_map_file(mod, start, len);
4583 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4584 size_t cnt, loff_t *ppos)
4586 struct trace_array *tr = filp->private_data;
4587 char buf[MAX_TRACER_SIZE+2];
4590 mutex_lock(&trace_types_lock);
4591 r = sprintf(buf, "%s\n", tr->current_trace->name);
4592 mutex_unlock(&trace_types_lock);
4594 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4597 int tracer_init(struct tracer *t, struct trace_array *tr)
4599 tracing_reset_online_cpus(&tr->trace_buffer);
4603 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4607 for_each_tracing_cpu(cpu)
4608 per_cpu_ptr(buf->data, cpu)->entries = val;
4611 #ifdef CONFIG_TRACER_MAX_TRACE
4612 /* resize @tr's buffer to the size of @size_tr's entries */
4613 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4614 struct trace_buffer *size_buf, int cpu_id)
4618 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4619 for_each_tracing_cpu(cpu) {
4620 ret = ring_buffer_resize(trace_buf->buffer,
4621 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4624 per_cpu_ptr(trace_buf->data, cpu)->entries =
4625 per_cpu_ptr(size_buf->data, cpu)->entries;
4628 ret = ring_buffer_resize(trace_buf->buffer,
4629 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4631 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4632 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4637 #endif /* CONFIG_TRACER_MAX_TRACE */
4639 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4640 unsigned long size, int cpu)
4645 * If kernel or user changes the size of the ring buffer
4646 * we use the size that was given, and we can forget about
4647 * expanding it later.
4649 ring_buffer_expanded = true;
4651 /* May be called before buffers are initialized */
4652 if (!tr->trace_buffer.buffer)
4655 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4659 #ifdef CONFIG_TRACER_MAX_TRACE
4660 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4661 !tr->current_trace->use_max_tr)
4664 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4666 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4667 &tr->trace_buffer, cpu);
4670 * AARGH! We are left with different
4671 * size max buffer!!!!
4672 * The max buffer is our "snapshot" buffer.
4673 * When a tracer needs a snapshot (one of the
4674 * latency tracers), it swaps the max buffer
4675 * with the saved snap shot. We succeeded to
4676 * update the size of the main buffer, but failed to
4677 * update the size of the max buffer. But when we tried
4678 * to reset the main buffer to the original size, we
4679 * failed there too. This is very unlikely to
4680 * happen, but if it does, warn and kill all
4684 tracing_disabled = 1;
4689 if (cpu == RING_BUFFER_ALL_CPUS)
4690 set_buffer_entries(&tr->max_buffer, size);
4692 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4695 #endif /* CONFIG_TRACER_MAX_TRACE */
4697 if (cpu == RING_BUFFER_ALL_CPUS)
4698 set_buffer_entries(&tr->trace_buffer, size);
4700 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4705 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4706 unsigned long size, int cpu_id)
4710 mutex_lock(&trace_types_lock);
4712 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4713 /* make sure, this cpu is enabled in the mask */
4714 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4720 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4725 mutex_unlock(&trace_types_lock);
4732 * tracing_update_buffers - used by tracing facility to expand ring buffers
4734 * To save on memory when the tracing is never used on a system with it
4735 * configured in. The ring buffers are set to a minimum size. But once
4736 * a user starts to use the tracing facility, then they need to grow
4737 * to their default size.
4739 * This function is to be called when a tracer is about to be used.
4741 int tracing_update_buffers(void)
4745 mutex_lock(&trace_types_lock);
4746 if (!ring_buffer_expanded)
4747 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4748 RING_BUFFER_ALL_CPUS);
4749 mutex_unlock(&trace_types_lock);
4754 struct trace_option_dentry;
4757 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4760 * Used to clear out the tracer before deletion of an instance.
4761 * Must have trace_types_lock held.
4763 static void tracing_set_nop(struct trace_array *tr)
4765 if (tr->current_trace == &nop_trace)
4768 tr->current_trace->enabled--;
4770 if (tr->current_trace->reset)
4771 tr->current_trace->reset(tr);
4773 tr->current_trace = &nop_trace;
4776 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4778 /* Only enable if the directory has been created already. */
4782 create_trace_option_files(tr, t);
4785 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4788 #ifdef CONFIG_TRACER_MAX_TRACE
4793 mutex_lock(&trace_types_lock);
4795 if (!ring_buffer_expanded) {
4796 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4797 RING_BUFFER_ALL_CPUS);
4803 for (t = trace_types; t; t = t->next) {
4804 if (strcmp(t->name, buf) == 0)
4811 if (t == tr->current_trace)
4814 /* Some tracers are only allowed for the top level buffer */
4815 if (!trace_ok_for_array(t, tr)) {
4820 /* If trace pipe files are being read, we can't change the tracer */
4821 if (tr->current_trace->ref) {
4826 trace_branch_disable();
4828 tr->current_trace->enabled--;
4830 if (tr->current_trace->reset)
4831 tr->current_trace->reset(tr);
4833 /* Current trace needs to be nop_trace before synchronize_sched */
4834 tr->current_trace = &nop_trace;
4836 #ifdef CONFIG_TRACER_MAX_TRACE
4837 had_max_tr = tr->allocated_snapshot;
4839 if (had_max_tr && !t->use_max_tr) {
4841 * We need to make sure that the update_max_tr sees that
4842 * current_trace changed to nop_trace to keep it from
4843 * swapping the buffers after we resize it.
4844 * The update_max_tr is called from interrupts disabled
4845 * so a synchronized_sched() is sufficient.
4847 synchronize_sched();
4852 #ifdef CONFIG_TRACER_MAX_TRACE
4853 if (t->use_max_tr && !had_max_tr) {
4854 ret = alloc_snapshot(tr);
4861 ret = tracer_init(t, tr);
4866 tr->current_trace = t;
4867 tr->current_trace->enabled++;
4868 trace_branch_enable(tr);
4870 mutex_unlock(&trace_types_lock);
4876 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4877 size_t cnt, loff_t *ppos)
4879 struct trace_array *tr = filp->private_data;
4880 char buf[MAX_TRACER_SIZE+1];
4887 if (cnt > MAX_TRACER_SIZE)
4888 cnt = MAX_TRACER_SIZE;
4890 if (copy_from_user(buf, ubuf, cnt))
4895 /* strip ending whitespace. */
4896 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4899 err = tracing_set_tracer(tr, buf);
4909 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4910 size_t cnt, loff_t *ppos)
4915 r = snprintf(buf, sizeof(buf), "%ld\n",
4916 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4917 if (r > sizeof(buf))
4919 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4923 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4924 size_t cnt, loff_t *ppos)
4929 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4939 tracing_thresh_read(struct file *filp, char __user *ubuf,
4940 size_t cnt, loff_t *ppos)
4942 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4946 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4947 size_t cnt, loff_t *ppos)
4949 struct trace_array *tr = filp->private_data;
4952 mutex_lock(&trace_types_lock);
4953 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4957 if (tr->current_trace->update_thresh) {
4958 ret = tr->current_trace->update_thresh(tr);
4965 mutex_unlock(&trace_types_lock);
4970 #ifdef CONFIG_TRACER_MAX_TRACE
4973 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4974 size_t cnt, loff_t *ppos)
4976 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4980 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4981 size_t cnt, loff_t *ppos)
4983 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4988 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4990 struct trace_array *tr = inode->i_private;
4991 struct trace_iterator *iter;
4994 if (tracing_disabled)
4997 if (trace_array_get(tr) < 0)
5000 mutex_lock(&trace_types_lock);
5002 /* create a buffer to store the information to pass to userspace */
5003 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5006 __trace_array_put(tr);
5010 trace_seq_init(&iter->seq);
5011 iter->trace = tr->current_trace;
5013 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5018 /* trace pipe does not show start of buffer */
5019 cpumask_setall(iter->started);
5021 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5022 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5024 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5025 if (trace_clocks[tr->clock_id].in_ns)
5026 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5029 iter->trace_buffer = &tr->trace_buffer;
5030 iter->cpu_file = tracing_get_cpu(inode);
5031 mutex_init(&iter->mutex);
5032 filp->private_data = iter;
5034 if (iter->trace->pipe_open)
5035 iter->trace->pipe_open(iter);
5037 nonseekable_open(inode, filp);
5039 tr->current_trace->ref++;
5041 mutex_unlock(&trace_types_lock);
5047 __trace_array_put(tr);
5048 mutex_unlock(&trace_types_lock);
5052 static int tracing_release_pipe(struct inode *inode, struct file *file)
5054 struct trace_iterator *iter = file->private_data;
5055 struct trace_array *tr = inode->i_private;
5057 mutex_lock(&trace_types_lock);
5059 tr->current_trace->ref--;
5061 if (iter->trace->pipe_close)
5062 iter->trace->pipe_close(iter);
5064 mutex_unlock(&trace_types_lock);
5066 free_cpumask_var(iter->started);
5067 mutex_destroy(&iter->mutex);
5070 trace_array_put(tr);
5076 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5078 struct trace_array *tr = iter->tr;
5080 /* Iterators are static, they should be filled or empty */
5081 if (trace_buffer_iter(iter, iter->cpu_file))
5082 return POLLIN | POLLRDNORM;
5084 if (tr->trace_flags & TRACE_ITER_BLOCK)
5086 * Always select as readable when in blocking mode
5088 return POLLIN | POLLRDNORM;
5090 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5095 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5097 struct trace_iterator *iter = filp->private_data;
5099 return trace_poll(iter, filp, poll_table);
5102 /* Must be called with iter->mutex held. */
5103 static int tracing_wait_pipe(struct file *filp)
5105 struct trace_iterator *iter = filp->private_data;
5108 while (trace_empty(iter)) {
5110 if ((filp->f_flags & O_NONBLOCK)) {
5115 * We block until we read something and tracing is disabled.
5116 * We still block if tracing is disabled, but we have never
5117 * read anything. This allows a user to cat this file, and
5118 * then enable tracing. But after we have read something,
5119 * we give an EOF when tracing is again disabled.
5121 * iter->pos will be 0 if we haven't read anything.
5123 if (!tracing_is_on() && iter->pos)
5126 mutex_unlock(&iter->mutex);
5128 ret = wait_on_pipe(iter, false);
5130 mutex_lock(&iter->mutex);
5143 tracing_read_pipe(struct file *filp, char __user *ubuf,
5144 size_t cnt, loff_t *ppos)
5146 struct trace_iterator *iter = filp->private_data;
5149 /* return any leftover data */
5150 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5154 trace_seq_init(&iter->seq);
5157 * Avoid more than one consumer on a single file descriptor
5158 * This is just a matter of traces coherency, the ring buffer itself
5161 mutex_lock(&iter->mutex);
5162 if (iter->trace->read) {
5163 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5169 sret = tracing_wait_pipe(filp);
5173 /* stop when tracing is finished */
5174 if (trace_empty(iter)) {
5179 if (cnt >= PAGE_SIZE)
5180 cnt = PAGE_SIZE - 1;
5182 /* reset all but tr, trace, and overruns */
5183 memset(&iter->seq, 0,
5184 sizeof(struct trace_iterator) -
5185 offsetof(struct trace_iterator, seq));
5186 cpumask_clear(iter->started);
5189 trace_event_read_lock();
5190 trace_access_lock(iter->cpu_file);
5191 while (trace_find_next_entry_inc(iter) != NULL) {
5192 enum print_line_t ret;
5193 int save_len = iter->seq.seq.len;
5195 ret = print_trace_line(iter);
5196 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5197 /* don't print partial lines */
5198 iter->seq.seq.len = save_len;
5201 if (ret != TRACE_TYPE_NO_CONSUME)
5202 trace_consume(iter);
5204 if (trace_seq_used(&iter->seq) >= cnt)
5208 * Setting the full flag means we reached the trace_seq buffer
5209 * size and we should leave by partial output condition above.
5210 * One of the trace_seq_* functions is not used properly.
5212 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5215 trace_access_unlock(iter->cpu_file);
5216 trace_event_read_unlock();
5218 /* Now copy what we have to the user */
5219 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5220 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
5221 trace_seq_init(&iter->seq);
5224 * If there was nothing to send to user, in spite of consuming trace
5225 * entries, go back to wait for more entries.
5231 mutex_unlock(&iter->mutex);
5236 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5239 __free_page(spd->pages[idx]);
5242 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5244 .confirm = generic_pipe_buf_confirm,
5245 .release = generic_pipe_buf_release,
5246 .steal = generic_pipe_buf_steal,
5247 .get = generic_pipe_buf_get,
5251 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5257 /* Seq buffer is page-sized, exactly what we need. */
5259 save_len = iter->seq.seq.len;
5260 ret = print_trace_line(iter);
5262 if (trace_seq_has_overflowed(&iter->seq)) {
5263 iter->seq.seq.len = save_len;
5268 * This should not be hit, because it should only
5269 * be set if the iter->seq overflowed. But check it
5270 * anyway to be safe.
5272 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5273 iter->seq.seq.len = save_len;
5277 count = trace_seq_used(&iter->seq) - save_len;
5280 iter->seq.seq.len = save_len;
5284 if (ret != TRACE_TYPE_NO_CONSUME)
5285 trace_consume(iter);
5287 if (!trace_find_next_entry_inc(iter)) {
5297 static ssize_t tracing_splice_read_pipe(struct file *filp,
5299 struct pipe_inode_info *pipe,
5303 struct page *pages_def[PIPE_DEF_BUFFERS];
5304 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5305 struct trace_iterator *iter = filp->private_data;
5306 struct splice_pipe_desc spd = {
5308 .partial = partial_def,
5309 .nr_pages = 0, /* This gets updated below. */
5310 .nr_pages_max = PIPE_DEF_BUFFERS,
5312 .ops = &tracing_pipe_buf_ops,
5313 .spd_release = tracing_spd_release_pipe,
5319 if (splice_grow_spd(pipe, &spd))
5322 mutex_lock(&iter->mutex);
5324 if (iter->trace->splice_read) {
5325 ret = iter->trace->splice_read(iter, filp,
5326 ppos, pipe, len, flags);
5331 ret = tracing_wait_pipe(filp);
5335 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5340 trace_event_read_lock();
5341 trace_access_lock(iter->cpu_file);
5343 /* Fill as many pages as possible. */
5344 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5345 spd.pages[i] = alloc_page(GFP_KERNEL);
5349 rem = tracing_fill_pipe_page(rem, iter);
5351 /* Copy the data into the page, so we can start over. */
5352 ret = trace_seq_to_buffer(&iter->seq,
5353 page_address(spd.pages[i]),
5354 trace_seq_used(&iter->seq));
5356 __free_page(spd.pages[i]);
5359 spd.partial[i].offset = 0;
5360 spd.partial[i].len = trace_seq_used(&iter->seq);
5362 trace_seq_init(&iter->seq);
5365 trace_access_unlock(iter->cpu_file);
5366 trace_event_read_unlock();
5367 mutex_unlock(&iter->mutex);
5372 ret = splice_to_pipe(pipe, &spd);
5376 splice_shrink_spd(&spd);
5380 mutex_unlock(&iter->mutex);
5385 tracing_entries_read(struct file *filp, char __user *ubuf,
5386 size_t cnt, loff_t *ppos)
5388 struct inode *inode = file_inode(filp);
5389 struct trace_array *tr = inode->i_private;
5390 int cpu = tracing_get_cpu(inode);
5395 mutex_lock(&trace_types_lock);
5397 if (cpu == RING_BUFFER_ALL_CPUS) {
5398 int cpu, buf_size_same;
5403 /* check if all cpu sizes are same */
5404 for_each_tracing_cpu(cpu) {
5405 /* fill in the size from first enabled cpu */
5407 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5408 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5414 if (buf_size_same) {
5415 if (!ring_buffer_expanded)
5416 r = sprintf(buf, "%lu (expanded: %lu)\n",
5418 trace_buf_size >> 10);
5420 r = sprintf(buf, "%lu\n", size >> 10);
5422 r = sprintf(buf, "X\n");
5424 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5426 mutex_unlock(&trace_types_lock);
5428 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5433 tracing_entries_write(struct file *filp, const char __user *ubuf,
5434 size_t cnt, loff_t *ppos)
5436 struct inode *inode = file_inode(filp);
5437 struct trace_array *tr = inode->i_private;
5441 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5445 /* must have at least 1 entry */
5449 /* value is in KB */
5451 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5461 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5462 size_t cnt, loff_t *ppos)
5464 struct trace_array *tr = filp->private_data;
5467 unsigned long size = 0, expanded_size = 0;
5469 mutex_lock(&trace_types_lock);
5470 for_each_tracing_cpu(cpu) {
5471 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5472 if (!ring_buffer_expanded)
5473 expanded_size += trace_buf_size >> 10;
5475 if (ring_buffer_expanded)
5476 r = sprintf(buf, "%lu\n", size);
5478 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5479 mutex_unlock(&trace_types_lock);
5481 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5485 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5486 size_t cnt, loff_t *ppos)
5489 * There is no need to read what the user has written, this function
5490 * is just to make sure that there is no error when "echo" is used
5499 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5501 struct trace_array *tr = inode->i_private;
5503 /* disable tracing ? */
5504 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5505 tracer_tracing_off(tr);
5506 /* resize the ring buffer to 0 */
5507 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5509 trace_array_put(tr);
5515 tracing_mark_write(struct file *filp, const char __user *ubuf,
5516 size_t cnt, loff_t *fpos)
5518 unsigned long addr = (unsigned long)ubuf;
5519 struct trace_array *tr = filp->private_data;
5520 struct ring_buffer_event *event;
5521 struct ring_buffer *buffer;
5522 struct print_entry *entry;
5523 unsigned long irq_flags;
5524 struct page *pages[2];
5534 if (tracing_disabled)
5537 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5540 if (cnt > TRACE_BUF_SIZE)
5541 cnt = TRACE_BUF_SIZE;
5544 * Userspace is injecting traces into the kernel trace buffer.
5545 * We want to be as non intrusive as possible.
5546 * To do so, we do not want to allocate any special buffers
5547 * or take any locks, but instead write the userspace data
5548 * straight into the ring buffer.
5550 * First we need to pin the userspace buffer into memory,
5551 * which, most likely it is, because it just referenced it.
5552 * But there's no guarantee that it is. By using get_user_pages_fast()
5553 * and kmap_atomic/kunmap_atomic() we can get access to the
5554 * pages directly. We then write the data directly into the
5557 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5559 /* check if we cross pages */
5560 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5563 offset = addr & (PAGE_SIZE - 1);
5566 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5567 if (ret < nr_pages) {
5569 put_page(pages[ret]);
5574 for (i = 0; i < nr_pages; i++)
5575 map_page[i] = kmap_atomic(pages[i]);
5577 local_save_flags(irq_flags);
5578 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5579 buffer = tr->trace_buffer.buffer;
5580 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5581 irq_flags, preempt_count());
5583 /* Ring buffer disabled, return as if not open for write */
5588 entry = ring_buffer_event_data(event);
5589 entry->ip = _THIS_IP_;
5591 if (nr_pages == 2) {
5592 len = PAGE_SIZE - offset;
5593 memcpy(&entry->buf, map_page[0] + offset, len);
5594 memcpy(&entry->buf[len], map_page[1], cnt - len);
5596 memcpy(&entry->buf, map_page[0] + offset, cnt);
5598 if (entry->buf[cnt - 1] != '\n') {
5599 entry->buf[cnt] = '\n';
5600 entry->buf[cnt + 1] = '\0';
5602 entry->buf[cnt] = '\0';
5604 __buffer_unlock_commit(buffer, event);
5611 for (i = nr_pages - 1; i >= 0; i--) {
5612 kunmap_atomic(map_page[i]);
5619 static int tracing_clock_show(struct seq_file *m, void *v)
5621 struct trace_array *tr = m->private;
5624 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5626 "%s%s%s%s", i ? " " : "",
5627 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5628 i == tr->clock_id ? "]" : "");
5634 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5638 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5639 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5642 if (i == ARRAY_SIZE(trace_clocks))
5645 mutex_lock(&trace_types_lock);
5649 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5652 * New clock may not be consistent with the previous clock.
5653 * Reset the buffer so that it doesn't have incomparable timestamps.
5655 tracing_reset_online_cpus(&tr->trace_buffer);
5657 #ifdef CONFIG_TRACER_MAX_TRACE
5658 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5659 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5660 tracing_reset_online_cpus(&tr->max_buffer);
5663 mutex_unlock(&trace_types_lock);
5668 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5669 size_t cnt, loff_t *fpos)
5671 struct seq_file *m = filp->private_data;
5672 struct trace_array *tr = m->private;
5674 const char *clockstr;
5677 if (cnt >= sizeof(buf))
5680 if (copy_from_user(buf, ubuf, cnt))
5685 clockstr = strstrip(buf);
5687 ret = tracing_set_clock(tr, clockstr);
5696 static int tracing_clock_open(struct inode *inode, struct file *file)
5698 struct trace_array *tr = inode->i_private;
5701 if (tracing_disabled)
5704 if (trace_array_get(tr))
5707 ret = single_open(file, tracing_clock_show, inode->i_private);
5709 trace_array_put(tr);
5714 struct ftrace_buffer_info {
5715 struct trace_iterator iter;
5720 #ifdef CONFIG_TRACER_SNAPSHOT
5721 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5723 struct trace_array *tr = inode->i_private;
5724 struct trace_iterator *iter;
5728 if (trace_array_get(tr) < 0)
5731 if (file->f_mode & FMODE_READ) {
5732 iter = __tracing_open(inode, file, true);
5734 ret = PTR_ERR(iter);
5736 /* Writes still need the seq_file to hold the private data */
5738 m = kzalloc(sizeof(*m), GFP_KERNEL);
5741 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5749 iter->trace_buffer = &tr->max_buffer;
5750 iter->cpu_file = tracing_get_cpu(inode);
5752 file->private_data = m;
5756 trace_array_put(tr);
5762 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5765 struct seq_file *m = filp->private_data;
5766 struct trace_iterator *iter = m->private;
5767 struct trace_array *tr = iter->tr;
5771 ret = tracing_update_buffers();
5775 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5779 mutex_lock(&trace_types_lock);
5781 if (tr->current_trace->use_max_tr) {
5788 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5792 if (tr->allocated_snapshot)
5796 /* Only allow per-cpu swap if the ring buffer supports it */
5797 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5798 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5803 if (!tr->allocated_snapshot) {
5804 ret = alloc_snapshot(tr);
5808 local_irq_disable();
5809 /* Now, we're going to swap */
5810 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5811 update_max_tr(tr, current, smp_processor_id());
5813 update_max_tr_single(tr, current, iter->cpu_file);
5817 if (tr->allocated_snapshot) {
5818 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5819 tracing_reset_online_cpus(&tr->max_buffer);
5821 tracing_reset(&tr->max_buffer, iter->cpu_file);
5831 mutex_unlock(&trace_types_lock);
5835 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5837 struct seq_file *m = file->private_data;
5840 ret = tracing_release(inode, file);
5842 if (file->f_mode & FMODE_READ)
5845 /* If write only, the seq_file is just a stub */
5853 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5854 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5855 size_t count, loff_t *ppos);
5856 static int tracing_buffers_release(struct inode *inode, struct file *file);
5857 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5858 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5860 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5862 struct ftrace_buffer_info *info;
5865 ret = tracing_buffers_open(inode, filp);
5869 info = filp->private_data;
5871 if (info->iter.trace->use_max_tr) {
5872 tracing_buffers_release(inode, filp);
5876 info->iter.snapshot = true;
5877 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5882 #endif /* CONFIG_TRACER_SNAPSHOT */
5885 static const struct file_operations tracing_thresh_fops = {
5886 .open = tracing_open_generic,
5887 .read = tracing_thresh_read,
5888 .write = tracing_thresh_write,
5889 .llseek = generic_file_llseek,
5892 #ifdef CONFIG_TRACER_MAX_TRACE
5893 static const struct file_operations tracing_max_lat_fops = {
5894 .open = tracing_open_generic,
5895 .read = tracing_max_lat_read,
5896 .write = tracing_max_lat_write,
5897 .llseek = generic_file_llseek,
5901 static const struct file_operations set_tracer_fops = {
5902 .open = tracing_open_generic,
5903 .read = tracing_set_trace_read,
5904 .write = tracing_set_trace_write,
5905 .llseek = generic_file_llseek,
5908 static const struct file_operations tracing_pipe_fops = {
5909 .open = tracing_open_pipe,
5910 .poll = tracing_poll_pipe,
5911 .read = tracing_read_pipe,
5912 .splice_read = tracing_splice_read_pipe,
5913 .release = tracing_release_pipe,
5914 .llseek = no_llseek,
5917 static const struct file_operations tracing_entries_fops = {
5918 .open = tracing_open_generic_tr,
5919 .read = tracing_entries_read,
5920 .write = tracing_entries_write,
5921 .llseek = generic_file_llseek,
5922 .release = tracing_release_generic_tr,
5925 static const struct file_operations tracing_total_entries_fops = {
5926 .open = tracing_open_generic_tr,
5927 .read = tracing_total_entries_read,
5928 .llseek = generic_file_llseek,
5929 .release = tracing_release_generic_tr,
5932 static const struct file_operations tracing_free_buffer_fops = {
5933 .open = tracing_open_generic_tr,
5934 .write = tracing_free_buffer_write,
5935 .release = tracing_free_buffer_release,
5938 static const struct file_operations tracing_mark_fops = {
5939 .open = tracing_open_generic_tr,
5940 .write = tracing_mark_write,
5941 .llseek = generic_file_llseek,
5942 .release = tracing_release_generic_tr,
5945 static const struct file_operations trace_clock_fops = {
5946 .open = tracing_clock_open,
5948 .llseek = seq_lseek,
5949 .release = tracing_single_release_tr,
5950 .write = tracing_clock_write,
5953 #ifdef CONFIG_TRACER_SNAPSHOT
5954 static const struct file_operations snapshot_fops = {
5955 .open = tracing_snapshot_open,
5957 .write = tracing_snapshot_write,
5958 .llseek = tracing_lseek,
5959 .release = tracing_snapshot_release,
5962 static const struct file_operations snapshot_raw_fops = {
5963 .open = snapshot_raw_open,
5964 .read = tracing_buffers_read,
5965 .release = tracing_buffers_release,
5966 .splice_read = tracing_buffers_splice_read,
5967 .llseek = no_llseek,
5970 #endif /* CONFIG_TRACER_SNAPSHOT */
5972 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5974 struct trace_array *tr = inode->i_private;
5975 struct ftrace_buffer_info *info;
5978 if (tracing_disabled)
5981 if (trace_array_get(tr) < 0)
5984 info = kzalloc(sizeof(*info), GFP_KERNEL);
5986 trace_array_put(tr);
5990 mutex_lock(&trace_types_lock);
5993 info->iter.cpu_file = tracing_get_cpu(inode);
5994 info->iter.trace = tr->current_trace;
5995 info->iter.trace_buffer = &tr->trace_buffer;
5997 /* Force reading ring buffer for first read */
5998 info->read = (unsigned int)-1;
6000 filp->private_data = info;
6002 tr->current_trace->ref++;
6004 mutex_unlock(&trace_types_lock);
6006 ret = nonseekable_open(inode, filp);
6008 trace_array_put(tr);
6014 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6016 struct ftrace_buffer_info *info = filp->private_data;
6017 struct trace_iterator *iter = &info->iter;
6019 return trace_poll(iter, filp, poll_table);
6023 tracing_buffers_read(struct file *filp, char __user *ubuf,
6024 size_t count, loff_t *ppos)
6026 struct ftrace_buffer_info *info = filp->private_data;
6027 struct trace_iterator *iter = &info->iter;
6034 #ifdef CONFIG_TRACER_MAX_TRACE
6035 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6040 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6045 /* Do we have previous read data to read? */
6046 if (info->read < PAGE_SIZE)
6050 trace_access_lock(iter->cpu_file);
6051 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6055 trace_access_unlock(iter->cpu_file);
6058 if (trace_empty(iter)) {
6059 if ((filp->f_flags & O_NONBLOCK))
6062 ret = wait_on_pipe(iter, false);
6073 size = PAGE_SIZE - info->read;
6077 ret = copy_to_user(ubuf, info->spare + info->read, size);
6089 static int tracing_buffers_release(struct inode *inode, struct file *file)
6091 struct ftrace_buffer_info *info = file->private_data;
6092 struct trace_iterator *iter = &info->iter;
6094 mutex_lock(&trace_types_lock);
6096 iter->tr->current_trace->ref--;
6098 __trace_array_put(iter->tr);
6101 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
6104 mutex_unlock(&trace_types_lock);
6110 struct ring_buffer *buffer;
6115 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6116 struct pipe_buffer *buf)
6118 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6123 ring_buffer_free_read_page(ref->buffer, ref->page);
6128 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6129 struct pipe_buffer *buf)
6131 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6136 /* Pipe buffer operations for a buffer. */
6137 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6139 .confirm = generic_pipe_buf_confirm,
6140 .release = buffer_pipe_buf_release,
6141 .steal = generic_pipe_buf_steal,
6142 .get = buffer_pipe_buf_get,
6146 * Callback from splice_to_pipe(), if we need to release some pages
6147 * at the end of the spd in case we error'ed out in filling the pipe.
6149 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6151 struct buffer_ref *ref =
6152 (struct buffer_ref *)spd->partial[i].private;
6157 ring_buffer_free_read_page(ref->buffer, ref->page);
6159 spd->partial[i].private = 0;
6163 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6164 struct pipe_inode_info *pipe, size_t len,
6167 struct ftrace_buffer_info *info = file->private_data;
6168 struct trace_iterator *iter = &info->iter;
6169 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6170 struct page *pages_def[PIPE_DEF_BUFFERS];
6171 struct splice_pipe_desc spd = {
6173 .partial = partial_def,
6174 .nr_pages_max = PIPE_DEF_BUFFERS,
6176 .ops = &buffer_pipe_buf_ops,
6177 .spd_release = buffer_spd_release,
6179 struct buffer_ref *ref;
6180 int entries, size, i;
6183 #ifdef CONFIG_TRACER_MAX_TRACE
6184 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6188 if (splice_grow_spd(pipe, &spd))
6191 if (*ppos & (PAGE_SIZE - 1))
6194 if (len & (PAGE_SIZE - 1)) {
6195 if (len < PAGE_SIZE)
6201 trace_access_lock(iter->cpu_file);
6202 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6204 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6208 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
6215 ref->buffer = iter->trace_buffer->buffer;
6216 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6223 r = ring_buffer_read_page(ref->buffer, &ref->page,
6224 len, iter->cpu_file, 1);
6226 ring_buffer_free_read_page(ref->buffer, ref->page);
6232 * zero out any left over data, this is going to
6235 size = ring_buffer_page_len(ref->page);
6236 if (size < PAGE_SIZE)
6237 memset(ref->page + size, 0, PAGE_SIZE - size);
6239 page = virt_to_page(ref->page);
6241 spd.pages[i] = page;
6242 spd.partial[i].len = PAGE_SIZE;
6243 spd.partial[i].offset = 0;
6244 spd.partial[i].private = (unsigned long)ref;
6248 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6251 trace_access_unlock(iter->cpu_file);
6254 /* did we read anything? */
6255 if (!spd.nr_pages) {
6259 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6262 ret = wait_on_pipe(iter, true);
6269 ret = splice_to_pipe(pipe, &spd);
6270 splice_shrink_spd(&spd);
6275 static const struct file_operations tracing_buffers_fops = {
6276 .open = tracing_buffers_open,
6277 .read = tracing_buffers_read,
6278 .poll = tracing_buffers_poll,
6279 .release = tracing_buffers_release,
6280 .splice_read = tracing_buffers_splice_read,
6281 .llseek = no_llseek,
6285 tracing_stats_read(struct file *filp, char __user *ubuf,
6286 size_t count, loff_t *ppos)
6288 struct inode *inode = file_inode(filp);
6289 struct trace_array *tr = inode->i_private;
6290 struct trace_buffer *trace_buf = &tr->trace_buffer;
6291 int cpu = tracing_get_cpu(inode);
6292 struct trace_seq *s;
6294 unsigned long long t;
6295 unsigned long usec_rem;
6297 s = kmalloc(sizeof(*s), GFP_KERNEL);
6303 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6304 trace_seq_printf(s, "entries: %ld\n", cnt);
6306 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6307 trace_seq_printf(s, "overrun: %ld\n", cnt);
6309 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6310 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6312 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6313 trace_seq_printf(s, "bytes: %ld\n", cnt);
6315 if (trace_clocks[tr->clock_id].in_ns) {
6316 /* local or global for trace_clock */
6317 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6318 usec_rem = do_div(t, USEC_PER_SEC);
6319 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6322 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6323 usec_rem = do_div(t, USEC_PER_SEC);
6324 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6326 /* counter or tsc mode for trace_clock */
6327 trace_seq_printf(s, "oldest event ts: %llu\n",
6328 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6330 trace_seq_printf(s, "now ts: %llu\n",
6331 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6334 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6335 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6337 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6338 trace_seq_printf(s, "read events: %ld\n", cnt);
6340 count = simple_read_from_buffer(ubuf, count, ppos,
6341 s->buffer, trace_seq_used(s));
6348 static const struct file_operations tracing_stats_fops = {
6349 .open = tracing_open_generic_tr,
6350 .read = tracing_stats_read,
6351 .llseek = generic_file_llseek,
6352 .release = tracing_release_generic_tr,
6355 #ifdef CONFIG_DYNAMIC_FTRACE
6357 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6363 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6364 size_t cnt, loff_t *ppos)
6366 static char ftrace_dyn_info_buffer[1024];
6367 static DEFINE_MUTEX(dyn_info_mutex);
6368 unsigned long *p = filp->private_data;
6369 char *buf = ftrace_dyn_info_buffer;
6370 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
6373 mutex_lock(&dyn_info_mutex);
6374 r = sprintf(buf, "%ld ", *p);
6376 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
6379 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6381 mutex_unlock(&dyn_info_mutex);
6386 static const struct file_operations tracing_dyn_info_fops = {
6387 .open = tracing_open_generic,
6388 .read = tracing_read_dyn_info,
6389 .llseek = generic_file_llseek,
6391 #endif /* CONFIG_DYNAMIC_FTRACE */
6393 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6395 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6401 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6403 unsigned long *count = (long *)data;
6415 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6416 struct ftrace_probe_ops *ops, void *data)
6418 long count = (long)data;
6420 seq_printf(m, "%ps:", (void *)ip);
6422 seq_puts(m, "snapshot");
6425 seq_puts(m, ":unlimited\n");
6427 seq_printf(m, ":count=%ld\n", count);
6432 static struct ftrace_probe_ops snapshot_probe_ops = {
6433 .func = ftrace_snapshot,
6434 .print = ftrace_snapshot_print,
6437 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6438 .func = ftrace_count_snapshot,
6439 .print = ftrace_snapshot_print,
6443 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6444 char *glob, char *cmd, char *param, int enable)
6446 struct ftrace_probe_ops *ops;
6447 void *count = (void *)-1;
6451 /* hash funcs only work with set_ftrace_filter */
6455 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6457 if (glob[0] == '!') {
6458 unregister_ftrace_function_probe_func(glob+1, ops);
6465 number = strsep(¶m, ":");
6467 if (!strlen(number))
6471 * We use the callback data field (which is a pointer)
6474 ret = kstrtoul(number, 0, (unsigned long *)&count);
6479 ret = register_ftrace_function_probe(glob, ops, count);
6482 alloc_snapshot(&global_trace);
6484 return ret < 0 ? ret : 0;
6487 static struct ftrace_func_command ftrace_snapshot_cmd = {
6489 .func = ftrace_trace_snapshot_callback,
6492 static __init int register_snapshot_cmd(void)
6494 return register_ftrace_command(&ftrace_snapshot_cmd);
6497 static inline __init int register_snapshot_cmd(void) { return 0; }
6498 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6500 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6502 if (WARN_ON(!tr->dir))
6503 return ERR_PTR(-ENODEV);
6505 /* Top directory uses NULL as the parent */
6506 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6509 /* All sub buffers have a descriptor */
6513 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6515 struct dentry *d_tracer;
6518 return tr->percpu_dir;
6520 d_tracer = tracing_get_dentry(tr);
6521 if (IS_ERR(d_tracer))
6524 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6526 WARN_ONCE(!tr->percpu_dir,
6527 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6529 return tr->percpu_dir;
6532 static struct dentry *
6533 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6534 void *data, long cpu, const struct file_operations *fops)
6536 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6538 if (ret) /* See tracing_get_cpu() */
6539 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6544 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6546 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6547 struct dentry *d_cpu;
6548 char cpu_dir[30]; /* 30 characters should be more than enough */
6553 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6554 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6556 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6560 /* per cpu trace_pipe */
6561 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6562 tr, cpu, &tracing_pipe_fops);
6565 trace_create_cpu_file("trace", 0644, d_cpu,
6566 tr, cpu, &tracing_fops);
6568 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6569 tr, cpu, &tracing_buffers_fops);
6571 trace_create_cpu_file("stats", 0444, d_cpu,
6572 tr, cpu, &tracing_stats_fops);
6574 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6575 tr, cpu, &tracing_entries_fops);
6577 #ifdef CONFIG_TRACER_SNAPSHOT
6578 trace_create_cpu_file("snapshot", 0644, d_cpu,
6579 tr, cpu, &snapshot_fops);
6581 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6582 tr, cpu, &snapshot_raw_fops);
6586 #ifdef CONFIG_FTRACE_SELFTEST
6587 /* Let selftest have access to static functions in this file */
6588 #include "trace_selftest.c"
6592 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6595 struct trace_option_dentry *topt = filp->private_data;
6598 if (topt->flags->val & topt->opt->bit)
6603 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6607 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6610 struct trace_option_dentry *topt = filp->private_data;
6614 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6618 if (val != 0 && val != 1)
6621 if (!!(topt->flags->val & topt->opt->bit) != val) {
6622 mutex_lock(&trace_types_lock);
6623 ret = __set_tracer_option(topt->tr, topt->flags,
6625 mutex_unlock(&trace_types_lock);
6636 static const struct file_operations trace_options_fops = {
6637 .open = tracing_open_generic,
6638 .read = trace_options_read,
6639 .write = trace_options_write,
6640 .llseek = generic_file_llseek,
6644 * In order to pass in both the trace_array descriptor as well as the index
6645 * to the flag that the trace option file represents, the trace_array
6646 * has a character array of trace_flags_index[], which holds the index
6647 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6648 * The address of this character array is passed to the flag option file
6649 * read/write callbacks.
6651 * In order to extract both the index and the trace_array descriptor,
6652 * get_tr_index() uses the following algorithm.
6656 * As the pointer itself contains the address of the index (remember
6659 * Then to get the trace_array descriptor, by subtracting that index
6660 * from the ptr, we get to the start of the index itself.
6662 * ptr - idx == &index[0]
6664 * Then a simple container_of() from that pointer gets us to the
6665 * trace_array descriptor.
6667 static void get_tr_index(void *data, struct trace_array **ptr,
6668 unsigned int *pindex)
6670 *pindex = *(unsigned char *)data;
6672 *ptr = container_of(data - *pindex, struct trace_array,
6677 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6680 void *tr_index = filp->private_data;
6681 struct trace_array *tr;
6685 get_tr_index(tr_index, &tr, &index);
6687 if (tr->trace_flags & (1 << index))
6692 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6696 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6699 void *tr_index = filp->private_data;
6700 struct trace_array *tr;
6705 get_tr_index(tr_index, &tr, &index);
6707 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6711 if (val != 0 && val != 1)
6714 mutex_lock(&trace_types_lock);
6715 ret = set_tracer_flag(tr, 1 << index, val);
6716 mutex_unlock(&trace_types_lock);
6726 static const struct file_operations trace_options_core_fops = {
6727 .open = tracing_open_generic,
6728 .read = trace_options_core_read,
6729 .write = trace_options_core_write,
6730 .llseek = generic_file_llseek,
6733 struct dentry *trace_create_file(const char *name,
6735 struct dentry *parent,
6737 const struct file_operations *fops)
6741 ret = tracefs_create_file(name, mode, parent, data, fops);
6743 pr_warn("Could not create tracefs '%s' entry\n", name);
6749 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6751 struct dentry *d_tracer;
6756 d_tracer = tracing_get_dentry(tr);
6757 if (IS_ERR(d_tracer))
6760 tr->options = tracefs_create_dir("options", d_tracer);
6762 pr_warn("Could not create tracefs directory 'options'\n");
6770 create_trace_option_file(struct trace_array *tr,
6771 struct trace_option_dentry *topt,
6772 struct tracer_flags *flags,
6773 struct tracer_opt *opt)
6775 struct dentry *t_options;
6777 t_options = trace_options_init_dentry(tr);
6781 topt->flags = flags;
6785 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6786 &trace_options_fops);
6791 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6793 struct trace_option_dentry *topts;
6794 struct trace_options *tr_topts;
6795 struct tracer_flags *flags;
6796 struct tracer_opt *opts;
6803 flags = tracer->flags;
6805 if (!flags || !flags->opts)
6809 * If this is an instance, only create flags for tracers
6810 * the instance may have.
6812 if (!trace_ok_for_array(tracer, tr))
6815 for (i = 0; i < tr->nr_topts; i++) {
6816 /* Make sure there's no duplicate flags. */
6817 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6823 for (cnt = 0; opts[cnt].name; cnt++)
6826 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6830 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6837 tr->topts = tr_topts;
6838 tr->topts[tr->nr_topts].tracer = tracer;
6839 tr->topts[tr->nr_topts].topts = topts;
6842 for (cnt = 0; opts[cnt].name; cnt++) {
6843 create_trace_option_file(tr, &topts[cnt], flags,
6845 WARN_ONCE(topts[cnt].entry == NULL,
6846 "Failed to create trace option: %s",
6851 static struct dentry *
6852 create_trace_option_core_file(struct trace_array *tr,
6853 const char *option, long index)
6855 struct dentry *t_options;
6857 t_options = trace_options_init_dentry(tr);
6861 return trace_create_file(option, 0644, t_options,
6862 (void *)&tr->trace_flags_index[index],
6863 &trace_options_core_fops);
6866 static void create_trace_options_dir(struct trace_array *tr)
6868 struct dentry *t_options;
6869 bool top_level = tr == &global_trace;
6872 t_options = trace_options_init_dentry(tr);
6876 for (i = 0; trace_options[i]; i++) {
6878 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6879 create_trace_option_core_file(tr, trace_options[i], i);
6884 rb_simple_read(struct file *filp, char __user *ubuf,
6885 size_t cnt, loff_t *ppos)
6887 struct trace_array *tr = filp->private_data;
6891 r = tracer_tracing_is_on(tr);
6892 r = sprintf(buf, "%d\n", r);
6894 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6898 rb_simple_write(struct file *filp, const char __user *ubuf,
6899 size_t cnt, loff_t *ppos)
6901 struct trace_array *tr = filp->private_data;
6902 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6906 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6911 mutex_lock(&trace_types_lock);
6913 tracer_tracing_on(tr);
6914 if (tr->current_trace->start)
6915 tr->current_trace->start(tr);
6917 tracer_tracing_off(tr);
6918 if (tr->current_trace->stop)
6919 tr->current_trace->stop(tr);
6921 mutex_unlock(&trace_types_lock);
6929 static const struct file_operations rb_simple_fops = {
6930 .open = tracing_open_generic_tr,
6931 .read = rb_simple_read,
6932 .write = rb_simple_write,
6933 .release = tracing_release_generic_tr,
6934 .llseek = default_llseek,
6937 struct dentry *trace_instance_dir;
6940 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6943 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6945 enum ring_buffer_flags rb_flags;
6947 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6951 buf->buffer = ring_buffer_alloc(size, rb_flags);
6955 buf->data = alloc_percpu(struct trace_array_cpu);
6957 ring_buffer_free(buf->buffer);
6961 /* Allocate the first page for all buffers */
6962 set_buffer_entries(&tr->trace_buffer,
6963 ring_buffer_size(tr->trace_buffer.buffer, 0));
6968 static int allocate_trace_buffers(struct trace_array *tr, int size)
6972 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6976 #ifdef CONFIG_TRACER_MAX_TRACE
6977 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6978 allocate_snapshot ? size : 1);
6980 ring_buffer_free(tr->trace_buffer.buffer);
6981 free_percpu(tr->trace_buffer.data);
6984 tr->allocated_snapshot = allocate_snapshot;
6987 * Only the top level trace array gets its snapshot allocated
6988 * from the kernel command line.
6990 allocate_snapshot = false;
6995 static void free_trace_buffer(struct trace_buffer *buf)
6998 ring_buffer_free(buf->buffer);
7000 free_percpu(buf->data);
7005 static void free_trace_buffers(struct trace_array *tr)
7010 free_trace_buffer(&tr->trace_buffer);
7012 #ifdef CONFIG_TRACER_MAX_TRACE
7013 free_trace_buffer(&tr->max_buffer);
7017 static void init_trace_flags_index(struct trace_array *tr)
7021 /* Used by the trace options files */
7022 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7023 tr->trace_flags_index[i] = i;
7026 static void __update_tracer_options(struct trace_array *tr)
7030 for (t = trace_types; t; t = t->next)
7031 add_tracer_options(tr, t);
7034 static void update_tracer_options(struct trace_array *tr)
7036 mutex_lock(&trace_types_lock);
7037 __update_tracer_options(tr);
7038 mutex_unlock(&trace_types_lock);
7041 static int instance_mkdir(const char *name)
7043 struct trace_array *tr;
7046 mutex_lock(&trace_types_lock);
7049 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7050 if (tr->name && strcmp(tr->name, name) == 0)
7055 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7059 tr->name = kstrdup(name, GFP_KERNEL);
7063 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7066 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
7068 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7070 raw_spin_lock_init(&tr->start_lock);
7072 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7074 tr->current_trace = &nop_trace;
7076 INIT_LIST_HEAD(&tr->systems);
7077 INIT_LIST_HEAD(&tr->events);
7079 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7082 tr->dir = tracefs_create_dir(name, trace_instance_dir);
7086 ret = event_trace_add_tracer(tr->dir, tr);
7088 tracefs_remove_recursive(tr->dir);
7092 init_tracer_tracefs(tr, tr->dir);
7093 init_trace_flags_index(tr);
7094 __update_tracer_options(tr);
7096 list_add(&tr->list, &ftrace_trace_arrays);
7098 mutex_unlock(&trace_types_lock);
7103 free_trace_buffers(tr);
7104 free_cpumask_var(tr->tracing_cpumask);
7109 mutex_unlock(&trace_types_lock);
7115 static int instance_rmdir(const char *name)
7117 struct trace_array *tr;
7122 mutex_lock(&trace_types_lock);
7125 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7126 if (tr->name && strcmp(tr->name, name) == 0) {
7135 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
7138 list_del(&tr->list);
7140 /* Disable all the flags that were enabled coming in */
7141 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7142 if ((1 << i) & ZEROED_TRACE_FLAGS)
7143 set_tracer_flag(tr, 1 << i, 0);
7146 tracing_set_nop(tr);
7147 event_trace_del_tracer(tr);
7148 ftrace_destroy_function_files(tr);
7149 tracefs_remove_recursive(tr->dir);
7150 free_trace_buffers(tr);
7152 for (i = 0; i < tr->nr_topts; i++) {
7153 kfree(tr->topts[i].topts);
7163 mutex_unlock(&trace_types_lock);
7168 static __init void create_trace_instances(struct dentry *d_tracer)
7170 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7173 if (WARN_ON(!trace_instance_dir))
7178 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7182 trace_create_file("available_tracers", 0444, d_tracer,
7183 tr, &show_traces_fops);
7185 trace_create_file("current_tracer", 0644, d_tracer,
7186 tr, &set_tracer_fops);
7188 trace_create_file("tracing_cpumask", 0644, d_tracer,
7189 tr, &tracing_cpumask_fops);
7191 trace_create_file("trace_options", 0644, d_tracer,
7192 tr, &tracing_iter_fops);
7194 trace_create_file("trace", 0644, d_tracer,
7197 trace_create_file("trace_pipe", 0444, d_tracer,
7198 tr, &tracing_pipe_fops);
7200 trace_create_file("buffer_size_kb", 0644, d_tracer,
7201 tr, &tracing_entries_fops);
7203 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7204 tr, &tracing_total_entries_fops);
7206 trace_create_file("free_buffer", 0200, d_tracer,
7207 tr, &tracing_free_buffer_fops);
7209 trace_create_file("trace_marker", 0220, d_tracer,
7210 tr, &tracing_mark_fops);
7212 trace_create_file("trace_clock", 0644, d_tracer, tr,
7215 trace_create_file("tracing_on", 0644, d_tracer,
7216 tr, &rb_simple_fops);
7218 create_trace_options_dir(tr);
7220 #ifdef CONFIG_TRACER_MAX_TRACE
7221 trace_create_file("tracing_max_latency", 0644, d_tracer,
7222 &tr->max_latency, &tracing_max_lat_fops);
7225 if (ftrace_create_function_files(tr, d_tracer))
7226 WARN(1, "Could not allocate function filter files");
7228 #ifdef CONFIG_TRACER_SNAPSHOT
7229 trace_create_file("snapshot", 0644, d_tracer,
7230 tr, &snapshot_fops);
7233 for_each_tracing_cpu(cpu)
7234 tracing_init_tracefs_percpu(tr, cpu);
7236 ftrace_init_tracefs(tr, d_tracer);
7239 static struct vfsmount *trace_automount(void *ingore)
7241 struct vfsmount *mnt;
7242 struct file_system_type *type;
7245 * To maintain backward compatibility for tools that mount
7246 * debugfs to get to the tracing facility, tracefs is automatically
7247 * mounted to the debugfs/tracing directory.
7249 type = get_fs_type("tracefs");
7252 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
7253 put_filesystem(type);
7262 * tracing_init_dentry - initialize top level trace array
7264 * This is called when creating files or directories in the tracing
7265 * directory. It is called via fs_initcall() by any of the boot up code
7266 * and expects to return the dentry of the top level tracing directory.
7268 struct dentry *tracing_init_dentry(void)
7270 struct trace_array *tr = &global_trace;
7272 /* The top level trace array uses NULL as parent */
7276 if (WARN_ON(!tracefs_initialized()) ||
7277 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7278 WARN_ON(!debugfs_initialized())))
7279 return ERR_PTR(-ENODEV);
7282 * As there may still be users that expect the tracing
7283 * files to exist in debugfs/tracing, we must automount
7284 * the tracefs file system there, so older tools still
7285 * work with the newer kerenl.
7287 tr->dir = debugfs_create_automount("tracing", NULL,
7288 trace_automount, NULL);
7290 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7291 return ERR_PTR(-ENOMEM);
7297 extern struct trace_enum_map *__start_ftrace_enum_maps[];
7298 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7300 static void __init trace_enum_init(void)
7304 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
7305 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
7308 #ifdef CONFIG_MODULES
7309 static void trace_module_add_enums(struct module *mod)
7311 if (!mod->num_trace_enums)
7315 * Modules with bad taint do not have events created, do
7316 * not bother with enums either.
7318 if (trace_module_has_bad_taint(mod))
7321 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
7324 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7325 static void trace_module_remove_enums(struct module *mod)
7327 union trace_enum_map_item *map;
7328 union trace_enum_map_item **last = &trace_enum_maps;
7330 if (!mod->num_trace_enums)
7333 mutex_lock(&trace_enum_mutex);
7335 map = trace_enum_maps;
7338 if (map->head.mod == mod)
7340 map = trace_enum_jmp_to_tail(map);
7341 last = &map->tail.next;
7342 map = map->tail.next;
7347 *last = trace_enum_jmp_to_tail(map)->tail.next;
7350 mutex_unlock(&trace_enum_mutex);
7353 static inline void trace_module_remove_enums(struct module *mod) { }
7354 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7356 static int trace_module_notify(struct notifier_block *self,
7357 unsigned long val, void *data)
7359 struct module *mod = data;
7362 case MODULE_STATE_COMING:
7363 trace_module_add_enums(mod);
7365 case MODULE_STATE_GOING:
7366 trace_module_remove_enums(mod);
7373 static struct notifier_block trace_module_nb = {
7374 .notifier_call = trace_module_notify,
7377 #endif /* CONFIG_MODULES */
7379 static __init int tracer_init_tracefs(void)
7381 struct dentry *d_tracer;
7383 trace_access_lock_init();
7385 d_tracer = tracing_init_dentry();
7386 if (IS_ERR(d_tracer))
7389 init_tracer_tracefs(&global_trace, d_tracer);
7391 trace_create_file("tracing_thresh", 0644, d_tracer,
7392 &global_trace, &tracing_thresh_fops);
7394 trace_create_file("README", 0444, d_tracer,
7395 NULL, &tracing_readme_fops);
7397 trace_create_file("saved_cmdlines", 0444, d_tracer,
7398 NULL, &tracing_saved_cmdlines_fops);
7400 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7401 NULL, &tracing_saved_cmdlines_size_fops);
7405 trace_create_enum_file(d_tracer);
7407 #ifdef CONFIG_MODULES
7408 register_module_notifier(&trace_module_nb);
7411 #ifdef CONFIG_DYNAMIC_FTRACE
7412 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7413 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7416 create_trace_instances(d_tracer);
7418 update_tracer_options(&global_trace);
7423 static int trace_panic_handler(struct notifier_block *this,
7424 unsigned long event, void *unused)
7426 if (ftrace_dump_on_oops)
7427 ftrace_dump(ftrace_dump_on_oops);
7431 static struct notifier_block trace_panic_notifier = {
7432 .notifier_call = trace_panic_handler,
7434 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7437 static int trace_die_handler(struct notifier_block *self,
7443 if (ftrace_dump_on_oops)
7444 ftrace_dump(ftrace_dump_on_oops);
7452 static struct notifier_block trace_die_notifier = {
7453 .notifier_call = trace_die_handler,
7458 * printk is set to max of 1024, we really don't need it that big.
7459 * Nothing should be printing 1000 characters anyway.
7461 #define TRACE_MAX_PRINT 1000
7464 * Define here KERN_TRACE so that we have one place to modify
7465 * it if we decide to change what log level the ftrace dump
7468 #define KERN_TRACE KERN_EMERG
7471 trace_printk_seq(struct trace_seq *s)
7473 /* Probably should print a warning here. */
7474 if (s->seq.len >= TRACE_MAX_PRINT)
7475 s->seq.len = TRACE_MAX_PRINT;
7478 * More paranoid code. Although the buffer size is set to
7479 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7480 * an extra layer of protection.
7482 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7483 s->seq.len = s->seq.size - 1;
7485 /* should be zero ended, but we are paranoid. */
7486 s->buffer[s->seq.len] = 0;
7488 printk(KERN_TRACE "%s", s->buffer);
7493 void trace_init_global_iter(struct trace_iterator *iter)
7495 iter->tr = &global_trace;
7496 iter->trace = iter->tr->current_trace;
7497 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7498 iter->trace_buffer = &global_trace.trace_buffer;
7500 if (iter->trace && iter->trace->open)
7501 iter->trace->open(iter);
7503 /* Annotate start of buffers if we had overruns */
7504 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7505 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7507 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7508 if (trace_clocks[iter->tr->clock_id].in_ns)
7509 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7512 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7514 /* use static because iter can be a bit big for the stack */
7515 static struct trace_iterator iter;
7516 static atomic_t dump_running;
7517 struct trace_array *tr = &global_trace;
7518 unsigned int old_userobj;
7519 unsigned long flags;
7522 /* Only allow one dump user at a time. */
7523 if (atomic_inc_return(&dump_running) != 1) {
7524 atomic_dec(&dump_running);
7529 * Always turn off tracing when we dump.
7530 * We don't need to show trace output of what happens
7531 * between multiple crashes.
7533 * If the user does a sysrq-z, then they can re-enable
7534 * tracing with echo 1 > tracing_on.
7538 local_irq_save(flags);
7540 /* Simulate the iterator */
7541 trace_init_global_iter(&iter);
7543 for_each_tracing_cpu(cpu) {
7544 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7547 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7549 /* don't look at user memory in panic mode */
7550 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7552 switch (oops_dump_mode) {
7554 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7557 iter.cpu_file = raw_smp_processor_id();
7562 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7563 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7566 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7568 /* Did function tracer already get disabled? */
7569 if (ftrace_is_dead()) {
7570 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7571 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7575 * We need to stop all tracing on all CPUS to read the
7576 * the next buffer. This is a bit expensive, but is
7577 * not done often. We fill all what we can read,
7578 * and then release the locks again.
7581 while (!trace_empty(&iter)) {
7584 printk(KERN_TRACE "---------------------------------\n");
7588 /* reset all but tr, trace, and overruns */
7589 memset(&iter.seq, 0,
7590 sizeof(struct trace_iterator) -
7591 offsetof(struct trace_iterator, seq));
7592 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7595 if (trace_find_next_entry_inc(&iter) != NULL) {
7598 ret = print_trace_line(&iter);
7599 if (ret != TRACE_TYPE_NO_CONSUME)
7600 trace_consume(&iter);
7602 touch_nmi_watchdog();
7604 trace_printk_seq(&iter.seq);
7608 printk(KERN_TRACE " (ftrace buffer empty)\n");
7610 printk(KERN_TRACE "---------------------------------\n");
7613 tr->trace_flags |= old_userobj;
7615 for_each_tracing_cpu(cpu) {
7616 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7618 atomic_dec(&dump_running);
7619 local_irq_restore(flags);
7621 EXPORT_SYMBOL_GPL(ftrace_dump);
7623 __init static int tracer_alloc_buffers(void)
7629 * Make sure we don't accidently add more trace options
7630 * than we have bits for.
7632 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7634 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7637 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7638 goto out_free_buffer_mask;
7640 /* Only allocate trace_printk buffers if a trace_printk exists */
7641 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7642 /* Must be called before global_trace.buffer is allocated */
7643 trace_printk_init_buffers();
7645 /* To save memory, keep the ring buffer size to its minimum */
7646 if (ring_buffer_expanded)
7647 ring_buf_size = trace_buf_size;
7651 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7652 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7654 raw_spin_lock_init(&global_trace.start_lock);
7656 /* Used for event triggers */
7657 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7659 goto out_free_cpumask;
7661 if (trace_create_savedcmd() < 0)
7662 goto out_free_temp_buffer;
7664 /* TODO: make the number of buffers hot pluggable with CPUS */
7665 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7666 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7668 goto out_free_savedcmd;
7671 if (global_trace.buffer_disabled)
7674 if (trace_boot_clock) {
7675 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7677 pr_warn("Trace clock %s not defined, going back to default\n",
7682 * register_tracer() might reference current_trace, so it
7683 * needs to be set before we register anything. This is
7684 * just a bootstrap of current_trace anyway.
7686 global_trace.current_trace = &nop_trace;
7688 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7690 ftrace_init_global_array_ops(&global_trace);
7692 init_trace_flags_index(&global_trace);
7694 register_tracer(&nop_trace);
7696 /* All seems OK, enable tracing */
7697 tracing_disabled = 0;
7699 atomic_notifier_chain_register(&panic_notifier_list,
7700 &trace_panic_notifier);
7702 register_die_notifier(&trace_die_notifier);
7704 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7706 INIT_LIST_HEAD(&global_trace.systems);
7707 INIT_LIST_HEAD(&global_trace.events);
7708 list_add(&global_trace.list, &ftrace_trace_arrays);
7710 apply_trace_boot_options();
7712 register_snapshot_cmd();
7717 free_saved_cmdlines_buffer(savedcmd);
7718 out_free_temp_buffer:
7719 ring_buffer_free(temp_buffer);
7721 free_cpumask_var(global_trace.tracing_cpumask);
7722 out_free_buffer_mask:
7723 free_cpumask_var(tracing_buffer_mask);
7728 void __init trace_init(void)
7730 if (tracepoint_printk) {
7731 tracepoint_print_iter =
7732 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7733 if (WARN_ON(!tracepoint_print_iter))
7734 tracepoint_printk = 0;
7736 tracer_alloc_buffers();
7740 __init static int clear_boot_tracer(void)
7743 * The default tracer at boot buffer is an init section.
7744 * This function is called in lateinit. If we did not
7745 * find the boot tracer, then clear it out, to prevent
7746 * later registration from accessing the buffer that is
7747 * about to be freed.
7749 if (!default_bootup_tracer)
7752 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7753 default_bootup_tracer);
7754 default_bootup_tracer = NULL;
7759 fs_initcall(tracer_init_tracefs);
7760 late_initcall(clear_boot_tracer);