Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net

[cascardo/linux.git] / kernel / events / core.c

diff --git a/kernel/events/core.c b/kernel/events/core.c
index a54f2c2..9fc3be0 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7049,7 +7049,7 @@ static int __perf_event_overflow(struct perf_event *event,
                irq_work_queue(&event->pending);
        }
 
-       event->overflow_handler(event, data, regs);
+       READ_ONCE(event->overflow_handler)(event, data, regs);
 
        if (*perf_event_fasync(event) && event->pending_kill) {
                event->pending_wakeup = 1;
@@ -7664,11 +7664,83 @@ static void perf_event_free_filter(struct perf_event *event)
        ftrace_profile_free_filter(event);
 }
 
+#ifdef CONFIG_BPF_SYSCALL
+static void bpf_overflow_handler(struct perf_event *event,
+                                struct perf_sample_data *data,
+                                struct pt_regs *regs)
+{
+       struct bpf_perf_event_data_kern ctx = {
+               .data = data,
+               .regs = regs,
+       };
+       int ret = 0;
+
+       preempt_disable();
+       if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
+               goto out;
+       rcu_read_lock();
+       ret = BPF_PROG_RUN(event->prog, (void *)&ctx);
+       rcu_read_unlock();
+out:
+       __this_cpu_dec(bpf_prog_active);
+       preempt_enable();
+       if (!ret)
+               return;
+
+       event->orig_overflow_handler(event, data, regs);
+}
+
+static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
+{
+       struct bpf_prog *prog;
+
+       if (event->overflow_handler_context)
+               /* hw breakpoint or kernel counter */
+               return -EINVAL;
+
+       if (event->prog)
+               return -EEXIST;
+
+       prog = bpf_prog_get_type(prog_fd, BPF_PROG_TYPE_PERF_EVENT);
+       if (IS_ERR(prog))
+               return PTR_ERR(prog);
+
+       event->prog = prog;
+       event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
+       WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
+       return 0;
+}
+
+static void perf_event_free_bpf_handler(struct perf_event *event)
+{
+       struct bpf_prog *prog = event->prog;
+
+       if (!prog)
+               return;
+
+       WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);
+       event->prog = NULL;
+       bpf_prog_put(prog);
+}
+#else
+static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
+{
+       return -EOPNOTSUPP;
+}
+static void perf_event_free_bpf_handler(struct perf_event *event)
+{
+}
+#endif
+
 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
 {
        bool is_kprobe, is_tracepoint;
        struct bpf_prog *prog;
 
+       if (event->attr.type == PERF_TYPE_HARDWARE ||
+           event->attr.type == PERF_TYPE_SOFTWARE)
+               return perf_event_set_bpf_handler(event, prog_fd);
+
        if (event->attr.type != PERF_TYPE_TRACEPOINT)
                return -EINVAL;
 
@@ -7709,6 +7781,8 @@ static void perf_event_free_bpf_prog(struct perf_event *event)
 {
        struct bpf_prog *prog;
 
+       perf_event_free_bpf_handler(event);
+
        if (!event->tp_event)
                return;
 
@@ -9025,6 +9099,19 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
        if (!overflow_handler && parent_event) {
                overflow_handler = parent_event->overflow_handler;
                context = parent_event->overflow_handler_context;
+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
+               if (overflow_handler == bpf_overflow_handler) {
+                       struct bpf_prog *prog = bpf_prog_inc(parent_event->prog);
+
+                       if (IS_ERR(prog)) {
+                               err = PTR_ERR(prog);
+                               goto err_ns;
+                       }
+                       event->prog = prog;
+                       event->orig_overflow_handler =
+                               parent_event->orig_overflow_handler;
+               }
+#endif
        }
 
        if (overflow_handler) {