Merge branch 'work.splice_read' of git://git.kernel.org/pub/scm/linux/kernel/git...
[cascardo/linux.git] / arch / arm64 / kernel / probes / kprobes.c
1 /*
2  * arch/arm64/kernel/probes/kprobes.c
3  *
4  * Kprobes support for ARM64
5  *
6  * Copyright (C) 2013 Linaro Limited.
7  * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  */
19 #include <linux/kasan.h>
20 #include <linux/kernel.h>
21 #include <linux/kprobes.h>
22 #include <linux/extable.h>
23 #include <linux/slab.h>
24 #include <linux/stop_machine.h>
25 #include <linux/stringify.h>
26 #include <asm/traps.h>
27 #include <asm/ptrace.h>
28 #include <asm/cacheflush.h>
29 #include <asm/debug-monitors.h>
30 #include <asm/system_misc.h>
31 #include <asm/insn.h>
32 #include <asm/uaccess.h>
33 #include <asm/irq.h>
34 #include <asm/sections.h>
35
36 #include "decode-insn.h"
37
38 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
39 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
40
41 static void __kprobes
42 post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
43
44 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
45 {
46         /* prepare insn slot */
47         p->ainsn.insn[0] = cpu_to_le32(p->opcode);
48
49         flush_icache_range((uintptr_t) (p->ainsn.insn),
50                            (uintptr_t) (p->ainsn.insn) +
51                            MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
52
53         /*
54          * Needs restoring of return address after stepping xol.
55          */
56         p->ainsn.restore = (unsigned long) p->addr +
57           sizeof(kprobe_opcode_t);
58 }
59
60 static void __kprobes arch_prepare_simulate(struct kprobe *p)
61 {
62         /* This instructions is not executed xol. No need to adjust the PC */
63         p->ainsn.restore = 0;
64 }
65
66 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
67 {
68         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
69
70         if (p->ainsn.handler)
71                 p->ainsn.handler((u32)p->opcode, (long)p->addr, regs);
72
73         /* single step simulated, now go for post processing */
74         post_kprobe_handler(kcb, regs);
75 }
76
77 int __kprobes arch_prepare_kprobe(struct kprobe *p)
78 {
79         unsigned long probe_addr = (unsigned long)p->addr;
80         extern char __start_rodata[];
81         extern char __end_rodata[];
82
83         if (probe_addr & 0x3)
84                 return -EINVAL;
85
86         /* copy instruction */
87         p->opcode = le32_to_cpu(*p->addr);
88
89         if (in_exception_text(probe_addr))
90                 return -EINVAL;
91         if (probe_addr >= (unsigned long) __start_rodata &&
92             probe_addr <= (unsigned long) __end_rodata)
93                 return -EINVAL;
94
95         /* decode instruction */
96         switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) {
97         case INSN_REJECTED:     /* insn not supported */
98                 return -EINVAL;
99
100         case INSN_GOOD_NO_SLOT: /* insn need simulation */
101                 p->ainsn.insn = NULL;
102                 break;
103
104         case INSN_GOOD: /* instruction uses slot */
105                 p->ainsn.insn = get_insn_slot();
106                 if (!p->ainsn.insn)
107                         return -ENOMEM;
108                 break;
109         };
110
111         /* prepare the instruction */
112         if (p->ainsn.insn)
113                 arch_prepare_ss_slot(p);
114         else
115                 arch_prepare_simulate(p);
116
117         return 0;
118 }
119
120 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
121 {
122         void *addrs[1];
123         u32 insns[1];
124
125         addrs[0] = (void *)addr;
126         insns[0] = (u32)opcode;
127
128         return aarch64_insn_patch_text(addrs, insns, 1);
129 }
130
131 /* arm kprobe: install breakpoint in text */
132 void __kprobes arch_arm_kprobe(struct kprobe *p)
133 {
134         patch_text(p->addr, BRK64_OPCODE_KPROBES);
135 }
136
137 /* disarm kprobe: remove breakpoint from text */
138 void __kprobes arch_disarm_kprobe(struct kprobe *p)
139 {
140         patch_text(p->addr, p->opcode);
141 }
142
143 void __kprobes arch_remove_kprobe(struct kprobe *p)
144 {
145         if (p->ainsn.insn) {
146                 free_insn_slot(p->ainsn.insn, 0);
147                 p->ainsn.insn = NULL;
148         }
149 }
150
151 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
152 {
153         kcb->prev_kprobe.kp = kprobe_running();
154         kcb->prev_kprobe.status = kcb->kprobe_status;
155 }
156
157 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
158 {
159         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
160         kcb->kprobe_status = kcb->prev_kprobe.status;
161 }
162
163 static void __kprobes set_current_kprobe(struct kprobe *p)
164 {
165         __this_cpu_write(current_kprobe, p);
166 }
167
168 /*
169  * When PSTATE.D is set (masked), then software step exceptions can not be
170  * generated.
171  * SPSR's D bit shows the value of PSTATE.D immediately before the
172  * exception was taken. PSTATE.D is set while entering into any exception
173  * mode, however software clears it for any normal (none-debug-exception)
174  * mode in the exception entry. Therefore, when we are entering into kprobe
175  * breakpoint handler from any normal mode then SPSR.D bit is already
176  * cleared, however it is set when we are entering from any debug exception
177  * mode.
178  * Since we always need to generate single step exception after a kprobe
179  * breakpoint exception therefore we need to clear it unconditionally, when
180  * we become sure that the current breakpoint exception is for kprobe.
181  */
182 static void __kprobes
183 spsr_set_debug_flag(struct pt_regs *regs, int mask)
184 {
185         unsigned long spsr = regs->pstate;
186
187         if (mask)
188                 spsr |= PSR_D_BIT;
189         else
190                 spsr &= ~PSR_D_BIT;
191
192         regs->pstate = spsr;
193 }
194
195 /*
196  * Interrupts need to be disabled before single-step mode is set, and not
197  * reenabled until after single-step mode ends.
198  * Without disabling interrupt on local CPU, there is a chance of
199  * interrupt occurrence in the period of exception return and  start of
200  * out-of-line single-step, that result in wrongly single stepping
201  * into the interrupt handler.
202  */
203 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
204                                                 struct pt_regs *regs)
205 {
206         kcb->saved_irqflag = regs->pstate;
207         regs->pstate |= PSR_I_BIT;
208 }
209
210 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
211                                                 struct pt_regs *regs)
212 {
213         if (kcb->saved_irqflag & PSR_I_BIT)
214                 regs->pstate |= PSR_I_BIT;
215         else
216                 regs->pstate &= ~PSR_I_BIT;
217 }
218
219 static void __kprobes
220 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
221 {
222         kcb->ss_ctx.ss_pending = true;
223         kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
224 }
225
226 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
227 {
228         kcb->ss_ctx.ss_pending = false;
229         kcb->ss_ctx.match_addr = 0;
230 }
231
232 static void __kprobes setup_singlestep(struct kprobe *p,
233                                        struct pt_regs *regs,
234                                        struct kprobe_ctlblk *kcb, int reenter)
235 {
236         unsigned long slot;
237
238         if (reenter) {
239                 save_previous_kprobe(kcb);
240                 set_current_kprobe(p);
241                 kcb->kprobe_status = KPROBE_REENTER;
242         } else {
243                 kcb->kprobe_status = KPROBE_HIT_SS;
244         }
245
246
247         if (p->ainsn.insn) {
248                 /* prepare for single stepping */
249                 slot = (unsigned long)p->ainsn.insn;
250
251                 set_ss_context(kcb, slot);      /* mark pending ss */
252
253                 spsr_set_debug_flag(regs, 0);
254
255                 /* IRQs and single stepping do not mix well. */
256                 kprobes_save_local_irqflag(kcb, regs);
257                 kernel_enable_single_step(regs);
258                 instruction_pointer_set(regs, slot);
259         } else {
260                 /* insn simulation */
261                 arch_simulate_insn(p, regs);
262         }
263 }
264
265 static int __kprobes reenter_kprobe(struct kprobe *p,
266                                     struct pt_regs *regs,
267                                     struct kprobe_ctlblk *kcb)
268 {
269         switch (kcb->kprobe_status) {
270         case KPROBE_HIT_SSDONE:
271         case KPROBE_HIT_ACTIVE:
272                 kprobes_inc_nmissed_count(p);
273                 setup_singlestep(p, regs, kcb, 1);
274                 break;
275         case KPROBE_HIT_SS:
276         case KPROBE_REENTER:
277                 pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr);
278                 dump_kprobe(p);
279                 BUG();
280                 break;
281         default:
282                 WARN_ON(1);
283                 return 0;
284         }
285
286         return 1;
287 }
288
289 static void __kprobes
290 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
291 {
292         struct kprobe *cur = kprobe_running();
293
294         if (!cur)
295                 return;
296
297         /* return addr restore if non-branching insn */
298         if (cur->ainsn.restore != 0)
299                 instruction_pointer_set(regs, cur->ainsn.restore);
300
301         /* restore back original saved kprobe variables and continue */
302         if (kcb->kprobe_status == KPROBE_REENTER) {
303                 restore_previous_kprobe(kcb);
304                 return;
305         }
306         /* call post handler */
307         kcb->kprobe_status = KPROBE_HIT_SSDONE;
308         if (cur->post_handler)  {
309                 /* post_handler can hit breakpoint and single step
310                  * again, so we enable D-flag for recursive exception.
311                  */
312                 cur->post_handler(cur, regs, 0);
313         }
314
315         reset_current_kprobe();
316 }
317
318 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
319 {
320         struct kprobe *cur = kprobe_running();
321         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
322
323         switch (kcb->kprobe_status) {
324         case KPROBE_HIT_SS:
325         case KPROBE_REENTER:
326                 /*
327                  * We are here because the instruction being single
328                  * stepped caused a page fault. We reset the current
329                  * kprobe and the ip points back to the probe address
330                  * and allow the page fault handler to continue as a
331                  * normal page fault.
332                  */
333                 instruction_pointer_set(regs, (unsigned long) cur->addr);
334                 if (!instruction_pointer(regs))
335                         BUG();
336
337                 kernel_disable_single_step();
338
339                 if (kcb->kprobe_status == KPROBE_REENTER)
340                         restore_previous_kprobe(kcb);
341                 else
342                         reset_current_kprobe();
343
344                 break;
345         case KPROBE_HIT_ACTIVE:
346         case KPROBE_HIT_SSDONE:
347                 /*
348                  * We increment the nmissed count for accounting,
349                  * we can also use npre/npostfault count for accounting
350                  * these specific fault cases.
351                  */
352                 kprobes_inc_nmissed_count(cur);
353
354                 /*
355                  * We come here because instructions in the pre/post
356                  * handler caused the page_fault, this could happen
357                  * if handler tries to access user space by
358                  * copy_from_user(), get_user() etc. Let the
359                  * user-specified handler try to fix it first.
360                  */
361                 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr))
362                         return 1;
363
364                 /*
365                  * In case the user-specified fault handler returned
366                  * zero, try to fix up.
367                  */
368                 if (fixup_exception(regs))
369                         return 1;
370         }
371         return 0;
372 }
373
374 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
375                                        unsigned long val, void *data)
376 {
377         return NOTIFY_DONE;
378 }
379
380 static void __kprobes kprobe_handler(struct pt_regs *regs)
381 {
382         struct kprobe *p, *cur_kprobe;
383         struct kprobe_ctlblk *kcb;
384         unsigned long addr = instruction_pointer(regs);
385
386         kcb = get_kprobe_ctlblk();
387         cur_kprobe = kprobe_running();
388
389         p = get_kprobe((kprobe_opcode_t *) addr);
390
391         if (p) {
392                 if (cur_kprobe) {
393                         if (reenter_kprobe(p, regs, kcb))
394                                 return;
395                 } else {
396                         /* Probe hit */
397                         set_current_kprobe(p);
398                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
399
400                         /*
401                          * If we have no pre-handler or it returned 0, we
402                          * continue with normal processing.  If we have a
403                          * pre-handler and it returned non-zero, it prepped
404                          * for calling the break_handler below on re-entry,
405                          * so get out doing nothing more here.
406                          *
407                          * pre_handler can hit a breakpoint and can step thru
408                          * before return, keep PSTATE D-flag enabled until
409                          * pre_handler return back.
410                          */
411                         if (!p->pre_handler || !p->pre_handler(p, regs)) {
412                                 setup_singlestep(p, regs, kcb, 0);
413                                 return;
414                         }
415                 }
416         } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) ==
417             BRK64_OPCODE_KPROBES) && cur_kprobe) {
418                 /* We probably hit a jprobe.  Call its break handler. */
419                 if (cur_kprobe->break_handler  &&
420                      cur_kprobe->break_handler(cur_kprobe, regs)) {
421                         setup_singlestep(cur_kprobe, regs, kcb, 0);
422                         return;
423                 }
424         }
425         /*
426          * The breakpoint instruction was removed right
427          * after we hit it.  Another cpu has removed
428          * either a probepoint or a debugger breakpoint
429          * at this address.  In either case, no further
430          * handling of this interrupt is appropriate.
431          * Return back to original instruction, and continue.
432          */
433 }
434
435 static int __kprobes
436 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
437 {
438         if ((kcb->ss_ctx.ss_pending)
439             && (kcb->ss_ctx.match_addr == addr)) {
440                 clear_ss_context(kcb);  /* clear pending ss */
441                 return DBG_HOOK_HANDLED;
442         }
443         /* not ours, kprobes should ignore it */
444         return DBG_HOOK_ERROR;
445 }
446
447 int __kprobes
448 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr)
449 {
450         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
451         int retval;
452
453         /* return error if this is not our step */
454         retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
455
456         if (retval == DBG_HOOK_HANDLED) {
457                 kprobes_restore_local_irqflag(kcb, regs);
458                 kernel_disable_single_step();
459
460                 post_kprobe_handler(kcb, regs);
461         }
462
463         return retval;
464 }
465
466 int __kprobes
467 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr)
468 {
469         kprobe_handler(regs);
470         return DBG_HOOK_HANDLED;
471 }
472
473 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
474 {
475         struct jprobe *jp = container_of(p, struct jprobe, kp);
476         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
477
478         kcb->jprobe_saved_regs = *regs;
479         /*
480          * Since we can't be sure where in the stack frame "stacked"
481          * pass-by-value arguments are stored we just don't try to
482          * duplicate any of the stack. Do not use jprobes on functions that
483          * use more than 64 bytes (after padding each to an 8 byte boundary)
484          * of arguments, or pass individual arguments larger than 16 bytes.
485          */
486
487         instruction_pointer_set(regs, (unsigned long) jp->entry);
488         preempt_disable();
489         pause_graph_tracing();
490         return 1;
491 }
492
493 void __kprobes jprobe_return(void)
494 {
495         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
496
497         /*
498          * Jprobe handler return by entering break exception,
499          * encoded same as kprobe, but with following conditions
500          * -a special PC to identify it from the other kprobes.
501          * -restore stack addr to original saved pt_regs
502          */
503         asm volatile("                          mov sp, %0      \n"
504                      "jprobe_return_break:      brk %1          \n"
505                      :
506                      : "r" (kcb->jprobe_saved_regs.sp),
507                        "I" (BRK64_ESR_KPROBES)
508                      : "memory");
509
510         unreachable();
511 }
512
513 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
514 {
515         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
516         long stack_addr = kcb->jprobe_saved_regs.sp;
517         long orig_sp = kernel_stack_pointer(regs);
518         struct jprobe *jp = container_of(p, struct jprobe, kp);
519         extern const char jprobe_return_break[];
520
521         if (instruction_pointer(regs) != (u64) jprobe_return_break)
522                 return 0;
523
524         if (orig_sp != stack_addr) {
525                 struct pt_regs *saved_regs =
526                     (struct pt_regs *)kcb->jprobe_saved_regs.sp;
527                 pr_err("current sp %lx does not match saved sp %lx\n",
528                        orig_sp, stack_addr);
529                 pr_err("Saved registers for jprobe %p\n", jp);
530                 show_regs(saved_regs);
531                 pr_err("Current registers\n");
532                 show_regs(regs);
533                 BUG();
534         }
535         unpause_graph_tracing();
536         *regs = kcb->jprobe_saved_regs;
537         preempt_enable_no_resched();
538         return 1;
539 }
540
541 bool arch_within_kprobe_blacklist(unsigned long addr)
542 {
543         if ((addr >= (unsigned long)__kprobes_text_start &&
544             addr < (unsigned long)__kprobes_text_end) ||
545             (addr >= (unsigned long)__entry_text_start &&
546             addr < (unsigned long)__entry_text_end) ||
547             (addr >= (unsigned long)__idmap_text_start &&
548             addr < (unsigned long)__idmap_text_end) ||
549             !!search_exception_tables(addr))
550                 return true;
551
552         if (!is_kernel_in_hyp_mode()) {
553                 if ((addr >= (unsigned long)__hyp_text_start &&
554                     addr < (unsigned long)__hyp_text_end) ||
555                     (addr >= (unsigned long)__hyp_idmap_text_start &&
556                     addr < (unsigned long)__hyp_idmap_text_end))
557                         return true;
558         }
559
560         return false;
561 }
562
563 void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
564 {
565         struct kretprobe_instance *ri = NULL;
566         struct hlist_head *head, empty_rp;
567         struct hlist_node *tmp;
568         unsigned long flags, orig_ret_address = 0;
569         unsigned long trampoline_address =
570                 (unsigned long)&kretprobe_trampoline;
571         kprobe_opcode_t *correct_ret_addr = NULL;
572
573         INIT_HLIST_HEAD(&empty_rp);
574         kretprobe_hash_lock(current, &head, &flags);
575
576         /*
577          * It is possible to have multiple instances associated with a given
578          * task either because multiple functions in the call path have
579          * return probes installed on them, and/or more than one
580          * return probe was registered for a target function.
581          *
582          * We can handle this because:
583          *     - instances are always pushed into the head of the list
584          *     - when multiple return probes are registered for the same
585          *       function, the (chronologically) first instance's ret_addr
586          *       will be the real return address, and all the rest will
587          *       point to kretprobe_trampoline.
588          */
589         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
590                 if (ri->task != current)
591                         /* another task is sharing our hash bucket */
592                         continue;
593
594                 orig_ret_address = (unsigned long)ri->ret_addr;
595
596                 if (orig_ret_address != trampoline_address)
597                         /*
598                          * This is the real return address. Any other
599                          * instances associated with this task are for
600                          * other calls deeper on the call stack
601                          */
602                         break;
603         }
604
605         kretprobe_assert(ri, orig_ret_address, trampoline_address);
606
607         correct_ret_addr = ri->ret_addr;
608         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
609                 if (ri->task != current)
610                         /* another task is sharing our hash bucket */
611                         continue;
612
613                 orig_ret_address = (unsigned long)ri->ret_addr;
614                 if (ri->rp && ri->rp->handler) {
615                         __this_cpu_write(current_kprobe, &ri->rp->kp);
616                         get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
617                         ri->ret_addr = correct_ret_addr;
618                         ri->rp->handler(ri, regs);
619                         __this_cpu_write(current_kprobe, NULL);
620                 }
621
622                 recycle_rp_inst(ri, &empty_rp);
623
624                 if (orig_ret_address != trampoline_address)
625                         /*
626                          * This is the real return address. Any other
627                          * instances associated with this task are for
628                          * other calls deeper on the call stack
629                          */
630                         break;
631         }
632
633         kretprobe_hash_unlock(current, &flags);
634
635         hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
636                 hlist_del(&ri->hlist);
637                 kfree(ri);
638         }
639         return (void *)orig_ret_address;
640 }
641
642 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
643                                       struct pt_regs *regs)
644 {
645         ri->ret_addr = (kprobe_opcode_t *)regs->regs[30];
646
647         /* replace return addr (x30) with trampoline */
648         regs->regs[30] = (long)&kretprobe_trampoline;
649 }
650
651 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
652 {
653         return 0;
654 }
655
656 int __init arch_init_kprobes(void)
657 {
658         return 0;
659 }