2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 * Copyright (C) 2007 Alan Stern
17 * Copyright (C) 2009 IBM Corporation
18 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
22 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
23 * using the CPU's debug registers.
26 #include <linux/perf_event.h>
27 #include <linux/hw_breakpoint.h>
28 #include <linux/irqflags.h>
29 #include <linux/notifier.h>
30 #include <linux/kallsyms.h>
31 #include <linux/kprobes.h>
32 #include <linux/percpu.h>
33 #include <linux/kdebug.h>
34 #include <linux/kernel.h>
35 #include <linux/module.h>
36 #include <linux/sched.h>
37 #include <linux/init.h>
38 #include <linux/smp.h>
40 #include <asm/hw_breakpoint.h>
41 #include <asm/processor.h>
42 #include <asm/debugreg.h>
44 /* Per cpu debug control register value */
45 DEFINE_PER_CPU(unsigned long, dr7);
46 EXPORT_PER_CPU_SYMBOL(dr7);
48 /* Per cpu debug address registers values */
49 static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
52 * Stores the breakpoints currently in use on each breakpoint address
53 * register for each cpus
55 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
59 * Encode the length, type, Exact, and Enable bits for a particular breakpoint
60 * as stored in debug register 7.
62 unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
64 unsigned long bp_info;
66 bp_info = (len | type) & 0xf;
67 bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
68 bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)) |
74 * Decode the length and type bits for a particular breakpoint as
75 * stored in debug register 7. Return the "enabled" status.
77 int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
79 int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
81 *len = (bp_info & 0xc) | 0x40;
82 *type = (bp_info & 0x3) | 0x80;
84 return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
88 * Install a perf counter breakpoint.
90 * We seek a free debug address register and use it for this
91 * breakpoint. Eventually we enable it in the debug control register.
93 * Atomic: we hold the counter->ctx->lock and we only handle variables
94 * and registers local to this cpu.
96 int arch_install_hw_breakpoint(struct perf_event *bp)
98 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
102 for (i = 0; i < HBP_NUM; i++) {
103 struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
111 if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
114 set_debugreg(info->address, i);
115 __get_cpu_var(cpu_debugreg[i]) = info->address;
117 dr7 = &__get_cpu_var(dr7);
118 *dr7 |= encode_dr7(i, info->len, info->type);
120 set_debugreg(*dr7, 7);
126 * Uninstall the breakpoint contained in the given counter.
128 * First we search the debug address register it uses and then we disable
131 * Atomic: we hold the counter->ctx->lock and we only handle variables
132 * and registers local to this cpu.
134 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
136 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
140 for (i = 0; i < HBP_NUM; i++) {
141 struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
149 if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
152 dr7 = &__get_cpu_var(dr7);
153 *dr7 &= ~encode_dr7(i, info->len, info->type);
155 set_debugreg(*dr7, 7);
158 static int get_hbp_len(u8 hbp_len)
160 unsigned int len_in_bytes = 0;
163 case X86_BREAKPOINT_LEN_1:
166 case X86_BREAKPOINT_LEN_2:
169 case X86_BREAKPOINT_LEN_4:
173 case X86_BREAKPOINT_LEN_8:
182 * Check for virtual address in user space.
184 int arch_check_va_in_userspace(unsigned long va, u8 hbp_len)
188 len = get_hbp_len(hbp_len);
190 return (va <= TASK_SIZE - len);
194 * Check for virtual address in kernel space.
196 static int arch_check_va_in_kernelspace(unsigned long va, u8 hbp_len)
200 len = get_hbp_len(hbp_len);
202 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
206 * Store a breakpoint's encoded address, length, and type.
208 static int arch_store_info(struct perf_event *bp)
210 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
212 * For kernel-addresses, either the address or symbol name can be
216 info->address = (unsigned long)
217 kallsyms_lookup_name(info->name);
224 int arch_bp_generic_fields(int x86_len, int x86_type,
225 int *gen_len, int *gen_type)
229 case X86_BREAKPOINT_LEN_1:
230 *gen_len = HW_BREAKPOINT_LEN_1;
232 case X86_BREAKPOINT_LEN_2:
233 *gen_len = HW_BREAKPOINT_LEN_2;
235 case X86_BREAKPOINT_LEN_4:
236 *gen_len = HW_BREAKPOINT_LEN_4;
239 case X86_BREAKPOINT_LEN_8:
240 *gen_len = HW_BREAKPOINT_LEN_8;
249 case X86_BREAKPOINT_EXECUTE:
250 *gen_type = HW_BREAKPOINT_X;
252 case X86_BREAKPOINT_WRITE:
253 *gen_type = HW_BREAKPOINT_W;
255 case X86_BREAKPOINT_RW:
256 *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
266 static int arch_build_bp_info(struct perf_event *bp)
268 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
270 info->address = bp->attr.bp_addr;
273 switch (bp->attr.bp_len) {
274 case HW_BREAKPOINT_LEN_1:
275 info->len = X86_BREAKPOINT_LEN_1;
277 case HW_BREAKPOINT_LEN_2:
278 info->len = X86_BREAKPOINT_LEN_2;
280 case HW_BREAKPOINT_LEN_4:
281 info->len = X86_BREAKPOINT_LEN_4;
284 case HW_BREAKPOINT_LEN_8:
285 info->len = X86_BREAKPOINT_LEN_8;
293 switch (bp->attr.bp_type) {
294 case HW_BREAKPOINT_W:
295 info->type = X86_BREAKPOINT_WRITE;
297 case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
298 info->type = X86_BREAKPOINT_RW;
300 case HW_BREAKPOINT_X:
301 info->type = X86_BREAKPOINT_EXECUTE;
310 * Validate the arch-specific HW Breakpoint register settings
312 int arch_validate_hwbkpt_settings(struct perf_event *bp,
313 struct task_struct *tsk)
315 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
320 ret = arch_build_bp_info(bp);
326 if (info->type == X86_BREAKPOINT_EXECUTE)
328 * Ptrace-refactoring code
329 * For now, we'll allow instruction breakpoint only for user-space
332 if ((!arch_check_va_in_userspace(info->address, info->len)) &&
333 info->len != X86_BREAKPOINT_EXECUTE)
337 case X86_BREAKPOINT_LEN_1:
340 case X86_BREAKPOINT_LEN_2:
343 case X86_BREAKPOINT_LEN_4:
347 case X86_BREAKPOINT_LEN_8:
356 ret = arch_store_info(bp);
361 * Check that the low-order bits of the address are appropriate
362 * for the alignment implied by len.
364 if (info->address & align)
367 /* Check that the virtual address is in the proper range */
369 if (!arch_check_va_in_userspace(info->address, info->len))
372 if (!arch_check_va_in_kernelspace(info->address, info->len))
380 * Dump the debug register contents to the user.
381 * We can't dump our per cpu values because it
382 * may contain cpu wide breakpoint, something that
383 * doesn't belong to the current task.
385 * TODO: include non-ptrace user breakpoints (perf)
387 void aout_dump_debugregs(struct user *dump)
391 struct perf_event *bp;
392 struct arch_hw_breakpoint *info;
393 struct thread_struct *thread = ¤t->thread;
395 for (i = 0; i < HBP_NUM; i++) {
396 bp = thread->ptrace_bps[i];
398 if (bp && !bp->attr.disabled) {
399 dump->u_debugreg[i] = bp->attr.bp_addr;
400 info = counter_arch_bp(bp);
401 dr7 |= encode_dr7(i, info->len, info->type);
403 dump->u_debugreg[i] = 0;
407 dump->u_debugreg[4] = 0;
408 dump->u_debugreg[5] = 0;
409 dump->u_debugreg[6] = current->thread.debugreg6;
411 dump->u_debugreg[7] = dr7;
413 EXPORT_SYMBOL_GPL(aout_dump_debugregs);
416 * Release the user breakpoints used by ptrace
418 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
421 struct thread_struct *t = &tsk->thread;
423 for (i = 0; i < HBP_NUM; i++) {
424 unregister_hw_breakpoint(t->ptrace_bps[i]);
425 t->ptrace_bps[i] = NULL;
429 void hw_breakpoint_restore(void)
431 set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
432 set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
433 set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
434 set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
435 set_debugreg(current->thread.debugreg6, 6);
436 set_debugreg(__get_cpu_var(dr7), 7);
438 EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
441 * Handle debug exception notifications.
443 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
445 * NOTIFY_DONE returned if one of the following conditions is true.
446 * i) When the causative address is from user-space and the exception
447 * is a valid one, i.e. not triggered as a result of lazy debug register
449 * ii) When there are more bits than trap<n> set in DR6 register (such
450 * as BD, BS or BT) indicating that more than one debug condition is
451 * met and requires some more action in do_debug().
453 * NOTIFY_STOP returned for all other cases
456 static int __kprobes hw_breakpoint_handler(struct die_args *args)
458 int i, cpu, rc = NOTIFY_STOP;
459 struct perf_event *bp;
460 unsigned long dr7, dr6;
461 unsigned long *dr6_p;
463 /* The DR6 value is pointed by args->err */
464 dr6_p = (unsigned long *)ERR_PTR(args->err);
467 /* Do an early return if no trap bits are set in DR6 */
468 if ((dr6 & DR_TRAP_BITS) == 0)
471 get_debugreg(dr7, 7);
472 /* Disable breakpoints during exception handling */
473 set_debugreg(0UL, 7);
475 * Assert that local interrupts are disabled
476 * Reset the DRn bits in the virtualized register value.
477 * The ptrace trigger routine will add in whatever is needed.
479 current->thread.debugreg6 &= ~DR_TRAP_BITS;
482 /* Handle all the breakpoints that were triggered */
483 for (i = 0; i < HBP_NUM; ++i) {
484 if (likely(!(dr6 & (DR_TRAP0 << i))))
488 * The counter may be concurrently released but that can only
489 * occur from a call_rcu() path. We can then safely fetch
490 * the breakpoint, use its callback, touch its counter
491 * while we are in an rcu_read_lock() path.
495 bp = per_cpu(bp_per_reg[i], cpu);
499 * Reset the 'i'th TRAP bit in dr6 to denote completion of
502 (*dr6_p) &= ~(DR_TRAP0 << i);
504 * bp can be NULL due to lazy debug register switching
505 * or due to concurrent perf counter removing.
512 (bp->callback)(bp, args->regs);
516 if (dr6 & (~DR_TRAP_BITS))
519 set_debugreg(dr7, 7);
526 * Handle debug exception notifications.
528 int __kprobes hw_breakpoint_exceptions_notify(
529 struct notifier_block *unused, unsigned long val, void *data)
531 if (val != DIE_DEBUG)
534 return hw_breakpoint_handler(data);
537 void hw_breakpoint_pmu_read(struct perf_event *bp)
542 void hw_breakpoint_pmu_unthrottle(struct perf_event *bp)