ftrace/x86: Have arch x86_64 use breakpoints instead of stop machine
This method changes x86 to add a breakpoint to the mcount locations instead of calling stop machine. Now that iret can be handled by NMIs, we perform the following to update code: 1) Add a breakpoint to all locations that will be modified 2) Sync all cores 3) Update all locations to be either a nop or call (except breakpoint op) 4) Sync all cores 5) Remove the breakpoint with the new code. 6) Sync all cores [ Added updates that Masami suggested: Use unlikely(modifying_ftrace_code) in int3 trap to keep kprobes efficient. Don't use NOTIFY_* in ftrace handler in int3 as it is not a notifier. ] Cc: H. Peter Anvin <hpa@zytor.com> Acked-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
This commit is contained in:
Steven Rostedt
Steven Rostedt
parent
438ced1720
commit
08d636b6d4
4 changed files with 358 additions and 1 deletions
|
|
@ -34,6 +34,7 @@
|
|||
|
||||
#ifndef __ASSEMBLY__
|
||||
extern void mcount(void);
|
||||
extern int modifying_ftrace_code;
|
||||
|
||||
static inline unsigned long ftrace_call_adjust(unsigned long addr)
|
||||
{
|
||||
|
|
@ -50,6 +51,8 @@ struct dyn_arch_ftrace {
|
|||
/* No extra data needed for x86 */
|
||||
};
|
||||
|
||||
int ftrace_int3_handler(struct pt_regs *regs);
|
||||
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE */
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* CONFIG_FUNCTION_TRACER */
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include <linux/init.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/kprobes.h>
|
||||
|
||||
#include <trace/syscall.h>
|
||||
|
||||
|
|
@ -334,6 +335,347 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
|
|||
return ret;
|
||||
}
|
||||
|
||||
int modifying_ftrace_code __read_mostly;
|
||||
|
||||
/*
|
||||
* A breakpoint was added to the code address we are about to
|
||||
* modify, and this is the handle that will just skip over it.
|
||||
* We are either changing a nop into a trace call, or a trace
|
||||
* call to a nop. While the change is taking place, we treat
|
||||
* it just like it was a nop.
|
||||
*/
|
||||
int ftrace_int3_handler(struct pt_regs *regs)
|
||||
{
|
||||
if (WARN_ON_ONCE(!regs))
|
||||
return 0;
|
||||
|
||||
if (!ftrace_location(regs->ip - 1))
|
||||
return 0;
|
||||
|
||||
regs->ip += MCOUNT_INSN_SIZE - 1;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int ftrace_write(unsigned long ip, const char *val, int size)
|
||||
{
|
||||
/*
|
||||
* On x86_64, kernel text mappings are mapped read-only with
|
||||
* CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
|
||||
* of the kernel text mapping to modify the kernel text.
|
||||
*
|
||||
* For 32bit kernels, these mappings are same and we can use
|
||||
* kernel identity mapping to modify code.
|
||||
*/
|
||||
if (within(ip, (unsigned long)_text, (unsigned long)_etext))
|
||||
ip = (unsigned long)__va(__pa(ip));
|
||||
|
||||
return probe_kernel_write((void *)ip, val, size);
|
||||
}
|
||||
|
||||
static int add_break(unsigned long ip, const char *old)
|
||||
{
|
||||
unsigned char replaced[MCOUNT_INSN_SIZE];
|
||||
unsigned char brk = BREAKPOINT_INSTRUCTION;
|
||||
|
||||
if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
|
||||
return -EFAULT;
|
||||
|
||||
/* Make sure it is what we expect it to be */
|
||||
if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (ftrace_write(ip, &brk, 1))
|
||||
return -EPERM;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||
{
|
||||
unsigned const char *old;
|
||||
unsigned long ip = rec->ip;
|
||||
|
||||
old = ftrace_call_replace(ip, addr);
|
||||
|
||||
return add_break(rec->ip, old);
|
||||
}
|
||||
|
||||
|
||||
static int add_brk_on_nop(struct dyn_ftrace *rec)
|
||||
{
|
||||
unsigned const char *old;
|
||||
|
||||
old = ftrace_nop_replace();
|
||||
|
||||
return add_break(rec->ip, old);
|
||||
}
|
||||
|
||||
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
|
||||
{
|
||||
unsigned long ftrace_addr;
|
||||
int ret;
|
||||
|
||||
ret = ftrace_test_record(rec, enable);
|
||||
|
||||
ftrace_addr = (unsigned long)FTRACE_ADDR;
|
||||
|
||||
switch (ret) {
|
||||
case FTRACE_UPDATE_IGNORE:
|
||||
return 0;
|
||||
|
||||
case FTRACE_UPDATE_MAKE_CALL:
|
||||
/* converting nop to call */
|
||||
return add_brk_on_nop(rec);
|
||||
|
||||
case FTRACE_UPDATE_MAKE_NOP:
|
||||
/* converting a call to a nop */
|
||||
return add_brk_on_call(rec, ftrace_addr);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* On error, we need to remove breakpoints. This needs to
|
||||
* be done caefully. If the address does not currently have a
|
||||
* breakpoint, we know we are done. Otherwise, we look at the
|
||||
* remaining 4 bytes of the instruction. If it matches a nop
|
||||
* we replace the breakpoint with the nop. Otherwise we replace
|
||||
* it with the call instruction.
|
||||
*/
|
||||
static int remove_breakpoint(struct dyn_ftrace *rec)
|
||||
{
|
||||
unsigned char ins[MCOUNT_INSN_SIZE];
|
||||
unsigned char brk = BREAKPOINT_INSTRUCTION;
|
||||
const unsigned char *nop;
|
||||
unsigned long ftrace_addr;
|
||||
unsigned long ip = rec->ip;
|
||||
|
||||
/* If we fail the read, just give up */
|
||||
if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
|
||||
return -EFAULT;
|
||||
|
||||
/* If this does not have a breakpoint, we are done */
|
||||
if (ins[0] != brk)
|
||||
return -1;
|
||||
|
||||
nop = ftrace_nop_replace();
|
||||
|
||||
/*
|
||||
* If the last 4 bytes of the instruction do not match
|
||||
* a nop, then we assume that this is a call to ftrace_addr.
|
||||
*/
|
||||
if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
|
||||
/*
|
||||
* For extra paranoidism, we check if the breakpoint is on
|
||||
* a call that would actually jump to the ftrace_addr.
|
||||
* If not, don't touch the breakpoint, we make just create
|
||||
* a disaster.
|
||||
*/
|
||||
ftrace_addr = (unsigned long)FTRACE_ADDR;
|
||||
nop = ftrace_call_replace(ip, ftrace_addr);
|
||||
|
||||
if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return probe_kernel_write((void *)ip, &nop[0], 1);
|
||||
}
|
||||
|
||||
static int add_update_code(unsigned long ip, unsigned const char *new)
|
||||
{
|
||||
/* skip breakpoint */
|
||||
ip++;
|
||||
new++;
|
||||
if (ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1))
|
||||
return -EPERM;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||
{
|
||||
unsigned long ip = rec->ip;
|
||||
unsigned const char *new;
|
||||
|
||||
new = ftrace_call_replace(ip, addr);
|
||||
return add_update_code(ip, new);
|
||||
}
|
||||
|
||||
static int add_update_nop(struct dyn_ftrace *rec)
|
||||
{
|
||||
unsigned long ip = rec->ip;
|
||||
unsigned const char *new;
|
||||
|
||||
new = ftrace_nop_replace();
|
||||
return add_update_code(ip, new);
|
||||
}
|
||||
|
||||
static int add_update(struct dyn_ftrace *rec, int enable)
|
||||
{
|
||||
unsigned long ftrace_addr;
|
||||
int ret;
|
||||
|
||||
ret = ftrace_test_record(rec, enable);
|
||||
|
||||
ftrace_addr = (unsigned long)FTRACE_ADDR;
|
||||
|
||||
switch (ret) {
|
||||
case FTRACE_UPDATE_IGNORE:
|
||||
return 0;
|
||||
|
||||
case FTRACE_UPDATE_MAKE_CALL:
|
||||
/* converting nop to call */
|
||||
return add_update_call(rec, ftrace_addr);
|
||||
|
||||
case FTRACE_UPDATE_MAKE_NOP:
|
||||
/* converting a call to a nop */
|
||||
return add_update_nop(rec);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||
{
|
||||
unsigned long ip = rec->ip;
|
||||
unsigned const char *new;
|
||||
|
||||
new = ftrace_call_replace(ip, addr);
|
||||
|
||||
if (ftrace_write(ip, new, 1))
|
||||
return -EPERM;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int finish_update_nop(struct dyn_ftrace *rec)
|
||||
{
|
||||
unsigned long ip = rec->ip;
|
||||
unsigned const char *new;
|
||||
|
||||
new = ftrace_nop_replace();
|
||||
|
||||
if (ftrace_write(ip, new, 1))
|
||||
return -EPERM;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int finish_update(struct dyn_ftrace *rec, int enable)
|
||||
{
|
||||
unsigned long ftrace_addr;
|
||||
int ret;
|
||||
|
||||
ret = ftrace_update_record(rec, enable);
|
||||
|
||||
ftrace_addr = (unsigned long)FTRACE_ADDR;
|
||||
|
||||
switch (ret) {
|
||||
case FTRACE_UPDATE_IGNORE:
|
||||
return 0;
|
||||
|
||||
case FTRACE_UPDATE_MAKE_CALL:
|
||||
/* converting nop to call */
|
||||
return finish_update_call(rec, ftrace_addr);
|
||||
|
||||
case FTRACE_UPDATE_MAKE_NOP:
|
||||
/* converting a call to a nop */
|
||||
return finish_update_nop(rec);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void do_sync_core(void *data)
|
||||
{
|
||||
sync_core();
|
||||
}
|
||||
|
||||
static void run_sync(void)
|
||||
{
|
||||
int enable_irqs = irqs_disabled();
|
||||
|
||||
/* We may be called with interrupts disbled (on bootup). */
|
||||
if (enable_irqs)
|
||||
local_irq_enable();
|
||||
on_each_cpu(do_sync_core, NULL, 1);
|
||||
if (enable_irqs)
|
||||
local_irq_disable();
|
||||
}
|
||||
|
||||
static void ftrace_replace_code(int enable)
|
||||
{
|
||||
struct ftrace_rec_iter *iter;
|
||||
struct dyn_ftrace *rec;
|
||||
const char *report = "adding breakpoints";
|
||||
int count = 0;
|
||||
int ret;
|
||||
|
||||
for_ftrace_rec_iter(iter) {
|
||||
rec = ftrace_rec_iter_record(iter);
|
||||
|
||||
ret = add_breakpoints(rec, enable);
|
||||
if (ret)
|
||||
goto remove_breakpoints;
|
||||
count++;
|
||||
}
|
||||
|
||||
run_sync();
|
||||
|
||||
report = "updating code";
|
||||
|
||||
for_ftrace_rec_iter(iter) {
|
||||
rec = ftrace_rec_iter_record(iter);
|
||||
|
||||
ret = add_update(rec, enable);
|
||||
if (ret)
|
||||
goto remove_breakpoints;
|
||||
}
|
||||
|
||||
run_sync();
|
||||
|
||||
report = "removing breakpoints";
|
||||
|
||||
for_ftrace_rec_iter(iter) {
|
||||
rec = ftrace_rec_iter_record(iter);
|
||||
|
||||
ret = finish_update(rec, enable);
|
||||
if (ret)
|
||||
goto remove_breakpoints;
|
||||
}
|
||||
|
||||
run_sync();
|
||||
|
||||
return;
|
||||
|
||||
remove_breakpoints:
|
||||
ftrace_bug(ret, rec ? rec->ip : 0);
|
||||
printk(KERN_WARNING "Failed on %s (%d):\n", report, count);
|
||||
for_ftrace_rec_iter(iter) {
|
||||
rec = ftrace_rec_iter_record(iter);
|
||||
remove_breakpoint(rec);
|
||||
}
|
||||
}
|
||||
|
||||
void arch_ftrace_update_code(int command)
|
||||
{
|
||||
modifying_ftrace_code++;
|
||||
|
||||
if (command & FTRACE_UPDATE_CALLS)
|
||||
ftrace_replace_code(1);
|
||||
else if (command & FTRACE_DISABLE_CALLS)
|
||||
ftrace_replace_code(0);
|
||||
|
||||
if (command & FTRACE_UPDATE_TRACE_FUNC)
|
||||
ftrace_update_ftrace_func(ftrace_trace_function);
|
||||
|
||||
if (command & FTRACE_START_FUNC_RET)
|
||||
ftrace_enable_ftrace_graph_caller();
|
||||
else if (command & FTRACE_STOP_FUNC_RET)
|
||||
ftrace_disable_ftrace_graph_caller();
|
||||
|
||||
modifying_ftrace_code--;
|
||||
}
|
||||
|
||||
int __init ftrace_dyn_arch_init(void *data)
|
||||
{
|
||||
/* The return code is retured via data */
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@
|
|||
#include <asm/processor.h>
|
||||
#include <asm/debugreg.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <asm/ftrace.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/desc.h>
|
||||
#include <asm/i387.h>
|
||||
|
|
@ -303,8 +304,13 @@ do_general_protection(struct pt_regs *regs, long error_code)
|
|||
}
|
||||
|
||||
/* May run on IST stack. */
|
||||
dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
|
||||
dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code)
|
||||
{
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE
|
||||
/* ftrace must be first, everything else may cause a recursive crash */
|
||||
if (unlikely(modifying_ftrace_code) && ftrace_int3_handler(regs))
|
||||
return;
|
||||
#endif
|
||||
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
|
||||
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
|
||||
SIGTRAP) == NOTIFY_STOP)
|
||||
|
|
|
|||
|
|
@ -286,6 +286,12 @@ struct ftrace_rec_iter *ftrace_rec_iter_start(void);
|
|||
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
|
||||
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
|
||||
|
||||
#define for_ftrace_rec_iter(iter) \
|
||||
for (iter = ftrace_rec_iter_start(); \
|
||||
iter; \
|
||||
iter = ftrace_rec_iter_next(iter))
|
||||
|
||||
|
||||
int ftrace_update_record(struct dyn_ftrace *rec, int enable);
|
||||
int ftrace_test_record(struct dyn_ftrace *rec, int enable);
|
||||
void ftrace_run_stop_machine(int command);
|
||||
|
|
|
|||
Loading…
Reference in a new issue