perf, core: Rate limit perf_sched_events jump_label patching
jump_lable patching is very expensive operation that involves pausing all cpus. The patching of perf_sched_events jump_label is easily controllable from userspace by unprivileged user. When te user runs a loop like this: "while true; do perf stat -e cycles true; done" ... the performance of my test application that just increments a counter for one second drops by 4%. This is on a 16 cpu box with my test application using only one of them. An impact on a real server doing real work will be worse. Performance of KVM PMU drops nearly 50% due to jump_lable for "perf record" since KVM PMU implementation creates and destroys perf event frequently. This patch introduces a way to rate limit jump_label patching and uses it to fix the above problem. I believe that as jump_label use will spread the problem will become more common and thus solving it in a generic code is appropriate. Also fixing it in the perf code would result in moving jump_label accounting logic to perf code with all the ifdefs in case of JUMP_LABEL=n kernel. With this patch all details are nicely hidden inside jump_label code. Signed-off-by: Gleb Natapov <gleb@redhat.com> Acked-by: Jason Baron <jbaron@redhat.com> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20111127155909.GO2557@redhat.com Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
b79387ef18
commit
b202952075
4 changed files with 68 additions and 10 deletions
|
@ -3,6 +3,7 @@
|
|||
|
||||
#include <linux/types.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/workqueue.h>
|
||||
|
||||
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
|
||||
|
||||
|
@ -14,6 +15,12 @@ struct jump_label_key {
|
|||
#endif
|
||||
};
|
||||
|
||||
struct jump_label_key_deferred {
|
||||
struct jump_label_key key;
|
||||
unsigned long timeout;
|
||||
struct delayed_work work;
|
||||
};
|
||||
|
||||
# include <asm/jump_label.h>
|
||||
# define HAVE_JUMP_LABEL
|
||||
#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
|
||||
|
@ -51,8 +58,11 @@ extern void arch_jump_label_transform_static(struct jump_entry *entry,
|
|||
extern int jump_label_text_reserved(void *start, void *end);
|
||||
extern void jump_label_inc(struct jump_label_key *key);
|
||||
extern void jump_label_dec(struct jump_label_key *key);
|
||||
extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
|
||||
extern bool jump_label_enabled(struct jump_label_key *key);
|
||||
extern void jump_label_apply_nops(struct module *mod);
|
||||
extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
|
||||
unsigned long rl);
|
||||
|
||||
#else /* !HAVE_JUMP_LABEL */
|
||||
|
||||
|
@ -68,6 +78,10 @@ static __always_inline void jump_label_init(void)
|
|||
{
|
||||
}
|
||||
|
||||
struct jump_label_key_deferred {
|
||||
struct jump_label_key key;
|
||||
};
|
||||
|
||||
static __always_inline bool static_branch(struct jump_label_key *key)
|
||||
{
|
||||
if (unlikely(atomic_read(&key->enabled)))
|
||||
|
@ -85,6 +99,11 @@ static inline void jump_label_dec(struct jump_label_key *key)
|
|||
atomic_dec(&key->enabled);
|
||||
}
|
||||
|
||||
static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
|
||||
{
|
||||
jump_label_dec(&key->key);
|
||||
}
|
||||
|
||||
static inline int jump_label_text_reserved(void *start, void *end)
|
||||
{
|
||||
return 0;
|
||||
|
@ -102,6 +121,11 @@ static inline int jump_label_apply_nops(struct module *mod)
|
|||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
|
||||
unsigned long rl)
|
||||
{
|
||||
}
|
||||
#endif /* HAVE_JUMP_LABEL */
|
||||
|
||||
#endif /* _LINUX_JUMP_LABEL_H */
|
||||
|
|
|
@ -1064,12 +1064,12 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
|
|||
}
|
||||
}
|
||||
|
||||
extern struct jump_label_key perf_sched_events;
|
||||
extern struct jump_label_key_deferred perf_sched_events;
|
||||
|
||||
static inline void perf_event_task_sched_in(struct task_struct *prev,
|
||||
struct task_struct *task)
|
||||
{
|
||||
if (static_branch(&perf_sched_events))
|
||||
if (static_branch(&perf_sched_events.key))
|
||||
__perf_event_task_sched_in(prev, task);
|
||||
}
|
||||
|
||||
|
@ -1078,7 +1078,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
|
|||
{
|
||||
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
|
||||
|
||||
if (static_branch(&perf_sched_events))
|
||||
if (static_branch(&perf_sched_events.key))
|
||||
__perf_event_task_sched_out(prev, next);
|
||||
}
|
||||
|
||||
|
|
|
@ -128,7 +128,7 @@ enum event_type_t {
|
|||
* perf_sched_events : >0 events exist
|
||||
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
|
||||
*/
|
||||
struct jump_label_key perf_sched_events __read_mostly;
|
||||
struct jump_label_key_deferred perf_sched_events __read_mostly;
|
||||
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
|
||||
|
||||
static atomic_t nr_mmap_events __read_mostly;
|
||||
|
@ -2748,7 +2748,7 @@ static void free_event(struct perf_event *event)
|
|||
|
||||
if (!event->parent) {
|
||||
if (event->attach_state & PERF_ATTACH_TASK)
|
||||
jump_label_dec(&perf_sched_events);
|
||||
jump_label_dec_deferred(&perf_sched_events);
|
||||
if (event->attr.mmap || event->attr.mmap_data)
|
||||
atomic_dec(&nr_mmap_events);
|
||||
if (event->attr.comm)
|
||||
|
@ -2759,7 +2759,7 @@ static void free_event(struct perf_event *event)
|
|||
put_callchain_buffers();
|
||||
if (is_cgroup_event(event)) {
|
||||
atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
|
||||
jump_label_dec(&perf_sched_events);
|
||||
jump_label_dec_deferred(&perf_sched_events);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -5784,7 +5784,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
|
|||
|
||||
if (!event->parent) {
|
||||
if (event->attach_state & PERF_ATTACH_TASK)
|
||||
jump_label_inc(&perf_sched_events);
|
||||
jump_label_inc(&perf_sched_events.key);
|
||||
if (event->attr.mmap || event->attr.mmap_data)
|
||||
atomic_inc(&nr_mmap_events);
|
||||
if (event->attr.comm)
|
||||
|
@ -6022,7 +6022,7 @@ SYSCALL_DEFINE5(perf_event_open,
|
|||
* - that may need work on context switch
|
||||
*/
|
||||
atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
|
||||
jump_label_inc(&perf_sched_events);
|
||||
jump_label_inc(&perf_sched_events.key);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -6868,6 +6868,9 @@ void __init perf_event_init(void)
|
|||
|
||||
ret = init_hw_breakpoint();
|
||||
WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
|
||||
|
||||
/* do not patch jump label more than once per second */
|
||||
jump_label_rate_limit(&perf_sched_events, HZ);
|
||||
}
|
||||
|
||||
static int __init perf_event_sysfs_init(void)
|
||||
|
|
|
@ -72,15 +72,46 @@ void jump_label_inc(struct jump_label_key *key)
|
|||
jump_label_unlock();
|
||||
}
|
||||
|
||||
void jump_label_dec(struct jump_label_key *key)
|
||||
static void __jump_label_dec(struct jump_label_key *key,
|
||||
unsigned long rate_limit, struct delayed_work *work)
|
||||
{
|
||||
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
|
||||
return;
|
||||
|
||||
if (rate_limit) {
|
||||
atomic_inc(&key->enabled);
|
||||
schedule_delayed_work(work, rate_limit);
|
||||
} else
|
||||
jump_label_update(key, JUMP_LABEL_DISABLE);
|
||||
|
||||
jump_label_unlock();
|
||||
}
|
||||
|
||||
static void jump_label_update_timeout(struct work_struct *work)
|
||||
{
|
||||
struct jump_label_key_deferred *key =
|
||||
container_of(work, struct jump_label_key_deferred, work.work);
|
||||
__jump_label_dec(&key->key, 0, NULL);
|
||||
}
|
||||
|
||||
void jump_label_dec(struct jump_label_key *key)
|
||||
{
|
||||
__jump_label_dec(key, 0, NULL);
|
||||
}
|
||||
|
||||
void jump_label_dec_deferred(struct jump_label_key_deferred *key)
|
||||
{
|
||||
__jump_label_dec(&key->key, key->timeout, &key->work);
|
||||
}
|
||||
|
||||
|
||||
void jump_label_rate_limit(struct jump_label_key_deferred *key,
|
||||
unsigned long rl)
|
||||
{
|
||||
key->timeout = rl;
|
||||
INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
|
||||
}
|
||||
|
||||
static int addr_conflict(struct jump_entry *entry, void *start, void *end)
|
||||
{
|
||||
if (entry->code <= (unsigned long)end &&
|
||||
|
|
Loading…
Reference in a new issue