Merge branch 'nohz/printk-v8' into irq/core
Conflicts: kernel/irq_work.c Add support for printk in full dynticks CPU. * Don't stop tick with irq works pending. This fix is generally useful and concerns archs that can't raise self IPIs. * Flush irq works before CPU offlining. * Introduce "lazy" irq works that can wait for the next tick to be executed, unless it's stopped. * Implement klogd wake up using irq work. This removes the ad-hoc printk_tick()/printk_needs_cpu() hooks and make it working even in dynticks mode. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
This commit is contained in:
Frederic Weisbecker
commit
077931446b
8 changed files with 147 additions and 50 deletions
|
|
@ -3,6 +3,20 @@
|
|||
|
||||
#include <linux/llist.h>
|
||||
|
||||
/*
|
||||
* An entry can be in one of four states:
|
||||
*
|
||||
* free NULL, 0 -> {claimed} : free to be used
|
||||
* claimed NULL, 3 -> {pending} : claimed to be enqueued
|
||||
* pending next, 3 -> {busy} : queued, pending callback
|
||||
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
|
||||
*/
|
||||
|
||||
#define IRQ_WORK_PENDING 1UL
|
||||
#define IRQ_WORK_BUSY 2UL
|
||||
#define IRQ_WORK_FLAGS 3UL
|
||||
#define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */
|
||||
|
||||
struct irq_work {
|
||||
unsigned long flags;
|
||||
struct llist_node llnode;
|
||||
|
|
@ -20,4 +34,10 @@ void irq_work_queue(struct irq_work *work);
|
|||
void irq_work_run(void);
|
||||
void irq_work_sync(struct irq_work *work);
|
||||
|
||||
#ifdef CONFIG_IRQ_WORK
|
||||
bool irq_work_needs_cpu(void);
|
||||
#else
|
||||
static bool irq_work_needs_cpu(void) { return false; }
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_IRQ_WORK_H */
|
||||
|
|
|
|||
|
|
@ -98,9 +98,6 @@ int no_printk(const char *fmt, ...)
|
|||
extern asmlinkage __printf(1, 2)
|
||||
void early_printk(const char *fmt, ...);
|
||||
|
||||
extern int printk_needs_cpu(int cpu);
|
||||
extern void printk_tick(void);
|
||||
|
||||
#ifdef CONFIG_PRINTK
|
||||
asmlinkage __printf(5, 0)
|
||||
int vprintk_emit(int facility, int level,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
|
||||
#include <linux/clockchips.h>
|
||||
#include <linux/irqflags.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/hrtimer.h>
|
||||
|
||||
#ifdef CONFIG_GENERIC_CLOCKEVENTS
|
||||
|
||||
|
|
@ -122,13 +124,26 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
|
|||
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
|
||||
|
||||
# ifdef CONFIG_NO_HZ
|
||||
DECLARE_PER_CPU(struct tick_sched, tick_cpu_sched);
|
||||
|
||||
static inline int tick_nohz_tick_stopped(void)
|
||||
{
|
||||
return __this_cpu_read(tick_cpu_sched.tick_stopped);
|
||||
}
|
||||
|
||||
extern void tick_nohz_idle_enter(void);
|
||||
extern void tick_nohz_idle_exit(void);
|
||||
extern void tick_nohz_irq_exit(void);
|
||||
extern ktime_t tick_nohz_get_sleep_length(void);
|
||||
extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
|
||||
extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
|
||||
# else
|
||||
|
||||
# else /* !CONFIG_NO_HZ */
|
||||
static inline int tick_nohz_tick_stopped(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void tick_nohz_idle_enter(void) { }
|
||||
static inline void tick_nohz_idle_exit(void) { }
|
||||
|
||||
|
|
|
|||
|
|
@ -1259,6 +1259,7 @@ config HOTPLUG
|
|||
config PRINTK
|
||||
default y
|
||||
bool "Enable support for printk" if EXPERT
|
||||
select IRQ_WORK
|
||||
help
|
||||
This option enables normal printk support. Removing it
|
||||
eliminates most of the message strings from the kernel image
|
||||
|
|
|
|||
|
|
@ -12,22 +12,15 @@
|
|||
#include <linux/percpu.h>
|
||||
#include <linux/hardirq.h>
|
||||
#include <linux/irqflags.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/tick.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/notifier.h>
|
||||
#include <asm/processor.h>
|
||||
|
||||
/*
|
||||
* An entry can be in one of four states:
|
||||
*
|
||||
* free NULL, 0 -> {claimed} : free to be used
|
||||
* claimed NULL, 3 -> {pending} : claimed to be enqueued
|
||||
* pending next, 3 -> {busy} : queued, pending callback
|
||||
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
|
||||
*/
|
||||
|
||||
#define IRQ_WORK_PENDING 1UL
|
||||
#define IRQ_WORK_BUSY 2UL
|
||||
#define IRQ_WORK_FLAGS 3UL
|
||||
|
||||
static DEFINE_PER_CPU(struct llist_head, irq_work_list);
|
||||
static DEFINE_PER_CPU(int, irq_work_raised);
|
||||
|
||||
/*
|
||||
* Claim the entry so that no one else will poke at it.
|
||||
|
|
@ -70,8 +63,6 @@ void __weak arch_irq_work_raise(void)
|
|||
*/
|
||||
void irq_work_queue(struct irq_work *work)
|
||||
{
|
||||
bool empty;
|
||||
|
||||
/* Only queue if not already pending */
|
||||
if (!irq_work_claim(work))
|
||||
return;
|
||||
|
|
@ -79,30 +70,55 @@ void irq_work_queue(struct irq_work *work)
|
|||
/* Queue the entry and raise the IPI if needed. */
|
||||
preempt_disable();
|
||||
|
||||
empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
|
||||
/* The list was empty, raise self-interrupt to start processing. */
|
||||
if (empty)
|
||||
arch_irq_work_raise();
|
||||
llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
|
||||
|
||||
/*
|
||||
* If the work is not "lazy" or the tick is stopped, raise the irq
|
||||
* work interrupt (if supported by the arch), otherwise, just wait
|
||||
* for the next tick.
|
||||
*/
|
||||
if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
|
||||
if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
|
||||
arch_irq_work_raise();
|
||||
}
|
||||
|
||||
preempt_enable();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_queue);
|
||||
|
||||
/*
|
||||
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
|
||||
* context with local IRQs disabled.
|
||||
*/
|
||||
void irq_work_run(void)
|
||||
bool irq_work_needs_cpu(void)
|
||||
{
|
||||
struct llist_head *this_list;
|
||||
|
||||
this_list = &__get_cpu_var(irq_work_list);
|
||||
if (llist_empty(this_list))
|
||||
return false;
|
||||
|
||||
/* All work should have been flushed before going offline */
|
||||
WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void __irq_work_run(void)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct irq_work *work;
|
||||
struct llist_head *this_list;
|
||||
struct llist_node *llnode;
|
||||
|
||||
|
||||
/*
|
||||
* Reset the "raised" state right before we check the list because
|
||||
* an NMI may enqueue after we find the list empty from the runner.
|
||||
*/
|
||||
__this_cpu_write(irq_work_raised, 0);
|
||||
barrier();
|
||||
|
||||
this_list = &__get_cpu_var(irq_work_list);
|
||||
if (llist_empty(this_list))
|
||||
return;
|
||||
|
||||
BUG_ON(!in_irq());
|
||||
BUG_ON(!irqs_disabled());
|
||||
|
||||
llnode = llist_del_all(this_list);
|
||||
|
|
@ -118,15 +134,27 @@ void irq_work_run(void)
|
|||
* to claim that work don't rely on us to handle their data
|
||||
* while we are in the middle of the func.
|
||||
*/
|
||||
xchg(&work->flags, IRQ_WORK_BUSY);
|
||||
flags = work->flags & ~IRQ_WORK_PENDING;
|
||||
xchg(&work->flags, flags);
|
||||
|
||||
work->func(work);
|
||||
/*
|
||||
* Clear the BUSY bit and return to the free state if
|
||||
* no-one else claimed it meanwhile.
|
||||
*/
|
||||
(void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0);
|
||||
(void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
|
||||
* context with local IRQs disabled.
|
||||
*/
|
||||
void irq_work_run(void)
|
||||
{
|
||||
BUG_ON(!in_irq());
|
||||
__irq_work_run();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_run);
|
||||
|
||||
/*
|
||||
|
|
@ -141,3 +169,35 @@ void irq_work_sync(struct irq_work *work)
|
|||
cpu_relax();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_sync);
|
||||
|
||||
#ifdef CONFIG_HOTPLUG_CPU
|
||||
static int irq_work_cpu_notify(struct notifier_block *self,
|
||||
unsigned long action, void *hcpu)
|
||||
{
|
||||
long cpu = (long)hcpu;
|
||||
|
||||
switch (action) {
|
||||
case CPU_DYING:
|
||||
/* Called from stop_machine */
|
||||
if (WARN_ON_ONCE(cpu != smp_processor_id()))
|
||||
break;
|
||||
__irq_work_run();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return NOTIFY_OK;
|
||||
}
|
||||
|
||||
static struct notifier_block cpu_notify;
|
||||
|
||||
static __init int irq_work_init_cpu_notifier(void)
|
||||
{
|
||||
cpu_notify.notifier_call = irq_work_cpu_notify;
|
||||
cpu_notify.priority = 0;
|
||||
register_cpu_notifier(&cpu_notify);
|
||||
return 0;
|
||||
}
|
||||
device_initcall(irq_work_init_cpu_notifier);
|
||||
|
||||
#endif /* CONFIG_HOTPLUG_CPU */
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@
|
|||
#include <linux/notifier.h>
|
||||
#include <linux/rculist.h>
|
||||
#include <linux/poll.h>
|
||||
#include <linux/irq_work.h>
|
||||
|
||||
#include <asm/uaccess.h>
|
||||
|
||||
|
|
@ -1967,30 +1968,32 @@ int is_console_locked(void)
|
|||
static DEFINE_PER_CPU(int, printk_pending);
|
||||
static DEFINE_PER_CPU(char [PRINTK_BUF_SIZE], printk_sched_buf);
|
||||
|
||||
void printk_tick(void)
|
||||
static void wake_up_klogd_work_func(struct irq_work *irq_work)
|
||||
{
|
||||
if (__this_cpu_read(printk_pending)) {
|
||||
int pending = __this_cpu_xchg(printk_pending, 0);
|
||||
if (pending & PRINTK_PENDING_SCHED) {
|
||||
char *buf = __get_cpu_var(printk_sched_buf);
|
||||
printk(KERN_WARNING "[sched_delayed] %s", buf);
|
||||
}
|
||||
if (pending & PRINTK_PENDING_WAKEUP)
|
||||
wake_up_interruptible(&log_wait);
|
||||
int pending = __this_cpu_xchg(printk_pending, 0);
|
||||
|
||||
if (pending & PRINTK_PENDING_SCHED) {
|
||||
char *buf = __get_cpu_var(printk_sched_buf);
|
||||
printk(KERN_WARNING "[sched_delayed] %s", buf);
|
||||
}
|
||||
|
||||
if (pending & PRINTK_PENDING_WAKEUP)
|
||||
wake_up_interruptible(&log_wait);
|
||||
}
|
||||
|
||||
int printk_needs_cpu(int cpu)
|
||||
{
|
||||
if (cpu_is_offline(cpu))
|
||||
printk_tick();
|
||||
return __this_cpu_read(printk_pending);
|
||||
}
|
||||
static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) = {
|
||||
.func = wake_up_klogd_work_func,
|
||||
.flags = IRQ_WORK_LAZY,
|
||||
};
|
||||
|
||||
void wake_up_klogd(void)
|
||||
{
|
||||
if (waitqueue_active(&log_wait))
|
||||
preempt_disable();
|
||||
if (waitqueue_active(&log_wait)) {
|
||||
this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP);
|
||||
irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
|
||||
}
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
static void console_cont_flush(char *text, size_t size)
|
||||
|
|
@ -2471,6 +2474,7 @@ int printk_sched(const char *fmt, ...)
|
|||
va_end(args);
|
||||
|
||||
__this_cpu_or(printk_pending, PRINTK_PENDING_SCHED);
|
||||
irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
|
||||
local_irq_restore(flags);
|
||||
|
||||
return r;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include <linux/profile.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/irq_work.h>
|
||||
|
||||
#include <asm/irq_regs.h>
|
||||
|
||||
|
|
@ -28,7 +29,7 @@
|
|||
/*
|
||||
* Per cpu nohz control structure
|
||||
*/
|
||||
static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
|
||||
DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
|
||||
|
||||
/*
|
||||
* The time, when the last jiffy update happened. Protected by jiffies_lock.
|
||||
|
|
@ -331,8 +332,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
|
|||
time_delta = timekeeping_max_deferment();
|
||||
} while (read_seqretry(&jiffies_lock, seq));
|
||||
|
||||
if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) ||
|
||||
arch_needs_cpu(cpu)) {
|
||||
if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
|
||||
arch_needs_cpu(cpu) || irq_work_needs_cpu()) {
|
||||
next_jiffies = last_jiffies + 1;
|
||||
delta_jiffies = 1;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1351,7 +1351,6 @@ void update_process_times(int user_tick)
|
|||
account_process_tick(p, user_tick);
|
||||
run_local_timers();
|
||||
rcu_check_callbacks(cpu, user_tick);
|
||||
printk_tick();
|
||||
#ifdef CONFIG_IRQ_WORK
|
||||
if (in_irq())
|
||||
irq_work_run();
|
||||
|
|
|
|||
Loading…
Reference in a new issue