From 5acac1be499d979e3aa463ea73a498888faefcbe Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 4 Dec 2013 18:28:20 +0100 Subject: [PATCH 1/4] tick: Rename tick_check_idle() to tick_irq_enter() This makes the code more symetric against the existing tick functions called on irq exit: tick_irq_exit() and tick_nohz_irq_exit(). These function are also symetric as they mirror each other's action: we start to account idle time on irq exit and we stop this accounting on irq entry. Also the tick is stopped on irq exit and timekeeping catches up with the tickless time elapsed until we reach irq entry. This rename was suggested by Peter Zijlstra a long while ago but it got forgotten in the mass. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Alex Shi Cc: Steven Rostedt Cc: Paul E. McKenney Cc: John Stultz Cc: Kevin Hilman Link: http://lkml.kernel.org/r/1387320692-28460-2-git-send-email-fweisbec@gmail.com Signed-off-by: Frederic Weisbecker --- include/linux/tick.h | 6 +++--- kernel/softirq.c | 2 +- kernel/time/tick-sched.c | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/linux/tick.h b/include/linux/tick.h index 0175d8663b6c..b84773cb9f4c 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -104,7 +104,7 @@ extern struct cpumask *tick_get_broadcast_oneshot_mask(void); extern void tick_clock_notify(void); extern int tick_check_oneshot_change(int allow_nohz); extern struct tick_sched *tick_get_tick_sched(int cpu); -extern void tick_check_idle(void); +extern void tick_irq_enter(void); extern int tick_oneshot_mode_active(void); # ifndef arch_needs_cpu # define arch_needs_cpu(cpu) (0) @@ -112,7 +112,7 @@ extern int tick_oneshot_mode_active(void); # else static inline void tick_clock_notify(void) { } static inline int tick_check_oneshot_change(int allow_nohz) { return 0; } -static inline void tick_check_idle(void) { } +static inline void tick_irq_enter(void) { } static inline int tick_oneshot_mode_active(void) { return 0; } # endif @@ -121,7 +121,7 @@ static inline void tick_init(void) { } static inline void tick_cancel_sched_timer(int cpu) { } static inline void tick_clock_notify(void) { } static inline int tick_check_oneshot_change(int allow_nohz) { return 0; } -static inline void tick_check_idle(void) { } +static inline void tick_irq_enter(void) { } static inline int tick_oneshot_mode_active(void) { return 0; } #endif /* !CONFIG_GENERIC_CLOCKEVENTS */ diff --git a/kernel/softirq.c b/kernel/softirq.c index 11348de09400..ca9cb35a96d4 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -318,7 +318,7 @@ void irq_enter(void) * here, as softirq will be serviced on return from interrupt. */ local_bh_disable(); - tick_check_idle(); + tick_irq_enter(); _local_bh_enable(); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 0ddd020bbaf2..e4d0f093061f 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1023,7 +1023,7 @@ static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now) #endif } -static inline void tick_check_nohz_this_cpu(void) +static inline void tick_nohz_irq_enter(void) { struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); ktime_t now; @@ -1042,17 +1042,17 @@ static inline void tick_check_nohz_this_cpu(void) #else static inline void tick_nohz_switch_to_nohz(void) { } -static inline void tick_check_nohz_this_cpu(void) { } +static inline void tick_nohz_irq_enter(void) { } #endif /* CONFIG_NO_HZ_COMMON */ /* * Called from irq_enter to notify about the possible interruption of idle() */ -void tick_check_idle(void) +void tick_irq_enter(void) { tick_check_oneshot_broadcast_this_cpu(); - tick_check_nohz_this_cpu(); + tick_nohz_irq_enter(); } /* From 855a0fc30b70d6ae681badd24d6625f9a9abb787 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 17 Dec 2013 00:16:37 +0100 Subject: [PATCH 2/4] nohz: Get timekeeping max deferment outside jiffies_lock We don't need to fetch the timekeeping max deferment under the jiffies_lock seqlock. If the clocksource is updated concurrently while we stop the tick, stop machine is called and the tick will be reevaluated again along with uptodate jiffies and its related values. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Alex Shi Cc: Steven Rostedt Cc: Paul E. McKenney Cc: John Stultz Cc: Kevin Hilman Link: http://lkml.kernel.org/r/1387320692-28460-9-git-send-email-fweisbec@gmail.com Signed-off-by: Frederic Weisbecker --- kernel/time/tick-sched.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index e4d0f093061f..68331d16b4ed 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -533,12 +533,13 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; u64 time_delta; + time_delta = timekeeping_max_deferment(); + /* Read jiffies and the time when jiffies were updated last */ do { seq = read_seqbegin(&jiffies_lock); last_update = last_jiffies_update; last_jiffies = jiffies; - time_delta = timekeeping_max_deferment(); } while (read_seqretry(&jiffies_lock, seq)); if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || From e9a2eb403bd953788cd2abfd0d2646d43bd22671 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Thu, 28 Nov 2013 14:27:11 +0800 Subject: [PATCH 3/4] nohz_full: fix code style issue of tick_nohz_full_stop_tick Code usually starts with 'tab' instead of 7 'space' in kernel Signed-off-by: Alex Shi Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Alex Shi Cc: Steven Rostedt Cc: Paul E. McKenney Cc: John Stultz Cc: Kevin Hilman Link: http://lkml.kernel.org/r/1386074112-30754-2-git-send-email-alex.shi@linaro.org Signed-off-by: Frederic Weisbecker --- kernel/time/tick-sched.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 68331d16b4ed..d603baddd52d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -679,18 +679,18 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, static void tick_nohz_full_stop_tick(struct tick_sched *ts) { #ifdef CONFIG_NO_HZ_FULL - int cpu = smp_processor_id(); + int cpu = smp_processor_id(); - if (!tick_nohz_full_cpu(cpu) || is_idle_task(current)) - return; + if (!tick_nohz_full_cpu(cpu) || is_idle_task(current)) + return; - if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) - return; + if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) + return; - if (!can_stop_full_tick()) - return; + if (!can_stop_full_tick()) + return; - tick_nohz_stop_sched_tick(ts, ktime_get(), cpu); + tick_nohz_stop_sched_tick(ts, ktime_get(), cpu); #endif } From 8fe8ff09ce3b5750e1f3e45a1f4a81d59c7ff1f1 Mon Sep 17 00:00:00 2001 From: Kevin Hilman Date: Wed, 15 Jan 2014 14:51:38 +0100 Subject: [PATCH 4/4] sched/nohz: Fix overflow error in scheduler_tick_max_deferment() While calculating the scheduler tick max deferment, the delta is converted from microseconds to nanoseconds through a multiplication against NSEC_PER_USEC. But this microseconds operand is an unsigned int, thus the result may likely overflow. The result is cast to u64 but only once the operation is completed, which is too late to avoid overflown result. This is currently not a problem because the scheduler tick max deferment is 1 second. But this may become an issue as we plan to make this value tunable. So lets fix this by casting the usecs value to u64 before multiplying by NSECS_PER_USEC. Also to prevent from this kind of mistake to happen again, move this ad-hoc jiffies -> nsecs conversion to a new helper. Signed-off-by: Kevin Hilman Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Alex Shi Cc: Steven Rostedt Cc: Paul E. McKenney Cc: John Stultz Cc: Kevin Hilman Link: http://lkml.kernel.org/r/1387315388-31676-2-git-send-email-khilman@linaro.org [move ad-hoc conversion to jiffies_to_nsecs helper] Signed-off-by: Frederic Weisbecker --- include/linux/jiffies.h | 6 ++++++ kernel/sched/core.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index d235e88cfd7c..1f44466c1e9d 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -294,6 +294,12 @@ extern unsigned long preset_lpj; */ extern unsigned int jiffies_to_msecs(const unsigned long j); extern unsigned int jiffies_to_usecs(const unsigned long j); + +static inline u64 jiffies_to_nsecs(const unsigned long j) +{ + return (u64)jiffies_to_usecs(j) * NSEC_PER_USEC; +} + extern unsigned long msecs_to_jiffies(const unsigned int m); extern unsigned long usecs_to_jiffies(const unsigned int u); extern unsigned long timespec_to_jiffies(const struct timespec *value); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a88f4a485c5e..61e601fc2b1e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2325,7 +2325,7 @@ u64 scheduler_tick_max_deferment(void) if (time_before_eq(next, now)) return 0; - return jiffies_to_usecs(next - now) * NSEC_PER_USEC; + return jiffies_to_nsecs(next - now); } #endif