diff options
| -rw-r--r-- | include/linux/tick.h | 2 | ||||
| -rw-r--r-- | kernel/sched/idle.c | 11 | ||||
| -rw-r--r-- | kernel/time/tick-sched.c | 61 | ||||
| -rw-r--r-- | kernel/time/tick-sched.h | 2 |
4 files changed, 59 insertions, 17 deletions
diff --git a/include/linux/tick.h b/include/linux/tick.h index ef0717e5e526..e8e7ff16b929 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -116,6 +116,7 @@ extern bool tick_nohz_enabled; extern bool tick_nohz_tick_stopped(void); extern bool tick_nohz_tick_stopped_cpu(int cpu); extern void tick_nohz_idle_stop_tick(void); +extern void tick_nohz_idle_retain_tick(void); extern void tick_nohz_idle_restart_tick(void); extern void tick_nohz_idle_enter(void); extern void tick_nohz_idle_exit(void); @@ -139,6 +140,7 @@ static inline void tick_nohz_idle_stop_tick_protected(void) static inline int tick_nohz_tick_stopped(void) { return 0; } static inline int tick_nohz_tick_stopped_cpu(int cpu) { return 0; } static inline void tick_nohz_idle_stop_tick(void) { } +static inline void tick_nohz_idle_retain_tick(void) { } static inline void tick_nohz_idle_restart_tick(void) { } static inline void tick_nohz_idle_enter(void) { } static inline void tick_nohz_idle_exit(void) { } diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index a966bd2a6fa0..1a3e9bddd17b 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -185,13 +185,18 @@ static void cpuidle_idle_call(void) } else { bool stop_tick = true; - tick_nohz_idle_stop_tick(); - rcu_idle_enter(); - /* * Ask the cpuidle framework to choose a convenient idle state. */ next_state = cpuidle_select(drv, dev, &stop_tick); + + if (stop_tick) + tick_nohz_idle_stop_tick(); + else + tick_nohz_idle_retain_tick(); + + rcu_idle_enter(); + entered_state = call_cpuidle(drv, dev, next_state); /* * Give the governor an opportunity to reflect on the outcome diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index f56d2c695712..c57c98c7e953 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -913,16 +913,19 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) { - struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); ktime_t expires; int cpu = smp_processor_id(); - WARN_ON_ONCE(ts->timer_expires_base); - - if (!can_stop_idle_tick(cpu, ts)) - goto out; - - expires = tick_nohz_next_event(ts, cpu); + /* + * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the + * tick timer expiration time is known already. + */ + if (ts->timer_expires_base) + expires = ts->timer_expires; + else if (can_stop_idle_tick(cpu, ts)) + expires = tick_nohz_next_event(ts, cpu); + else + return; ts->idle_calls++; @@ -941,9 +944,6 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) } else { tick_nohz_retain_tick(ts); } - -out: - ts->sleep_length = ktime_sub(dev->next_event, ts->idle_entrytime); } /** @@ -956,6 +956,16 @@ void tick_nohz_idle_stop_tick(void) __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched)); } +void tick_nohz_idle_retain_tick(void) +{ + tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched)); + /* + * Undo the effect of get_next_timer_interrupt() called from + * tick_nohz_next_event(). + */ + timer_clear_idle(); +} + /** * tick_nohz_idle_enter - prepare for entering idle on the current CPU * @@ -1012,15 +1022,42 @@ bool tick_nohz_idle_got_tick(void) } /** - * tick_nohz_get_sleep_length - return the length of the current sleep + * tick_nohz_get_sleep_length - return the expected length of the current sleep * * Called from power state control code with interrupts disabled */ ktime_t tick_nohz_get_sleep_length(void) { + struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); + int cpu = smp_processor_id(); + /* + * The idle entry time is expected to be a sufficient approximation of + * the current time at this point. + */ + ktime_t now = ts->idle_entrytime; + ktime_t next_event; + + WARN_ON_ONCE(!ts->inidle); + + if (!can_stop_idle_tick(cpu, ts)) + goto out_dev; + + next_event = tick_nohz_next_event(ts, cpu); + if (!next_event) + goto out_dev; + + /* + * If the next highres timer to expire is earlier than next_event, the + * idle governor needs to know that. + */ + next_event = min_t(u64, next_event, + hrtimer_next_event_without(&ts->sched_timer)); + + return ktime_sub(next_event, now); - return ts->sleep_length; +out_dev: + return ktime_sub(dev->next_event, now); } /** diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h index 53e45a39bdbc..2b845f2c44b1 100644 --- a/kernel/time/tick-sched.h +++ b/kernel/time/tick-sched.h @@ -38,7 +38,6 @@ enum tick_nohz_mode { * @idle_exittime: Time when the idle state was left * @idle_sleeptime: Sum of the time slept in idle with sched tick stopped * @iowait_sleeptime: Sum of the time slept in idle with sched tick stopped, with IO outstanding - * @sleep_length: Duration of the current idle sleep * @timer_expires: Anticipated timer expiration time (in case sched tick is stopped) * @timer_expires_base: Base time clock monotonic for @timer_expires * @do_timer_lst: CPU was the last one doing do_timer before going idle @@ -60,7 +59,6 @@ struct tick_sched { ktime_t idle_exittime; ktime_t idle_sleeptime; ktime_t iowait_sleeptime; - ktime_t sleep_length; unsigned long last_jiffies; u64 timer_expires; u64 timer_expires_base; |