From 00c5fb774e3fa8c9d082c62eac7e3d178c006f56 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 11:23:15 -0800 Subject: time: Move total_sleep_time into the timekeeper structure Move total_sleep_time into the timekeeper structure in preparation for locking cleanups CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 0c635818640..8427cc20bad 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -47,6 +47,10 @@ struct timekeeper { int ntp_error_shift; /* NTP adjusted clock multiplier */ u32 mult; + + /* time spent in suspend */ + struct timespec total_sleep_time; + }; static struct timekeeper timekeeper; @@ -159,7 +163,6 @@ __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); */ static struct timespec xtime __attribute__ ((aligned (16))); static struct timespec wall_to_monotonic __attribute__ ((aligned (16))); -static struct timespec total_sleep_time; /* * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. @@ -591,8 +594,8 @@ void __init timekeeping_init(void) } set_normalized_timespec(&wall_to_monotonic, -boot.tv_sec, -boot.tv_nsec); - total_sleep_time.tv_sec = 0; - total_sleep_time.tv_nsec = 0; + timekeeper.total_sleep_time.tv_sec = 0; + timekeeper.total_sleep_time.tv_nsec = 0; write_sequnlock_irqrestore(&xtime_lock, flags); } @@ -616,7 +619,8 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) xtime = timespec_add(xtime, *delta); wall_to_monotonic = timespec_sub(wall_to_monotonic, *delta); - total_sleep_time = timespec_add(total_sleep_time, *delta); + timekeeper.total_sleep_time = timespec_add( + timekeeper.total_sleep_time, *delta); } @@ -1074,8 +1078,10 @@ static void update_wall_time(void) void getboottime(struct timespec *ts) { struct timespec boottime = { - .tv_sec = wall_to_monotonic.tv_sec + total_sleep_time.tv_sec, - .tv_nsec = wall_to_monotonic.tv_nsec + total_sleep_time.tv_nsec + .tv_sec = wall_to_monotonic.tv_sec + + timekeeper.total_sleep_time.tv_sec, + .tv_nsec = wall_to_monotonic.tv_nsec + + timekeeper.total_sleep_time.tv_nsec }; set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); @@ -1104,7 +1110,7 @@ void get_monotonic_boottime(struct timespec *ts) seq = read_seqbegin(&xtime_lock); *ts = xtime; tomono = wall_to_monotonic; - sleep = total_sleep_time; + sleep = timekeeper.total_sleep_time; nsecs = timekeeping_get_ns(); } while (read_seqretry(&xtime_lock, seq)); @@ -1137,7 +1143,7 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); */ void monotonic_to_bootbased(struct timespec *ts) { - *ts = timespec_add(*ts, total_sleep_time); + *ts = timespec_add(*ts, timekeeper.total_sleep_time); } EXPORT_SYMBOL_GPL(monotonic_to_bootbased); @@ -1212,7 +1218,7 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, seq = read_seqbegin(&xtime_lock); *xtim = xtime; *wtom = wall_to_monotonic; - *sleep = total_sleep_time; + *sleep = timekeeper.total_sleep_time; } while (read_seqretry(&xtime_lock, seq)); } -- cgit v1.2.3 From d9f7217aac6833cc634741f2f771a87fd1518fee Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 11:29:32 -0800 Subject: time: Move wall_to_monotonic into the timekeeper structure In preparation for locking cleanups, move wall_to_monotonic into the timekeeper structure. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 69 +++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 32 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 8427cc20bad..5655ca3a86d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -48,6 +48,21 @@ struct timekeeper { /* NTP adjusted clock multiplier */ u32 mult; + /* + * wall_to_monotonic is what we need to add to xtime (or xtime corrected + * for sub jiffie times) to get to monotonic time. Monotonic is pegged + * at zero at system boot time, so wall_to_monotonic will be negative, + * however, we will ALWAYS keep the tv_nsec part positive so we can use + * the usual normalization. + * + * wall_to_monotonic is moved after resume from suspend for the + * monotonic time not to jump. We need to add total_sleep_time to + * wall_to_monotonic to get the real boot based time offset. + * + * - wall_to_monotonic is no longer the boot time, getboottime must be + * used instead. + */ + struct timespec wall_to_monotonic; /* time spent in suspend */ struct timespec total_sleep_time; @@ -148,21 +163,8 @@ __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); /* * The current time - * wall_to_monotonic is what we need to add to xtime (or xtime corrected - * for sub jiffie times) to get to monotonic time. Monotonic is pegged - * at zero at system boot time, so wall_to_monotonic will be negative, - * however, we will ALWAYS keep the tv_nsec part positive so we can use - * the usual normalization. - * - * wall_to_monotonic is moved after resume from suspend for the monotonic - * time not to jump. We need to add total_sleep_time to wall_to_monotonic - * to get the real boot based time offset. - * - * - wall_to_monotonic is no longer the boot time, getboottime must be - * used instead. */ static struct timespec xtime __attribute__ ((aligned (16))); -static struct timespec wall_to_monotonic __attribute__ ((aligned (16))); /* * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. @@ -176,8 +178,8 @@ int __read_mostly timekeeping_suspended; void timekeeping_leap_insert(int leapsecond) { xtime.tv_sec += leapsecond; - wall_to_monotonic.tv_sec -= leapsecond; - update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock, + timekeeper.wall_to_monotonic.tv_sec -= leapsecond; + update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); } @@ -249,8 +251,8 @@ ktime_t ktime_get(void) do { seq = read_seqbegin(&xtime_lock); - secs = xtime.tv_sec + wall_to_monotonic.tv_sec; - nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec; + secs = xtime.tv_sec + timekeeper.wall_to_monotonic.tv_sec; + nsecs = xtime.tv_nsec + timekeeper.wall_to_monotonic.tv_nsec; nsecs += timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); @@ -283,7 +285,7 @@ void ktime_get_ts(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); *ts = xtime; - tomono = wall_to_monotonic; + tomono = timekeeper.wall_to_monotonic; nsecs = timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); @@ -374,14 +376,15 @@ int do_settimeofday(const struct timespec *tv) ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec; ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec; - wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta); + timekeeper.wall_to_monotonic = + timespec_sub(timekeeper.wall_to_monotonic, ts_delta); xtime = *tv; timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock, + update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -413,12 +416,13 @@ int timekeeping_inject_offset(struct timespec *ts) timekeeping_forward_now(); xtime = timespec_add(xtime, *ts); - wall_to_monotonic = timespec_sub(wall_to_monotonic, *ts); + timekeeper.wall_to_monotonic = + timespec_sub(timekeeper.wall_to_monotonic, *ts); timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock, + update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -592,7 +596,7 @@ void __init timekeeping_init(void) boot.tv_sec = xtime.tv_sec; boot.tv_nsec = xtime.tv_nsec; } - set_normalized_timespec(&wall_to_monotonic, + set_normalized_timespec(&timekeeper.wall_to_monotonic, -boot.tv_sec, -boot.tv_nsec); timekeeper.total_sleep_time.tv_sec = 0; timekeeper.total_sleep_time.tv_nsec = 0; @@ -618,7 +622,8 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) } xtime = timespec_add(xtime, *delta); - wall_to_monotonic = timespec_sub(wall_to_monotonic, *delta); + timekeeper.wall_to_monotonic = + timespec_sub(timekeeper.wall_to_monotonic, *delta); timekeeper.total_sleep_time = timespec_add( timekeeper.total_sleep_time, *delta); } @@ -651,7 +656,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock, + update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -1060,7 +1065,7 @@ static void update_wall_time(void) } /* check to see if there is a new clocksource to use */ - update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock, + update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); } @@ -1078,9 +1083,9 @@ static void update_wall_time(void) void getboottime(struct timespec *ts) { struct timespec boottime = { - .tv_sec = wall_to_monotonic.tv_sec + + .tv_sec = timekeeper.wall_to_monotonic.tv_sec + timekeeper.total_sleep_time.tv_sec, - .tv_nsec = wall_to_monotonic.tv_nsec + + .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + timekeeper.total_sleep_time.tv_nsec }; @@ -1109,7 +1114,7 @@ void get_monotonic_boottime(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); *ts = xtime; - tomono = wall_to_monotonic; + tomono = timekeeper.wall_to_monotonic; sleep = timekeeper.total_sleep_time; nsecs = timekeeping_get_ns(); @@ -1182,7 +1187,7 @@ struct timespec get_monotonic_coarse(void) seq = read_seqbegin(&xtime_lock); now = xtime; - mono = wall_to_monotonic; + mono = timekeeper.wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, @@ -1217,7 +1222,7 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, do { seq = read_seqbegin(&xtime_lock); *xtim = xtime; - *wtom = wall_to_monotonic; + *wtom = timekeeper.wall_to_monotonic; *sleep = timekeeper.total_sleep_time; } while (read_seqretry(&xtime_lock, seq)); } @@ -1232,7 +1237,7 @@ ktime_t ktime_get_monotonic_offset(void) do { seq = read_seqbegin(&xtime_lock); - wtom = wall_to_monotonic; + wtom = timekeeper.wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); return timespec_to_ktime(wtom); } -- cgit v1.2.3 From 8ff2cb92dd1afcf23e7b5287c43a900b16f40bad Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 11:40:54 -0800 Subject: time: Move xtime into timekeeeper structure In preparation for locking cleanups, move xtime into timekeeper structure. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 91 ++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 44 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 5655ca3a86d..b30ffe6c6b0 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -48,6 +48,8 @@ struct timekeeper { /* NTP adjusted clock multiplier */ u32 mult; + /* The current time */ + struct timespec xtime; /* * wall_to_monotonic is what we need to add to xtime (or xtime corrected * for sub jiffie times) to get to monotonic time. Monotonic is pegged @@ -161,10 +163,6 @@ static inline s64 timekeeping_get_ns_raw(void) __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); -/* - * The current time - */ -static struct timespec xtime __attribute__ ((aligned (16))); /* * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. @@ -177,10 +175,10 @@ int __read_mostly timekeeping_suspended; /* must hold xtime_lock */ void timekeeping_leap_insert(int leapsecond) { - xtime.tv_sec += leapsecond; + timekeeper.xtime.tv_sec += leapsecond; timekeeper.wall_to_monotonic.tv_sec -= leapsecond; - update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, - timekeeper.mult); + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); } /** @@ -207,7 +205,7 @@ static void timekeeping_forward_now(void) /* If arch requires, add in gettimeoffset() */ nsec += arch_gettimeoffset(); - timespec_add_ns(&xtime, nsec); + timespec_add_ns(&timekeeper.xtime, nsec); nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); timespec_add_ns(&raw_time, nsec); @@ -229,7 +227,7 @@ void getnstimeofday(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); - *ts = xtime; + *ts = timekeeper.xtime; nsecs = timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ @@ -251,8 +249,10 @@ ktime_t ktime_get(void) do { seq = read_seqbegin(&xtime_lock); - secs = xtime.tv_sec + timekeeper.wall_to_monotonic.tv_sec; - nsecs = xtime.tv_nsec + timekeeper.wall_to_monotonic.tv_nsec; + secs = timekeeper.xtime.tv_sec + + timekeeper.wall_to_monotonic.tv_sec; + nsecs = timekeeper.xtime.tv_nsec + + timekeeper.wall_to_monotonic.tv_nsec; nsecs += timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); @@ -284,7 +284,7 @@ void ktime_get_ts(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); - *ts = xtime; + *ts = timekeeper.xtime; tomono = timekeeper.wall_to_monotonic; nsecs = timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ @@ -321,7 +321,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) seq = read_seqbegin(&xtime_lock); *ts_raw = raw_time; - *ts_real = xtime; + *ts_real = timekeeper.xtime; nsecs_raw = timekeeping_get_ns_raw(); nsecs_real = timekeeping_get_ns(); @@ -374,18 +374,18 @@ int do_settimeofday(const struct timespec *tv) timekeeping_forward_now(); - ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec; - ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec; + ts_delta.tv_sec = tv->tv_sec - timekeeper.xtime.tv_sec; + ts_delta.tv_nsec = tv->tv_nsec - timekeeper.xtime.tv_nsec; timekeeper.wall_to_monotonic = timespec_sub(timekeeper.wall_to_monotonic, ts_delta); - xtime = *tv; + timekeeper.xtime = *tv; timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, - timekeeper.mult); + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -415,15 +415,15 @@ int timekeeping_inject_offset(struct timespec *ts) timekeeping_forward_now(); - xtime = timespec_add(xtime, *ts); + timekeeper.xtime = timespec_add(timekeeper.xtime, *ts); timekeeper.wall_to_monotonic = timespec_sub(timekeeper.wall_to_monotonic, *ts); timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, - timekeeper.mult); + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -588,13 +588,13 @@ void __init timekeeping_init(void) clock->enable(clock); timekeeper_setup_internals(clock); - xtime.tv_sec = now.tv_sec; - xtime.tv_nsec = now.tv_nsec; + timekeeper.xtime.tv_sec = now.tv_sec; + timekeeper.xtime.tv_nsec = now.tv_nsec; raw_time.tv_sec = 0; raw_time.tv_nsec = 0; if (boot.tv_sec == 0 && boot.tv_nsec == 0) { - boot.tv_sec = xtime.tv_sec; - boot.tv_nsec = xtime.tv_nsec; + boot.tv_sec = timekeeper.xtime.tv_sec; + boot.tv_nsec = timekeeper.xtime.tv_nsec; } set_normalized_timespec(&timekeeper.wall_to_monotonic, -boot.tv_sec, -boot.tv_nsec); @@ -621,7 +621,7 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) return; } - xtime = timespec_add(xtime, *delta); + timekeeper.xtime = timespec_add(timekeeper.xtime, *delta); timekeeper.wall_to_monotonic = timespec_sub(timekeeper.wall_to_monotonic, *delta); timekeeper.total_sleep_time = timespec_add( @@ -656,8 +656,8 @@ void timekeeping_inject_sleeptime(struct timespec *delta) timekeeper.ntp_error = 0; ntp_clear(); - update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, - timekeeper.mult); + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); write_sequnlock_irqrestore(&xtime_lock, flags); @@ -720,7 +720,7 @@ static int timekeeping_suspend(void) * try to compensate so the difference in system time * and persistent_clock time stays close to constant. */ - delta = timespec_sub(xtime, timekeeping_suspend_time); + delta = timespec_sub(timekeeper.xtime, timekeeping_suspend_time); delta_delta = timespec_sub(delta, old_delta); if (abs(delta_delta.tv_sec) >= 2) { /* @@ -952,7 +952,7 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift) timekeeper.xtime_nsec += timekeeper.xtime_interval << shift; while (timekeeper.xtime_nsec >= nsecps) { timekeeper.xtime_nsec -= nsecps; - xtime.tv_sec++; + timekeeper.xtime.tv_sec++; second_overflow(); } @@ -998,7 +998,8 @@ static void update_wall_time(void) #else offset = (clock->read(clock) - clock->cycle_last) & clock->mask; #endif - timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift; + timekeeper.xtime_nsec = (s64)timekeeper.xtime.tv_nsec << + timekeeper.shift; /* * With NO_HZ we may have to accumulate many cycle_intervals @@ -1049,8 +1050,10 @@ static void update_wall_time(void) * Store full nanoseconds into xtime after rounding it up and * add the remainder to the error difference. */ - xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1; - timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift; + timekeeper.xtime.tv_nsec = ((s64)timekeeper.xtime_nsec >> + timekeeper.shift) + 1; + timekeeper.xtime_nsec -= (s64)timekeeper.xtime.tv_nsec << + timekeeper.shift; timekeeper.ntp_error += timekeeper.xtime_nsec << timekeeper.ntp_error_shift; @@ -1058,15 +1061,15 @@ static void update_wall_time(void) * Finally, make sure that after the rounding * xtime.tv_nsec isn't larger then NSEC_PER_SEC */ - if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) { - xtime.tv_nsec -= NSEC_PER_SEC; - xtime.tv_sec++; + if (unlikely(timekeeper.xtime.tv_nsec >= NSEC_PER_SEC)) { + timekeeper.xtime.tv_nsec -= NSEC_PER_SEC; + timekeeper.xtime.tv_sec++; second_overflow(); } /* check to see if there is a new clocksource to use */ - update_vsyscall(&xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, - timekeeper.mult); + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); } /** @@ -1113,7 +1116,7 @@ void get_monotonic_boottime(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); - *ts = xtime; + *ts = timekeeper.xtime; tomono = timekeeper.wall_to_monotonic; sleep = timekeeper.total_sleep_time; nsecs = timekeeping_get_ns(); @@ -1154,13 +1157,13 @@ EXPORT_SYMBOL_GPL(monotonic_to_bootbased); unsigned long get_seconds(void) { - return xtime.tv_sec; + return timekeeper.xtime.tv_sec; } EXPORT_SYMBOL(get_seconds); struct timespec __current_kernel_time(void) { - return xtime; + return timekeeper.xtime; } struct timespec current_kernel_time(void) @@ -1171,7 +1174,7 @@ struct timespec current_kernel_time(void) do { seq = read_seqbegin(&xtime_lock); - now = xtime; + now = timekeeper.xtime; } while (read_seqretry(&xtime_lock, seq)); return now; @@ -1186,7 +1189,7 @@ struct timespec get_monotonic_coarse(void) do { seq = read_seqbegin(&xtime_lock); - now = xtime; + now = timekeeper.xtime; mono = timekeeper.wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); @@ -1221,7 +1224,7 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, do { seq = read_seqbegin(&xtime_lock); - *xtim = xtime; + *xtim = timekeeper.xtime; *wtom = timekeeper.wall_to_monotonic; *sleep = timekeeper.total_sleep_time; } while (read_seqretry(&xtime_lock, seq)); -- cgit v1.2.3 From 01f71b47e08f2a062c4e77c94dfa9a7e0ae65fcb Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 11:43:49 -0800 Subject: time: Move raw_time into timekeeper structure In preparation for locking cleanups, move raw_time into timekeeper structure. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index b30ffe6c6b0..fbbc3c7ce7d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -67,7 +67,8 @@ struct timekeeper { struct timespec wall_to_monotonic; /* time spent in suspend */ struct timespec total_sleep_time; - + /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ + struct timespec raw_time; }; static struct timekeeper timekeeper; @@ -164,10 +165,6 @@ __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); -/* - * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. - */ -static struct timespec raw_time; /* flag for if timekeeping is suspended */ int __read_mostly timekeeping_suspended; @@ -208,7 +205,7 @@ static void timekeeping_forward_now(void) timespec_add_ns(&timekeeper.xtime, nsec); nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); - timespec_add_ns(&raw_time, nsec); + timespec_add_ns(&timekeeper.raw_time, nsec); } /** @@ -320,7 +317,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) seq = read_seqbegin(&xtime_lock); - *ts_raw = raw_time; + *ts_raw = timekeeper.raw_time; *ts_real = timekeeper.xtime; nsecs_raw = timekeeping_get_ns_raw(); @@ -499,7 +496,7 @@ void getrawmonotonic(struct timespec *ts) do { seq = read_seqbegin(&xtime_lock); nsecs = timekeeping_get_ns_raw(); - *ts = raw_time; + *ts = timekeeper.raw_time; } while (read_seqretry(&xtime_lock, seq)); @@ -590,8 +587,8 @@ void __init timekeeping_init(void) timekeeper.xtime.tv_sec = now.tv_sec; timekeeper.xtime.tv_nsec = now.tv_nsec; - raw_time.tv_sec = 0; - raw_time.tv_nsec = 0; + timekeeper.raw_time.tv_sec = 0; + timekeeper.raw_time.tv_nsec = 0; if (boot.tv_sec == 0 && boot.tv_nsec == 0) { boot.tv_sec = timekeeper.xtime.tv_sec; boot.tv_nsec = timekeeper.xtime.tv_nsec; @@ -958,13 +955,13 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift) /* Accumulate raw time */ raw_nsecs = timekeeper.raw_interval << shift; - raw_nsecs += raw_time.tv_nsec; + raw_nsecs += timekeeper.raw_time.tv_nsec; if (raw_nsecs >= NSEC_PER_SEC) { u64 raw_secs = raw_nsecs; raw_nsecs = do_div(raw_secs, NSEC_PER_SEC); - raw_time.tv_sec += raw_secs; + timekeeper.raw_time.tv_sec += raw_secs; } - raw_time.tv_nsec = raw_nsecs; + timekeeper.raw_time.tv_nsec = raw_nsecs; /* Accumulate error between NTP and clock interval */ timekeeper.ntp_error += tick_length << shift; -- cgit v1.2.3 From 8fcce546be16130865550136831f71097d7fc228 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 11:46:39 -0800 Subject: time: Cleanup global variables and move them to the top Move global xtime_lock and timekeeping_suspended values up to the top of timekeeping.c CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index fbbc3c7ce7d..5df2e7e556c 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -73,6 +73,18 @@ struct timekeeper { static struct timekeeper timekeeper; +/* + * This read-write spinlock protects us from races in SMP while + * playing with xtime. + */ +__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); + + +/* flag for if timekeeping is suspended */ +int __read_mostly timekeeping_suspended; + + + /** * timekeeper_setup_internals - Set up internals to use clocksource clock. * @@ -157,18 +169,6 @@ static inline s64 timekeeping_get_ns_raw(void) return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); } -/* - * This read-write spinlock protects us from races in SMP while - * playing with xtime. - */ -__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); - - - - -/* flag for if timekeeping is suspended */ -int __read_mostly timekeeping_suspended; - /* must hold xtime_lock */ void timekeeping_leap_insert(int leapsecond) { -- cgit v1.2.3 From 70471f2f061d59375e959b4e7d47ee62121babb1 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 12:48:10 -0800 Subject: time: Add timekeeper lock Now that all the timekeeping variables are stored in the timekeeper structure, add a new lock to protect the structure. For now, this lock nests under the xtime_lock for writes. For readers, we don't need to take xtime_lock anymore. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 126 ++++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 44 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 5df2e7e556c..f5d4d226def 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -69,6 +69,9 @@ struct timekeeper { struct timespec total_sleep_time; /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ struct timespec raw_time; + + /* Seqlock for all timekeeper values */ + seqlock_t lock; }; static struct timekeeper timekeeper; @@ -172,10 +175,17 @@ static inline s64 timekeeping_get_ns_raw(void) /* must hold xtime_lock */ void timekeeping_leap_insert(int leapsecond) { + unsigned long flags; + + write_seqlock_irqsave(&timekeeper.lock, flags); + timekeeper.xtime.tv_sec += leapsecond; timekeeper.wall_to_monotonic.tv_sec -= leapsecond; update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); + + write_sequnlock_irqrestore(&timekeeper.lock, flags); + } /** @@ -222,7 +232,7 @@ void getnstimeofday(struct timespec *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); *ts = timekeeper.xtime; nsecs = timekeeping_get_ns(); @@ -230,7 +240,7 @@ void getnstimeofday(struct timespec *ts) /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); timespec_add_ns(ts, nsecs); } @@ -245,7 +255,7 @@ ktime_t ktime_get(void) WARN_ON(timekeeping_suspended); do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); secs = timekeeper.xtime.tv_sec + timekeeper.wall_to_monotonic.tv_sec; nsecs = timekeeper.xtime.tv_nsec + @@ -254,7 +264,7 @@ ktime_t ktime_get(void) /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); /* * Use ktime_set/ktime_add_ns to create a proper ktime on * 32-bit architectures without CONFIG_KTIME_SCALAR. @@ -280,14 +290,14 @@ void ktime_get_ts(struct timespec *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); *ts = timekeeper.xtime; tomono = timekeeper.wall_to_monotonic; nsecs = timekeeping_get_ns(); /* If arch requires, add in gettimeoffset() */ nsecs += arch_gettimeoffset(); - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, ts->tv_nsec + tomono.tv_nsec + nsecs); @@ -315,7 +325,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) do { u32 arch_offset; - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); *ts_raw = timekeeper.raw_time; *ts_real = timekeeper.xtime; @@ -328,7 +338,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) nsecs_raw += arch_offset; nsecs_real += arch_offset; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); timespec_add_ns(ts_raw, nsecs_raw); timespec_add_ns(ts_real, nsecs_real); @@ -362,12 +372,13 @@ EXPORT_SYMBOL(do_gettimeofday); int do_settimeofday(const struct timespec *tv) { struct timespec ts_delta; - unsigned long flags; + unsigned long flags1,flags2; if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) return -EINVAL; - write_seqlock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags1); + write_seqlock_irqsave(&timekeeper.lock, flags2); timekeeping_forward_now(); @@ -384,7 +395,8 @@ int do_settimeofday(const struct timespec *tv) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags2); + write_sequnlock_irqrestore(&xtime_lock, flags1); /* signal hrtimers about time change */ clock_was_set(); @@ -403,12 +415,13 @@ EXPORT_SYMBOL(do_settimeofday); */ int timekeeping_inject_offset(struct timespec *ts) { - unsigned long flags; + unsigned long flags1,flags2; if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) return -EINVAL; - write_seqlock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags1); + write_seqlock_irqsave(&timekeeper.lock, flags2); timekeeping_forward_now(); @@ -422,7 +435,8 @@ int timekeeping_inject_offset(struct timespec *ts) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags2); + write_sequnlock_irqrestore(&xtime_lock, flags1); /* signal hrtimers about time change */ clock_was_set(); @@ -494,11 +508,11 @@ void getrawmonotonic(struct timespec *ts) s64 nsecs; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); nsecs = timekeeping_get_ns_raw(); *ts = timekeeper.raw_time; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); timespec_add_ns(ts, nsecs); } @@ -514,24 +528,30 @@ int timekeeping_valid_for_hres(void) int ret; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); return ret; } /** * timekeeping_max_deferment - Returns max time the clocksource can be deferred - * - * Caller must observe xtime_lock via read_seqbegin/read_seqretry to - * ensure that the clocksource does not change! */ u64 timekeeping_max_deferment(void) { - return timekeeper.clock->max_idle_ns; + unsigned long seq; + u64 ret; + do { + seq = read_seqbegin(&timekeeper.lock); + + ret = timekeeper.clock->max_idle_ns; + + } while (read_seqretry(&timekeeper.lock, seq)); + + return ret; } /** @@ -576,10 +596,13 @@ void __init timekeeping_init(void) read_persistent_clock(&now); read_boot_clock(&boot); - write_seqlock_irqsave(&xtime_lock, flags); + seqlock_init(&timekeeper.lock); + write_seqlock_irqsave(&xtime_lock, flags); ntp_init(); + write_sequnlock_irqrestore(&xtime_lock, flags); + write_seqlock_irqsave(&timekeeper.lock, flags); clock = clocksource_default_clock(); if (clock->enable) clock->enable(clock); @@ -597,7 +620,7 @@ void __init timekeeping_init(void) -boot.tv_sec, -boot.tv_nsec); timekeeper.total_sleep_time.tv_sec = 0; timekeeper.total_sleep_time.tv_nsec = 0; - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags); } /* time in seconds when suspend began */ @@ -638,7 +661,7 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) */ void timekeeping_inject_sleeptime(struct timespec *delta) { - unsigned long flags; + unsigned long flags1,flags2; struct timespec ts; /* Make sure we don't set the clock twice */ @@ -646,7 +669,9 @@ void timekeeping_inject_sleeptime(struct timespec *delta) if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) return; - write_seqlock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags1); + write_seqlock_irqsave(&timekeeper.lock, flags2); + timekeeping_forward_now(); __timekeeping_inject_sleeptime(delta); @@ -656,7 +681,8 @@ void timekeeping_inject_sleeptime(struct timespec *delta) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags2); + write_sequnlock_irqrestore(&xtime_lock, flags1); /* signal hrtimers about time change */ clock_was_set(); @@ -672,14 +698,15 @@ void timekeeping_inject_sleeptime(struct timespec *delta) */ static void timekeeping_resume(void) { - unsigned long flags; + unsigned long flags1,flags2; struct timespec ts; read_persistent_clock(&ts); clocksource_resume(); - write_seqlock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags1); + write_seqlock_irqsave(&timekeeper.lock, flags2); if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { ts = timespec_sub(ts, timekeeping_suspend_time); @@ -689,7 +716,8 @@ static void timekeeping_resume(void) timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); timekeeper.ntp_error = 0; timekeeping_suspended = 0; - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags2); + write_sequnlock_irqrestore(&xtime_lock, flags1); touch_softlockup_watchdog(); @@ -701,13 +729,14 @@ static void timekeeping_resume(void) static int timekeeping_suspend(void) { - unsigned long flags; + unsigned long flags1,flags2; struct timespec delta, delta_delta; static struct timespec old_delta; read_persistent_clock(&timekeeping_suspend_time); - write_seqlock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags1); + write_seqlock_irqsave(&timekeeper.lock, flags2); timekeeping_forward_now(); timekeeping_suspended = 1; @@ -730,7 +759,8 @@ static int timekeeping_suspend(void) timekeeping_suspend_time = timespec_add(timekeeping_suspend_time, delta_delta); } - write_sequnlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&timekeeper.lock, flags2); + write_sequnlock_irqrestore(&xtime_lock, flags1); clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); clocksource_suspend(); @@ -983,10 +1013,13 @@ static void update_wall_time(void) struct clocksource *clock; cycle_t offset; int shift = 0, maxshift; + unsigned long flags; + + write_seqlock_irqsave(&timekeeper.lock, flags); /* Make sure we're fully resumed: */ if (unlikely(timekeeping_suspended)) - return; + goto out; clock = timekeeper.clock; @@ -1067,6 +1100,10 @@ static void update_wall_time(void) /* check to see if there is a new clocksource to use */ update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); + +out: + write_sequnlock_irqrestore(&timekeeper.lock, flags); + } /** @@ -1112,13 +1149,13 @@ void get_monotonic_boottime(struct timespec *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); *ts = timekeeper.xtime; tomono = timekeeper.wall_to_monotonic; sleep = timekeeper.total_sleep_time; nsecs = timekeeping_get_ns(); - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs); @@ -1169,10 +1206,10 @@ struct timespec current_kernel_time(void) unsigned long seq; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); now = timekeeper.xtime; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); return now; } @@ -1184,11 +1221,11 @@ struct timespec get_monotonic_coarse(void) unsigned long seq; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); now = timekeeper.xtime; mono = timekeeper.wall_to_monotonic; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, now.tv_nsec + mono.tv_nsec); @@ -1220,11 +1257,11 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, unsigned long seq; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); *xtim = timekeeper.xtime; *wtom = timekeeper.wall_to_monotonic; *sleep = timekeeper.total_sleep_time; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); } /** @@ -1236,9 +1273,10 @@ ktime_t ktime_get_monotonic_offset(void) struct timespec wtom; do { - seq = read_seqbegin(&xtime_lock); + seq = read_seqbegin(&timekeeper.lock); wtom = timekeeper.wall_to_monotonic; - } while (read_seqretry(&xtime_lock, seq)); + } while (read_seqretry(&timekeeper.lock, seq)); + return timespec_to_ktime(wtom); } -- cgit v1.2.3 From 8357929e6ae3661d5a3a7378a717f29873ea18c6 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 13:06:21 -0800 Subject: ntp: Cleanup timex.h Move ntp_sycned to ntp.c and mark time_status as static. Also yank function declaration for non-existant function. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- include/linux/timex.h | 15 --------------- kernel/time/ntp.c | 13 ++++++++++++- 2 files changed, 12 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/include/linux/timex.h b/include/linux/timex.h index aa60fe7b6ed..92e01fc31ca 100644 --- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -234,23 +234,9 @@ struct timex { extern unsigned long tick_usec; /* USER_HZ period (usec) */ extern unsigned long tick_nsec; /* ACTHZ period (nsec) */ -/* - * phase-lock loop variables - */ -extern int time_status; /* clock synchronization status bits */ - extern void ntp_init(void); extern void ntp_clear(void); -/** - * ntp_synced - Returns 1 if the NTP status is not UNSYNC - * - */ -static inline int ntp_synced(void) -{ - return !(time_status & STA_UNSYNC); -} - /* Required to safely shift negative values */ #define shift_right(x, s) ({ \ __typeof__(x) __x = (x); \ @@ -267,7 +253,6 @@ static inline int ntp_synced(void) extern u64 tick_length; extern void second_overflow(void); -extern void update_ntp_one_tick(void); extern int do_adjtimex(struct timex *); extern void hardpps(const struct timespec *, const struct timespec *); diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index f6117a4c7cb..ae7e13607d9 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -49,7 +49,7 @@ static struct hrtimer leap_timer; static int time_state = TIME_OK; /* clock status bits: */ -int time_status = STA_UNSYNC; +static int time_status = STA_UNSYNC; /* TAI offset (secs): */ static long time_tai; @@ -233,6 +233,17 @@ static inline void pps_fill_timex(struct timex *txc) #endif /* CONFIG_NTP_PPS */ + +/** + * ntp_synced - Returns 1 if the NTP status is not UNSYNC + * + */ +static inline int ntp_synced(void) +{ + return !(time_status & STA_UNSYNC); +} + + /* * NTP methods: */ -- cgit v1.2.3 From ea7cf49a7633c2b70125f59b4e3553d9181cb15d Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 13:18:07 -0800 Subject: ntp: Access tick_length variable via ntp_tick_length() Currently the NTP managed tick_length value is accessed globally, in preparations for locking cleanups, make sure it is accessed via a function and mark it as static. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- include/linux/timex.h | 2 +- kernel/time/ntp.c | 9 ++++++++- kernel/time/timekeeping.c | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/include/linux/timex.h b/include/linux/timex.h index 92e01fc31ca..b75e1864ed1 100644 --- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -250,7 +250,7 @@ extern void ntp_clear(void); #define NTP_INTERVAL_LENGTH (NSEC_PER_SEC/NTP_INTERVAL_FREQ) /* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */ -extern u64 tick_length; +extern u64 ntp_tick_length(void); extern void second_overflow(void); extern int do_adjtimex(struct timex *); diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index ae7e13607d9..f131ba62da6 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -28,7 +28,7 @@ unsigned long tick_usec = TICK_USEC; /* ACTHZ period (nsecs): */ unsigned long tick_nsec; -u64 tick_length; +static u64 tick_length; static u64 tick_length_base; static struct hrtimer leap_timer; @@ -360,6 +360,13 @@ void ntp_clear(void) pps_clear(); } + +u64 ntp_tick_length(void) +{ + return tick_length; +} + + /* * Leap second processing. If in leap-insert state at the end of the * day, the system clock is set back one second; if in leap-delete diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index f5d4d226def..cdae24655c8 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -811,7 +811,7 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, * Now calculate the error in (1 << look_ahead) ticks, but first * remove the single look ahead already included in the error. */ - tick_error = tick_length >> (timekeeper.ntp_error_shift + 1); + tick_error = ntp_tick_length() >> (timekeeper.ntp_error_shift + 1); tick_error -= timekeeper.xtime_interval >> 1; error = ((error - tick_error) >> look_ahead) + tick_error; @@ -994,7 +994,7 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift) timekeeper.raw_time.tv_nsec = raw_nsecs; /* Accumulate error between NTP and clock interval */ - timekeeper.ntp_error += tick_length << shift; + timekeeper.ntp_error += ntp_tick_length() << shift; timekeeper.ntp_error -= (timekeeper.xtime_interval + timekeeper.xtime_remainder) << (timekeeper.ntp_error_shift + shift); @@ -1042,7 +1042,7 @@ static void update_wall_time(void) shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); shift = max(0, shift); /* Bound shift to one less then what overflows tick_length */ - maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1; + maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; shift = min(shift, maxshift); while (offset >= timekeeper.cycle_interval) { offset = logarithmic_accumulation(offset, shift); -- cgit v1.2.3 From bd3312681f69207a40431981c1bce1afdc9b7975 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 13:48:36 -0800 Subject: ntp: Add ntp_lock to replace xtime_locking Use a ntp_lock spin lock to replace xtime_lock locking in ntp.c CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/ntp.c | 63 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index f131ba62da6..17fb1b9807d 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -22,6 +22,9 @@ * NTP timekeeping variables: */ +DEFINE_SPINLOCK(ntp_lock); + + /* USER_HZ period (usecs): */ unsigned long tick_usec = TICK_USEC; @@ -133,7 +136,7 @@ static inline void pps_reset_freq_interval(void) /** * pps_clear - Clears the PPS state variables * - * Must be called while holding a write on the xtime_lock + * Must be called while holding a write on the ntp_lock */ static inline void pps_clear(void) { @@ -149,7 +152,7 @@ static inline void pps_clear(void) * the last PPS signal. When it reaches 0, indicate that PPS signal is * missing. * - * Must be called while holding a write on the xtime_lock + * Must be called while holding a write on the ntp_lock */ static inline void pps_dec_valid(void) { @@ -341,11 +344,13 @@ static void ntp_update_offset(long offset) /** * ntp_clear - Clears the NTP state variables - * - * Must be called while holding a write on the xtime_lock */ void ntp_clear(void) { + unsigned long flags; + + spin_lock_irqsave(&ntp_lock, flags); + time_adjust = 0; /* stop active adjtime() */ time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; @@ -358,12 +363,20 @@ void ntp_clear(void) /* Clear PPS state variables */ pps_clear(); + spin_unlock_irqrestore(&ntp_lock, flags); + } u64 ntp_tick_length(void) { - return tick_length; + unsigned long flags; + s64 ret; + + spin_lock_irqsave(&ntp_lock, flags); + ret = tick_length; + spin_unlock_irqrestore(&ntp_lock, flags); + return ret; } @@ -375,14 +388,15 @@ u64 ntp_tick_length(void) static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) { enum hrtimer_restart res = HRTIMER_NORESTART; + unsigned long flags; + int leap = 0; - write_seqlock(&xtime_lock); - + spin_lock_irqsave(&ntp_lock, flags); switch (time_state) { case TIME_OK: break; case TIME_INS: - timekeeping_leap_insert(-1); + leap = -1; time_state = TIME_OOP; printk(KERN_NOTICE "Clock: inserting leap second 23:59:60 UTC\n"); @@ -390,7 +404,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) res = HRTIMER_RESTART; break; case TIME_DEL: - timekeeping_leap_insert(1); + leap = 1; time_tai--; time_state = TIME_WAIT; printk(KERN_NOTICE @@ -405,8 +419,14 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) time_state = TIME_OK; break; } + spin_unlock_irqrestore(&ntp_lock, flags); - write_sequnlock(&xtime_lock); + /* + * We have to call this outside of the ntp_lock to keep + * the proper locking hierarchy + */ + if (leap) + timekeeping_leap_insert(leap); return res; } @@ -422,6 +442,9 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) void second_overflow(void) { s64 delta; + unsigned long flags; + + spin_lock_irqsave(&ntp_lock, flags); /* Bump the maxerror field */ time_maxerror += MAXFREQ / NSEC_PER_USEC; @@ -441,23 +464,25 @@ void second_overflow(void) pps_dec_valid(); if (!time_adjust) - return; + goto out; if (time_adjust > MAX_TICKADJ) { time_adjust -= MAX_TICKADJ; tick_length += MAX_TICKADJ_SCALED; - return; + goto out; } if (time_adjust < -MAX_TICKADJ) { time_adjust += MAX_TICKADJ; tick_length -= MAX_TICKADJ_SCALED; - return; + goto out; } tick_length += (s64)(time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ) << NTP_SCALE_SHIFT; time_adjust = 0; +out: + spin_unlock_irqrestore(&ntp_lock, flags); } #ifdef CONFIG_GENERIC_CMOS_UPDATE @@ -681,7 +706,7 @@ int do_adjtimex(struct timex *txc) getnstimeofday(&ts); - write_seqlock_irq(&xtime_lock); + spin_lock_irq(&ntp_lock); if (txc->modes & ADJ_ADJTIME) { long save_adjust = time_adjust; @@ -723,7 +748,7 @@ int do_adjtimex(struct timex *txc) /* fill PPS status fields */ pps_fill_timex(txc); - write_sequnlock_irq(&xtime_lock); + spin_unlock_irq(&ntp_lock); txc->time.tv_sec = ts.tv_sec; txc->time.tv_usec = ts.tv_nsec; @@ -921,7 +946,7 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts) pts_norm = pps_normalize_ts(*phase_ts); - write_seqlock_irqsave(&xtime_lock, flags); + spin_lock_irqsave(&ntp_lock, flags); /* clear the error bits, they will be set again if needed */ time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR); @@ -934,7 +959,7 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts) * just start the frequency interval */ if (unlikely(pps_fbase.tv_sec == 0)) { pps_fbase = *raw_ts; - write_sequnlock_irqrestore(&xtime_lock, flags); + spin_unlock_irqrestore(&ntp_lock, flags); return; } @@ -949,7 +974,7 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts) time_status |= STA_PPSJITTER; /* restart the frequency calibration interval */ pps_fbase = *raw_ts; - write_sequnlock_irqrestore(&xtime_lock, flags); + spin_unlock_irqrestore(&ntp_lock, flags); pr_err("hardpps: PPSJITTER: bad pulse\n"); return; } @@ -966,7 +991,7 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts) hardpps_update_phase(pts_norm.nsec); - write_sequnlock_irqrestore(&xtime_lock, flags); + spin_unlock_irqrestore(&ntp_lock, flags); } EXPORT_SYMBOL(hardpps); -- cgit v1.2.3 From 92c1d3ed4dc0b8cfb10e85ed0c9934db41efc027 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 14 Nov 2011 14:05:44 -0800 Subject: time: Remove most of xtime_lock usage in timekeeping.c Now that ntp.c's locking is reworked, we can remove most of the xtime_lock usage in timekeeping.c The remaining xtime_lock presence is really for jiffies access and the global load calculation. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index cdae24655c8..74bb5701e0b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -172,7 +172,6 @@ static inline s64 timekeeping_get_ns_raw(void) return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); } -/* must hold xtime_lock */ void timekeeping_leap_insert(int leapsecond) { unsigned long flags; @@ -372,13 +371,12 @@ EXPORT_SYMBOL(do_gettimeofday); int do_settimeofday(const struct timespec *tv) { struct timespec ts_delta; - unsigned long flags1,flags2; + unsigned long flags; if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) return -EINVAL; - write_seqlock_irqsave(&xtime_lock, flags1); - write_seqlock_irqsave(&timekeeper.lock, flags2); + write_seqlock_irqsave(&timekeeper.lock, flags); timekeeping_forward_now(); @@ -395,8 +393,7 @@ int do_settimeofday(const struct timespec *tv) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&timekeeper.lock, flags2); - write_sequnlock_irqrestore(&xtime_lock, flags1); + write_sequnlock_irqrestore(&timekeeper.lock, flags); /* signal hrtimers about time change */ clock_was_set(); @@ -415,13 +412,12 @@ EXPORT_SYMBOL(do_settimeofday); */ int timekeeping_inject_offset(struct timespec *ts) { - unsigned long flags1,flags2; + unsigned long flags; if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) return -EINVAL; - write_seqlock_irqsave(&xtime_lock, flags1); - write_seqlock_irqsave(&timekeeper.lock, flags2); + write_seqlock_irqsave(&timekeeper.lock, flags); timekeeping_forward_now(); @@ -435,8 +431,7 @@ int timekeeping_inject_offset(struct timespec *ts) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&timekeeper.lock, flags2); - write_sequnlock_irqrestore(&xtime_lock, flags1); + write_sequnlock_irqrestore(&timekeeper.lock, flags); /* signal hrtimers about time change */ clock_was_set(); @@ -598,9 +593,7 @@ void __init timekeeping_init(void) seqlock_init(&timekeeper.lock); - write_seqlock_irqsave(&xtime_lock, flags); ntp_init(); - write_sequnlock_irqrestore(&xtime_lock, flags); write_seqlock_irqsave(&timekeeper.lock, flags); clock = clocksource_default_clock(); @@ -661,7 +654,7 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) */ void timekeeping_inject_sleeptime(struct timespec *delta) { - unsigned long flags1,flags2; + unsigned long flags; struct timespec ts; /* Make sure we don't set the clock twice */ @@ -669,8 +662,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) return; - write_seqlock_irqsave(&xtime_lock, flags1); - write_seqlock_irqsave(&timekeeper.lock, flags2); + write_seqlock_irqsave(&timekeeper.lock, flags); timekeeping_forward_now(); @@ -681,8 +673,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, timekeeper.clock, timekeeper.mult); - write_sequnlock_irqrestore(&timekeeper.lock, flags2); - write_sequnlock_irqrestore(&xtime_lock, flags1); + write_sequnlock_irqrestore(&timekeeper.lock, flags); /* signal hrtimers about time change */ clock_was_set(); @@ -698,15 +689,14 @@ void timekeeping_inject_sleeptime(struct timespec *delta) */ static void timekeeping_resume(void) { - unsigned long flags1,flags2; + unsigned long flags; struct timespec ts; read_persistent_clock(&ts); clocksource_resume(); - write_seqlock_irqsave(&xtime_lock, flags1); - write_seqlock_irqsave(&timekeeper.lock, flags2); + write_seqlock_irqsave(&timekeeper.lock, flags); if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { ts = timespec_sub(ts, timekeeping_suspend_time); @@ -716,8 +706,7 @@ static void timekeeping_resume(void) timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); timekeeper.ntp_error = 0; timekeeping_suspended = 0; - write_sequnlock_irqrestore(&timekeeper.lock, flags2); - write_sequnlock_irqrestore(&xtime_lock, flags1); + write_sequnlock_irqrestore(&timekeeper.lock, flags); touch_softlockup_watchdog(); @@ -729,14 +718,13 @@ static void timekeeping_resume(void) static int timekeeping_suspend(void) { - unsigned long flags1,flags2; + unsigned long flags; struct timespec delta, delta_delta; static struct timespec old_delta; read_persistent_clock(&timekeeping_suspend_time); - write_seqlock_irqsave(&xtime_lock, flags1); - write_seqlock_irqsave(&timekeeper.lock, flags2); + write_seqlock_irqsave(&timekeeper.lock, flags); timekeeping_forward_now(); timekeeping_suspended = 1; @@ -759,8 +747,7 @@ static int timekeeping_suspend(void) timekeeping_suspend_time = timespec_add(timekeeping_suspend_time, delta_delta); } - write_sequnlock_irqrestore(&timekeeper.lock, flags2); - write_sequnlock_irqrestore(&xtime_lock, flags1); + write_sequnlock_irqrestore(&timekeeper.lock, flags); clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); clocksource_suspend(); @@ -1006,7 +993,6 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift) /** * update_wall_time - Uses the current clocksource to increment the wall time * - * Called from the timer interrupt, must hold a write on xtime_lock. */ static void update_wall_time(void) { -- cgit v1.2.3 From 058892e632aa53be8255c2f0a42f9ace7bed66bb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 13 Nov 2011 23:19:48 +0000 Subject: time: Reorder so the hot data is together Keep all the interesting data in a single cache line. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 74bb5701e0b..06f40ae13b7 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -25,6 +25,8 @@ struct timekeeper { /* Current clocksource used for timekeeping. */ struct clocksource *clock; + /* NTP adjusted clock multiplier */ + u32 mult; /* The shift value of the current clocksource. */ int shift; @@ -45,8 +47,6 @@ struct timekeeper { /* Shift conversion between clock shifted nano seconds and * ntp shifted nano seconds. */ int ntp_error_shift; - /* NTP adjusted clock multiplier */ - u32 mult; /* The current time */ struct timespec xtime; -- cgit v1.2.3 From cc06268c6a87db156af2daed6e96a936b955cc82 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 13 Nov 2011 23:19:49 +0000 Subject: time: Move common updates to a function CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 06f40ae13b7..403c2a09283 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -172,17 +172,26 @@ static inline s64 timekeeping_get_ns_raw(void) return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); } +/* must hold write on timekeeper.lock */ +static void timekeeping_update(bool clearntp) +{ + if (clearntp) { + timekeeper.ntp_error = 0; + ntp_clear(); + } + update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, + timekeeper.clock, timekeeper.mult); +} + + void timekeeping_leap_insert(int leapsecond) { unsigned long flags; write_seqlock_irqsave(&timekeeper.lock, flags); - timekeeper.xtime.tv_sec += leapsecond; timekeeper.wall_to_monotonic.tv_sec -= leapsecond; - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, - timekeeper.clock, timekeeper.mult); - + timekeeping_update(false); write_sequnlock_irqrestore(&timekeeper.lock, flags); } @@ -386,12 +395,7 @@ int do_settimeofday(const struct timespec *tv) timespec_sub(timekeeper.wall_to_monotonic, ts_delta); timekeeper.xtime = *tv; - - timekeeper.ntp_error = 0; - ntp_clear(); - - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, - timekeeper.clock, timekeeper.mult); + timekeeping_update(true); write_sequnlock_irqrestore(&timekeeper.lock, flags); @@ -425,11 +429,7 @@ int timekeeping_inject_offset(struct timespec *ts) timekeeper.wall_to_monotonic = timespec_sub(timekeeper.wall_to_monotonic, *ts); - timekeeper.ntp_error = 0; - ntp_clear(); - - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, - timekeeper.clock, timekeeper.mult); + timekeeping_update(true); write_sequnlock_irqrestore(&timekeeper.lock, flags); @@ -668,10 +668,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) __timekeeping_inject_sleeptime(delta); - timekeeper.ntp_error = 0; - ntp_clear(); - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, - timekeeper.clock, timekeeper.mult); + timekeeping_update(true); write_sequnlock_irqrestore(&timekeeper.lock, flags); @@ -1083,9 +1080,7 @@ static void update_wall_time(void) second_overflow(); } - /* check to see if there is a new clocksource to use */ - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, - timekeeper.clock, timekeeper.mult); + timekeeping_update(false); out: write_sequnlock_irqrestore(&timekeeper.lock, flags); -- cgit v1.2.3 From 77b0d60c5adf39c74039e2142a1d3cd1e4d53799 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Fri, 4 Nov 2011 17:18:21 -0700 Subject: clockevents: Leave the broadcast device in shutdown mode when not needed Platforms with Always Running APIC Timer doesn't use the broadcast timer but the kernel is leaving the broadcast timer (HPET in this case) in oneshot mode. On these platforms, before the switch to oneshot mode, broadcast device is actually in shutdown mode. Code checks for empty tick_broadcast_mask and avoids going into the periodic mode. During switch to oneshot mode, add the same tick_broadcast_mask checks in the tick_broadcast_switch_to_oneshot() and avoid the broadcast device going into the oneshot mode. Signed-off-by: Suresh Siddha Cc: john stultz Cc: venki@google.com Link: http://lkml.kernel.org/r/1320452301.15071.16.camel@sbsiddha-desk.sc.intel.com Signed-off-by: Thomas Gleixner --- kernel/time/tick-broadcast.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index fd4a7b1625a..e883f57a3cd 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -575,11 +575,15 @@ void tick_broadcast_switch_to_oneshot(void) unsigned long flags; raw_spin_lock_irqsave(&tick_broadcast_lock, flags); + if (cpumask_empty(tick_get_broadcast_mask())) + goto end; tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; bc = tick_broadcast_device.evtdev; if (bc) tick_broadcast_setup_oneshot(bc); + +end: raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); } -- cgit v1.2.3 From 430ee8819553f66fe00e36f676a45886d76e7e8b Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Thu, 1 Dec 2011 17:00:22 +0100 Subject: nohz: Remove update_ts_time_stat from tick_nohz_start_idle There is no reason to call update_ts_time_stat from tick_nohz_start_idle anymore (after e0e37c20 sched: Eliminate the ts->idle_lastupdate field) when we updated idle_lastupdate unconditionally. We haven't set idle_active yet and do not provide last_update_time so the whole call end up being just 2 wasted branches. Signed-off-by: Michal Hocko Cc: Arjan van de Ven Link: http://lkml.kernel.org/r/1322755222-6951-1-git-send-email-mhocko@suse.cz Signed-off-by: Thomas Gleixner --- kernel/time/tick-sched.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 7656642e4b8..8cfffd9d9ce 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -182,11 +182,7 @@ static void tick_nohz_stop_idle(int cpu, ktime_t now) static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts) { - ktime_t now; - - now = ktime_get(); - - update_ts_time_stats(cpu, ts, now, NULL); + ktime_t now = ktime_get(); ts->idle_entrytime = now; ts->idle_active = 1; -- cgit v1.2.3 From 15f827be93928890bba965bc175caee50c4406d2 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 24 Jan 2012 18:59:43 +0100 Subject: nohz: Remove ts->Einidle checks before restarting the tick ts->inidle is set by tick_nohz_idle_enter() and unset by tick_nohz_idle_exit(). However these two calls are assumed to be always paired. This means that by the time we call tick_nohz_idle_exit(), ts->inidle is supposed to be always set to 1. Remove the checks for ts->inidle in tick_nohz_idle_exit(). This simplifies a bit the code and improves its debuggability (ie: ensure the call is paired with a tick_nohz_idle_enter() call). Signed-off-by: Frederic Weisbecker Reviewed-by: Yong Zhang Cc: Peter Zijlstra Cc: John Stultz Cc: Ingo Molnar Link: http://lkml.kernel.org/r/1327427984-23282-2-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- kernel/time/tick-sched.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 8cfffd9d9ce..3526038f283 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -558,20 +558,21 @@ void tick_nohz_idle_exit(void) local_irq_disable(); - if (ts->idle_active || (ts->inidle && ts->tick_stopped)) + WARN_ON_ONCE(!ts->inidle); + + ts->inidle = 0; + + if (ts->idle_active || ts->tick_stopped) now = ktime_get(); if (ts->idle_active) tick_nohz_stop_idle(cpu, now); - if (!ts->inidle || !ts->tick_stopped) { - ts->inidle = 0; + if (!ts->tick_stopped) { local_irq_enable(); return; } - ts->inidle = 0; - /* Update jiffies first */ select_nohz_load_balancer(0); tick_do_update_jiffies64(now); -- cgit v1.2.3 From 0a8a2e78b7eece7c65884fcff9f98dc0fce89ee4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 24 Jan 2012 18:59:44 +0100 Subject: timer: Fix bad idle check on irq entry idle_cpu() is called on irq entry to guess if we need to call tick_check_idle(). This way we can catch up with jiffies if the tick was stopped, stop accounting idle time during the interrupt and maintain the sched clock if it is unstable. But if we are going to exit the idle loop to schedule a new task (ie: if we have a task in the runqueue or a remotely enqueued ttwu to perform), the idle_cpu() check will return 0 such that we miss the call to tick_check_idle() for all interrupts happening before we schedule the new task. As a result these interrupts and the softirqs coming along may deal with stale jiffies values, bad sched clock values, and won't substract their time from the idle time accounting. Fix this with using is_idle_task() instead that strictly checks that we are running the idle task, without caring about the fact we are going to schedule a task soon. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: John Stultz Cc: Ingo Molnar Link: http://lkml.kernel.org/r/1327427984-23282-3-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- kernel/softirq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index 4eb3a0fa351..5ace266bc0e 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -297,7 +297,7 @@ void irq_enter(void) int cpu = smp_processor_id(); rcu_irq_enter(); - if (idle_cpu(cpu) && !in_interrupt()) { + if (is_idle_task(current) && !in_interrupt()) { /* * Prevent raise_softirq from needlessly waking up ksoftirqd * here, as softirq will be serviced on return from interrupt. -- cgit v1.2.3 From a078c6d0e6288fad6d83fb6d5edd91ddb7b6ab33 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Thu, 15 Mar 2012 12:36:14 -0400 Subject: ntp: Fix integer overflow when setting time 'long secs' is passed as divisor to div_s64, which accepts a 32bit divisor. On 64bit machines that value is trimmed back from 8 bytes back to 4, causing a divide by zero when the number is bigger than (1 << 32) - 1 and all 32 lower bits are 0. Use div64_long() instead. Signed-off-by: Sasha Levin Cc: johnstul@us.ibm.com Link: http://lkml.kernel.org/r/1331829374-31543-2-git-send-email-levinsasha928@gmail.com Cc: stable@vger.kernel.org Signed-off-by: Thomas Gleixner --- kernel/time/ntp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 17fb1b9807d..6e039b144da 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -289,7 +289,7 @@ static inline s64 ntp_update_offset_fll(s64 offset64, long secs) time_status |= STA_MODE; - return div_s64(offset64 << (NTP_SCALE_SHIFT - SHIFT_FLL), secs); + return div64_long(offset64 << (NTP_SCALE_SHIFT - SHIFT_FLL), secs); } static void ntp_update_offset(long offset) -- cgit v1.2.3