diff options
author | Peter Zijlstra <peterz@infradead.org> | 2015-06-08 16:00:30 +0200 |
---|---|---|
committer | Ingo Molnar <mingo@kernel.org> | 2015-06-19 10:06:45 +0200 |
commit | 1cde2930e15473cb4dd7e5a07d83e605a969bd6e (patch) | |
tree | a00119b3140020378468877e4b7dd65431f9237a | |
parent | d84525a845cc2617d638349f8756a9fec9ac8113 (diff) |
sched/preempt: Add static_key() to preempt_notifiers
Avoid touching the curr->preempt_notifier cacheline when not needed.
Provides a small improvement on pipe-bench:
taskset 01 perf stat --repeat 10 -- perf bench sched pipe
before:
Performance counter stats for 'perf bench sched pipe' (10 runs):
12385.016204 task-clock (msec) # 1.001 CPUs utilized ( +- 0.34% )
2,000,023 context-switches # 0.161 M/sec ( +- 0.00% )
0 cpu-migrations # 0.000 K/sec
175 page-faults # 0.014 K/sec ( +- 0.26% )
41,376,162,250 cycles # 3.341 GHz ( +- 0.11% )
17,389,139,321 stalled-cycles-frontend # 42.03% frontend cycles idle ( +- 0.25% )
<not supported> stalled-cycles-backend
68,788,588,003 instructions # 1.66 insns per cycle
# 0.25 stalled cycles per insn ( +- 0.02% )
13,449,387,620 branches # 1085.940 M/sec ( +- 0.02% )
20,880,690 branch-misses # 0.16% of all branches ( +- 0.98% )
12.372646094 seconds time elapsed ( +- 0.34% )
after:
Performance counter stats for 'perf bench sched pipe' (10 runs):
12180.936528 task-clock (msec) # 1.001 CPUs utilized ( +- 0.33% )
2,000,077 context-switches # 0.164 M/sec ( +- 0.00% )
0 cpu-migrations # 0.000 K/sec
174 page-faults # 0.014 K/sec ( +- 0.27% )
40,691,545,577 cycles # 3.341 GHz ( +- 0.06% )
16,446,333,371 stalled-cycles-frontend # 40.42% frontend cycles idle ( +- 0.18% )
<not supported> stalled-cycles-backend
68,570,100,387 instructions # 1.69 insns per cycle
# 0.24 stalled cycles per insn ( +- 0.01% )
13,389,740,014 branches # 1099.237 M/sec ( +- 0.01% )
20,175,440 branch-misses # 0.15% of all branches ( +- 0.52% )
12.169253010 seconds time elapsed ( +- 0.33% )
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
-rw-r--r-- | kernel/sched/core.c | 28 |
1 files changed, 23 insertions, 5 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index bdb7aa67baef..1428c7cebe2f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2151,12 +2151,15 @@ void wake_up_new_task(struct task_struct *p) #ifdef CONFIG_PREEMPT_NOTIFIERS +static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE; + /** * preempt_notifier_register - tell me when current is being preempted & rescheduled * @notifier: notifier struct to register */ void preempt_notifier_register(struct preempt_notifier *notifier) { + static_key_slow_inc(&preempt_notifier_key); hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); } EXPORT_SYMBOL_GPL(preempt_notifier_register); @@ -2170,10 +2173,11 @@ EXPORT_SYMBOL_GPL(preempt_notifier_register); void preempt_notifier_unregister(struct preempt_notifier *notifier) { hlist_del(¬ifier->link); + static_key_slow_dec(&preempt_notifier_key); } EXPORT_SYMBOL_GPL(preempt_notifier_unregister); -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) +static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) { struct preempt_notifier *notifier; @@ -2181,9 +2185,15 @@ static void fire_sched_in_preempt_notifiers(struct task_struct *curr) notifier->ops->sched_in(notifier, raw_smp_processor_id()); } +static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ + if (static_key_false(&preempt_notifier_key)) + __fire_sched_in_preempt_notifiers(curr); +} + static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) +__fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) { struct preempt_notifier *notifier; @@ -2191,13 +2201,21 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr, notifier->ops->sched_out(notifier, next); } +static __always_inline void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ + if (static_key_false(&preempt_notifier_key)) + __fire_sched_out_preempt_notifiers(curr, next); +} + #else /* !CONFIG_PREEMPT_NOTIFIERS */ -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) +static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) { } -static void +static inline void fire_sched_out_preempt_notifiers(struct task_struct *curr, struct task_struct *next) { |