diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/exit.c | 1 | ||||
| -rw-r--r-- | kernel/fork.c | 1 | ||||
| -rw-r--r-- | kernel/signal.c | 44 |
3 files changed, 44 insertions, 2 deletions
diff --git a/kernel/exit.c b/kernel/exit.c index 04029e35e69a..0596526ed9ea 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -162,6 +162,7 @@ static void __exit_signal(struct task_struct *tsk) flush_sigqueue(&sig->shared_pending); tty_kref_put(tty); } + exit_task_sigqueue_cache(tsk); } static void delayed_put_task_struct(struct rcu_head *rhp) diff --git a/kernel/fork.c b/kernel/fork.c index d3171e8e88e5..3c43a9f3e75c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1995,6 +1995,7 @@ static __latent_entropy struct task_struct *copy_process( spin_lock_init(&p->alloc_lock); init_sigpending(&p->pending); + p->sigqueue_cache = NULL; p->utime = p->stime = p->gtime = 0; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME diff --git a/kernel/signal.c b/kernel/signal.c index 568a2e2fc9ab..2d9463e05ae6 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -433,7 +433,16 @@ __sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags, rcu_read_unlock(); if (override_rlimit || likely(sigpending <= task_rlimit(t, RLIMIT_SIGPENDING))) { - q = kmem_cache_alloc(sigqueue_cachep, gfp_flags); + /* + * Preallocation does not hold sighand::siglock so it can't + * use the cache. The lockless caching requires that only + * one consumer and only one producer run at a time. + */ + q = READ_ONCE(t->sigqueue_cache); + if (!q || sigqueue_flags) + q = kmem_cache_alloc(sigqueue_cachep, gfp_flags); + else + WRITE_ONCE(t->sigqueue_cache, NULL); } else { print_dropped_signal(sig); } @@ -450,13 +459,44 @@ __sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags, return q; } +void exit_task_sigqueue_cache(struct task_struct *tsk) +{ + /* Race free because @tsk is mopped up */ + struct sigqueue *q = tsk->sigqueue_cache; + + if (q) { + tsk->sigqueue_cache = NULL; + /* + * Hand it back to the cache as the task might + * be self reaping which would leak the object. + */ + kmem_cache_free(sigqueue_cachep, q); + } +} + +static void sigqueue_cache_or_free(struct sigqueue *q) +{ + /* + * Cache one sigqueue per task. This pairs with the consumer side + * in __sigqueue_alloc() and needs READ/WRITE_ONCE() to prevent the + * compiler from store tearing and to tell KCSAN that the data race + * is intentional when run without holding current->sighand->siglock, + * which is fine as current obviously cannot run __sigqueue_free() + * concurrently. + */ + if (!READ_ONCE(current->sigqueue_cache)) + WRITE_ONCE(current->sigqueue_cache, q); + else + kmem_cache_free(sigqueue_cachep, q); +} + static void __sigqueue_free(struct sigqueue *q) { if (q->flags & SIGQUEUE_PREALLOC) return; if (atomic_dec_and_test(&q->user->sigpending)) free_uid(q->user); - kmem_cache_free(sigqueue_cachep, q); + sigqueue_cache_or_free(q); } void flush_sigqueue(struct sigpending *queue) |