1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_ENTRYCOMMON_H
#define __LINUX_ENTRYCOMMON_H
#include <linux/tracehook.h>
#include <linux/syscalls.h>
#include <linux/seccomp.h>
#include <linux/sched.h>
#include <asm/entry-common.h>
/*
* Define dummy _TIF work flags if not defined by the architecture or for
* disabled functionality.
*/
#ifndef _TIF_SYSCALL_EMU
# define _TIF_SYSCALL_EMU (0)
#endif
#ifndef _TIF_SYSCALL_TRACEPOINT
# define _TIF_SYSCALL_TRACEPOINT (0)
#endif
#ifndef _TIF_SECCOMP
# define _TIF_SECCOMP (0)
#endif
#ifndef _TIF_SYSCALL_AUDIT
# define _TIF_SYSCALL_AUDIT (0)
#endif
#ifndef _TIF_PATCH_PENDING
# define _TIF_PATCH_PENDING (0)
#endif
#ifndef _TIF_UPROBE
# define _TIF_UPROBE (0)
#endif
/*
* TIF flags handled in syscall_enter_from_usermode()
*/
#ifndef ARCH_SYSCALL_ENTER_WORK
# define ARCH_SYSCALL_ENTER_WORK (0)
#endif
#define SYSCALL_ENTER_WORK \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
_TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_EMU | \
ARCH_SYSCALL_ENTER_WORK)
/*
* TIF flags handled in syscall_exit_to_user_mode()
*/
#ifndef ARCH_SYSCALL_EXIT_WORK
# define ARCH_SYSCALL_EXIT_WORK (0)
#endif
#define SYSCALL_EXIT_WORK \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SYSCALL_TRACEPOINT | ARCH_SYSCALL_EXIT_WORK)
/*
* TIF flags handled in exit_to_user_mode_loop()
*/
#ifndef ARCH_EXIT_TO_USER_MODE_WORK
# define ARCH_EXIT_TO_USER_MODE_WORK (0)
#endif
#define EXIT_TO_USER_MODE_WORK \
(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
_TIF_NEED_RESCHED | _TIF_PATCH_PENDING | \
ARCH_EXIT_TO_USER_MODE_WORK)
/**
* arch_check_user_regs - Architecture specific sanity check for user mode regs
* @regs: Pointer to currents pt_regs
*
* Defaults to an empty implementation. Can be replaced by architecture
* specific code.
*
* Invoked from syscall_enter_from_user_mode() in the non-instrumentable
* section. Use __always_inline so the compiler cannot push it out of line
* and make it instrumentable.
*/
static __always_inline void arch_check_user_regs(struct pt_regs *regs);
#ifndef arch_check_user_regs
static __always_inline void arch_check_user_regs(struct pt_regs *regs) {}
#endif
/**
* arch_syscall_enter_tracehook - Wrapper around tracehook_report_syscall_entry()
* @regs: Pointer to currents pt_regs
*
* Returns: 0 on success or an error code to skip the syscall.
*
* Defaults to tracehook_report_syscall_entry(). Can be replaced by
* architecture specific code.
*
* Invoked from syscall_enter_from_user_mode()
*/
static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs);
#ifndef arch_syscall_enter_tracehook
static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs)
{
return tracehook_report_syscall_entry(regs);
}
#endif
/**
* syscall_enter_from_user_mode - Check and handle work before invoking
* a syscall
* @regs: Pointer to currents pt_regs
* @syscall: The syscall number
*
* Invoked from architecture specific syscall entry code with interrupts
* disabled. The calling code has to be non-instrumentable. When the
* function returns all state is correct and the subsequent functions can be
* instrumented.
*
* Returns: The original or a modified syscall number
*
* If the returned syscall number is -1 then the syscall should be
* skipped. In this case the caller may invoke syscall_set_error() or
* syscall_set_return_value() first. If neither of those are called and -1
* is returned, then the syscall will fail with ENOSYS.
*
* The following functionality is handled here:
*
* 1) Establish state (lockdep, RCU (context tracking), tracing)
* 2) TIF flag dependent invocations of arch_syscall_enter_tracehook(),
* __secure_computing(), trace_sys_enter()
* 3) Invocation of audit_syscall_entry()
*/
long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
/**
* local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable()
* @ti_work: Cached TIF flags gathered with interrupts disabled
*
* Defaults to local_irq_enable(). Can be supplied by architecture specific
* code.
*/
static inline void local_irq_enable_exit_to_user(unsigned long ti_work);
#ifndef local_irq_enable_exit_to_user
static inline void local_irq_enable_exit_to_user(unsigned long ti_work)
{
local_irq_enable();
}
#endif
/**
* local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable()
*
* Defaults to local_irq_disable(). Can be supplied by architecture specific
* code.
*/
static inline void local_irq_disable_exit_to_user(void);
#ifndef local_irq_disable_exit_to_user
static inline void local_irq_disable_exit_to_user(void)
{
local_irq_disable();
}
#endif
/**
* arch_exit_to_user_mode_work - Architecture specific TIF work for exit
* to user mode.
* @regs: Pointer to currents pt_regs
* @ti_work: Cached TIF flags gathered with interrupts disabled
*
* Invoked from exit_to_user_mode_loop() with interrupt enabled
*
* Defaults to NOOP. Can be supplied by architecture specific code.
*/
static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
unsigned long ti_work);
#ifndef arch_exit_to_user_mode_work
static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
unsigned long ti_work)
{
}
#endif
/**
* arch_exit_to_user_mode_prepare - Architecture specific preparation for
* exit to user mode.
* @regs: Pointer to currents pt_regs
* @ti_work: Cached TIF flags gathered with interrupts disabled
*
* Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last
* function before return. Defaults to NOOP.
*/
static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
unsigned long ti_work);
#ifndef arch_exit_to_user_mode_prepare
static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
unsigned long ti_work)
{
}
#endif
/**
* arch_exit_to_user_mode - Architecture specific final work before
* exit to user mode.
*
* Invoked from exit_to_user_mode() with interrupt disabled as the last
* function before return. Defaults to NOOP.
*
* This needs to be __always_inline because it is non-instrumentable code
* invoked after context tracking switched to user mode.
*
* An architecture implementation must not do anything complex, no locking
* etc. The main purpose is for speculation mitigations.
*/
static __always_inline void arch_exit_to_user_mode(void);
#ifndef arch_exit_to_user_mode
static __always_inline void arch_exit_to_user_mode(void) { }
#endif
/**
* arch_do_signal - Architecture specific signal delivery function
* @regs: Pointer to currents pt_regs
*
* Invoked from exit_to_user_mode_loop().
*/
void arch_do_signal(struct pt_regs *regs);
/**
* arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit()
* @regs: Pointer to currents pt_regs
* @step: Indicator for single step
*
* Defaults to tracehook_report_syscall_exit(). Can be replaced by
* architecture specific code.
*
* Invoked from syscall_exit_to_user_mode()
*/
static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step);
#ifndef arch_syscall_exit_tracehook
static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step)
{
tracehook_report_syscall_exit(regs, step);
}
#endif
/**
* syscall_exit_to_user_mode - Handle work before returning to user mode
* @regs: Pointer to currents pt_regs
*
* Invoked with interrupts enabled and fully valid regs. Returns with all
* work handled, interrupts disabled such that the caller can immediately
* switch to user mode. Called from architecture specific syscall and ret
* from fork code.
*
* The call order is:
* 1) One-time syscall exit work:
* - rseq syscall exit
* - audit
* - syscall tracing
* - tracehook (single stepping)
*
* 2) Preparatory work
* - Exit to user mode loop (common TIF handling). Invokes
* arch_exit_to_user_mode_work() for architecture specific TIF work
* - Architecture specific one time work arch_exit_to_user_mode_prepare()
* - Address limit and lockdep checks
*
* 3) Final transition (lockdep, tracing, context tracking, RCU). Invokes
* arch_exit_to_user_mode() to handle e.g. speculation mitigations
*/
void syscall_exit_to_user_mode(struct pt_regs *regs);
/**
* irqentry_enter_from_user_mode - Establish state before invoking the irq handler
* @regs: Pointer to currents pt_regs
*
* Invoked from architecture specific entry code with interrupts disabled.
* Can only be called when the interrupt entry came from user mode. The
* calling code must be non-instrumentable. When the function returns all
* state is correct and the subsequent functions can be instrumented.
*
* The function establishes state (lockdep, RCU (context tracking), tracing)
*/
void irqentry_enter_from_user_mode(struct pt_regs *regs);
/**
* irqentry_exit_to_user_mode - Interrupt exit work
* @regs: Pointer to current's pt_regs
*
* Invoked with interrupts disbled and fully valid regs. Returns with all
* work handled, interrupts disabled such that the caller can immediately
* switch to user mode. Called from architecture specific interrupt
* handling code.
*
* The call order is #2 and #3 as described in syscall_exit_to_user_mode().
* Interrupt exit is not invoking #1 which is the syscall specific one time
* work.
*/
void irqentry_exit_to_user_mode(struct pt_regs *regs);
#endif
|
