1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
// SPDX-License-Identifier: GPL-2.0
/*
* Intel Transactional Synchronization Extensions (TSX) control.
*
* Copyright (C) 2019-2021 Intel Corporation
*
* Author:
* Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
*/
#include <linux/cpufeature.h>
#include <asm/cmdline.h>
#include "cpu.h"
#undef pr_fmt
#define pr_fmt(fmt) "tsx: " fmt
enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;
void tsx_disable(void)
{
u64 tsx;
rdmsrl(MSR_IA32_TSX_CTRL, tsx);
/* Force all transactions to immediately abort */
tsx |= TSX_CTRL_RTM_DISABLE;
/*
* Ensure TSX support is not enumerated in CPUID.
* This is visible to userspace and will ensure they
* do not waste resources trying TSX transactions that
* will always abort.
*/
tsx |= TSX_CTRL_CPUID_CLEAR;
wrmsrl(MSR_IA32_TSX_CTRL, tsx);
}
void tsx_enable(void)
{
u64 tsx;
rdmsrl(MSR_IA32_TSX_CTRL, tsx);
/* Enable the RTM feature in the cpu */
tsx &= ~TSX_CTRL_RTM_DISABLE;
/*
* Ensure TSX support is enumerated in CPUID.
* This is visible to userspace and will ensure they
* can enumerate and use the TSX feature.
*/
tsx &= ~TSX_CTRL_CPUID_CLEAR;
wrmsrl(MSR_IA32_TSX_CTRL, tsx);
}
static bool tsx_ctrl_is_supported(void)
{
u64 ia32_cap = x86_read_arch_cap_msr();
/*
* TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this
* MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
*
* TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
* microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
* bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
* MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
* tsx= cmdline requests will do nothing on CPUs without
* MSR_IA32_TSX_CTRL support.
*/
return !!(ia32_cap & ARCH_CAP_TSX_CTRL_MSR);
}
static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
{
if (boot_cpu_has_bug(X86_BUG_TAA))
return TSX_CTRL_DISABLE;
return TSX_CTRL_ENABLE;
}
/*
* Disabling TSX is not a trivial business.
*
* First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
* which says that TSX is practically disabled (all transactions are
* aborted by default). When that bit is set, the kernel unconditionally
* disables TSX.
*
* In order to do that, however, it needs to dance a bit:
*
* 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
* the MSR is present only when *two* CPUID bits are set:
*
* - X86_FEATURE_RTM_ALWAYS_ABORT
* - X86_FEATURE_TSX_FORCE_ABORT
*
* 2. The second method is for CPUs which do not have the above-mentioned
* MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
* through that one. Those CPUs can also have the initially mentioned
* CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
* applies: TSX gets disabled unconditionally.
*
* When either of the two methods are present, the kernel disables TSX and
* clears the respective RTM and HLE feature flags.
*
* An additional twist in the whole thing presents late microcode loading
* which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
* bit to be set after the update.
*
* A subsequent hotplug operation on any logical CPU except the BSP will
* cause for the supported CPUID feature bits to get re-detected and, if
* RTM and HLE get cleared all of a sudden, but, userspace did consult
* them before the update, then funny explosions will happen. Long story
* short: the kernel doesn't modify CPUID feature bits after booting.
*
* That's why, this function's call in init_intel() doesn't clear the
* feature flags.
*/
void tsx_clear_cpuid(void)
{
u64 msr;
/*
* MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
* bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
*/
if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
rdmsrl(MSR_TSX_FORCE_ABORT, msr);
msr |= MSR_TFA_TSX_CPUID_CLEAR;
wrmsrl(MSR_TSX_FORCE_ABORT, msr);
} else if (tsx_ctrl_is_supported()) {
rdmsrl(MSR_IA32_TSX_CTRL, msr);
msr |= TSX_CTRL_CPUID_CLEAR;
wrmsrl(MSR_IA32_TSX_CTRL, msr);
}
}
void __init tsx_init(void)
{
char arg[5] = {};
int ret;
/*
* Hardware will always abort a TSX transaction when the CPUID bit
* RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
* CPUID.RTM and CPUID.HLE bits. Clear them here.
*/
if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
tsx_clear_cpuid();
setup_clear_cpu_cap(X86_FEATURE_RTM);
setup_clear_cpu_cap(X86_FEATURE_HLE);
return;
}
if (!tsx_ctrl_is_supported()) {
tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
return;
}
ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg));
if (ret >= 0) {
if (!strcmp(arg, "on")) {
tsx_ctrl_state = TSX_CTRL_ENABLE;
} else if (!strcmp(arg, "off")) {
tsx_ctrl_state = TSX_CTRL_DISABLE;
} else if (!strcmp(arg, "auto")) {
tsx_ctrl_state = x86_get_tsx_auto_mode();
} else {
tsx_ctrl_state = TSX_CTRL_DISABLE;
pr_err("invalid option, defaulting to off\n");
}
} else {
/* tsx= not provided */
if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO))
tsx_ctrl_state = x86_get_tsx_auto_mode();
else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF))
tsx_ctrl_state = TSX_CTRL_DISABLE;
else
tsx_ctrl_state = TSX_CTRL_ENABLE;
}
if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
tsx_disable();
/*
* tsx_disable() will change the state of the RTM and HLE CPUID
* bits. Clear them here since they are now expected to be not
* set.
*/
setup_clear_cpu_cap(X86_FEATURE_RTM);
setup_clear_cpu_cap(X86_FEATURE_HLE);
} else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {
/*
* HW defaults TSX to be enabled at bootup.
* We may still need the TSX enable support
* during init for special cases like
* kexec after TSX is disabled.
*/
tsx_enable();
/*
* tsx_enable() will change the state of the RTM and HLE CPUID
* bits. Force them here since they are now expected to be set.
*/
setup_force_cpu_cap(X86_FEATURE_RTM);
setup_force_cpu_cap(X86_FEATURE_HLE);
}
}
|
