1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2023 Ventana Micro Systems Inc.
*/
#define pr_fmt(fmt) "riscv-pv: " fmt
#include <linux/cpuhotplug.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/jump_label.h>
#include <linux/kconfig.h>
#include <linux/kernel.h>
#include <linux/percpu-defs.h>
#include <linux/printk.h>
#include <linux/static_call.h>
#include <linux/types.h>
#include <asm/barrier.h>
#include <asm/page.h>
#include <asm/paravirt.h>
#include <asm/sbi.h>
struct static_key paravirt_steal_enabled;
struct static_key paravirt_steal_rq_enabled;
static u64 native_steal_clock(int cpu)
{
return 0;
}
DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
static bool steal_acc = true;
static int __init parse_no_stealacc(char *arg)
{
steal_acc = false;
return 0;
}
early_param("no-steal-acc", parse_no_stealacc);
DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64);
static bool __init has_pv_steal_clock(void)
{
if (sbi_spec_version >= sbi_mk_version(2, 0) &&
sbi_probe_extension(SBI_EXT_STA) > 0) {
pr_info("SBI STA extension detected\n");
return true;
}
return false;
}
static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi,
unsigned long flags)
{
struct sbiret ret;
ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM,
lo, hi, flags, 0, 0, 0);
if (ret.error) {
if (lo == SBI_STA_SHMEM_DISABLE && hi == SBI_STA_SHMEM_DISABLE)
pr_warn("Failed to disable steal-time shmem");
else
pr_warn("Failed to set steal-time shmem");
return sbi_err_map_linux_errno(ret.error);
}
return 0;
}
static int pv_time_cpu_online(unsigned int cpu)
{
struct sbi_sta_struct *st = this_cpu_ptr(&steal_time);
phys_addr_t pa = __pa(st);
unsigned long lo = (unsigned long)pa;
unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0;
return sbi_sta_steal_time_set_shmem(lo, hi, 0);
}
static int pv_time_cpu_down_prepare(unsigned int cpu)
{
return sbi_sta_steal_time_set_shmem(SBI_STA_SHMEM_DISABLE,
SBI_STA_SHMEM_DISABLE, 0);
}
static u64 pv_time_steal_clock(int cpu)
{
struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu);
u32 sequence;
u64 steal;
/*
* Check the sequence field before and after reading the steal
* field. Repeat the read if it is different or odd.
*/
do {
sequence = READ_ONCE(st->sequence);
virt_rmb();
steal = READ_ONCE(st->steal);
virt_rmb();
} while ((le32_to_cpu(sequence) & 1) ||
sequence != READ_ONCE(st->sequence));
return le64_to_cpu(steal);
}
int __init pv_time_init(void)
{
int ret;
if (!has_pv_steal_clock())
return 0;
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
"riscv/pv_time:online",
pv_time_cpu_online,
pv_time_cpu_down_prepare);
if (ret < 0)
return ret;
static_call_update(pv_steal_clock, pv_time_steal_clock);
static_key_slow_inc(¶virt_steal_enabled);
if (steal_acc)
static_key_slow_inc(¶virt_steal_rq_enabled);
pr_info("Computing paravirt steal-time\n");
return 0;
}
|
