1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// SPDX-License-Identifier: GPL-2.0
#include <linux/types.h>
#include <linux/nodemask.h>
#include <linux/slab.h>
#include <linux/lockdep.h>
#include <linux/memory-tiers.h>
struct memory_tier {
/* hierarchy of memory tiers */
struct list_head list;
/* list of all memory types part of this tier */
struct list_head memory_types;
/*
* start value of abstract distance. memory tier maps
* an abstract distance range,
* adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
*/
int adistance_start;
};
struct memory_dev_type {
/* list of memory types that are part of same tier as this type */
struct list_head tier_sibiling;
/* abstract distance for this specific memory type */
int adistance;
/* Nodes of same abstract distance */
nodemask_t nodes;
struct memory_tier *memtier;
};
static DEFINE_MUTEX(memory_tier_lock);
static LIST_HEAD(memory_tiers);
static struct memory_dev_type *node_memory_types[MAX_NUMNODES];
/*
* For now we can have 4 faster memory tiers with smaller adistance
* than default DRAM tier.
*/
static struct memory_dev_type default_dram_type = {
.adistance = MEMTIER_ADISTANCE_DRAM,
.tier_sibiling = LIST_HEAD_INIT(default_dram_type.tier_sibiling),
};
static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype)
{
bool found_slot = false;
struct memory_tier *memtier, *new_memtier;
int adistance = memtype->adistance;
unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE;
lockdep_assert_held_once(&memory_tier_lock);
/*
* If the memtype is already part of a memory tier,
* just return that.
*/
if (memtype->memtier)
return memtype->memtier;
adistance = round_down(adistance, memtier_adistance_chunk_size);
list_for_each_entry(memtier, &memory_tiers, list) {
if (adistance == memtier->adistance_start) {
memtype->memtier = memtier;
list_add(&memtype->tier_sibiling, &memtier->memory_types);
return memtier;
} else if (adistance < memtier->adistance_start) {
found_slot = true;
break;
}
}
new_memtier = kmalloc(sizeof(struct memory_tier), GFP_KERNEL);
if (!new_memtier)
return ERR_PTR(-ENOMEM);
new_memtier->adistance_start = adistance;
INIT_LIST_HEAD(&new_memtier->list);
INIT_LIST_HEAD(&new_memtier->memory_types);
if (found_slot)
list_add_tail(&new_memtier->list, &memtier->list);
else
list_add_tail(&new_memtier->list, &memory_tiers);
memtype->memtier = new_memtier;
list_add(&memtype->tier_sibiling, &new_memtier->memory_types);
return new_memtier;
}
static struct memory_tier *set_node_memory_tier(int node)
{
struct memory_tier *memtier;
struct memory_dev_type *memtype;
lockdep_assert_held_once(&memory_tier_lock);
if (!node_state(node, N_MEMORY))
return ERR_PTR(-EINVAL);
if (!node_memory_types[node])
node_memory_types[node] = &default_dram_type;
memtype = node_memory_types[node];
node_set(node, memtype->nodes);
memtier = find_create_memory_tier(memtype);
return memtier;
}
static int __init memory_tier_init(void)
{
int node;
struct memory_tier *memtier;
mutex_lock(&memory_tier_lock);
/*
* Look at all the existing N_MEMORY nodes and add them to
* default memory tier or to a tier if we already have memory
* types assigned.
*/
for_each_node_state(node, N_MEMORY) {
memtier = set_node_memory_tier(node);
if (IS_ERR(memtier))
/*
* Continue with memtiers we are able to setup
*/
break;
}
mutex_unlock(&memory_tier_lock);
return 0;
}
subsys_initcall(memory_tier_init);
|
