1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2019 Intel Corporation
*/
#include <linux/slab.h>
#include <linux/workqueue.h>
#include "i915_active.h"
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_object.h"
#include "i915_globals.h"
#include "i915_request.h"
#include "i915_scheduler.h"
#include "i915_vma.h"
static LIST_HEAD(globals);
static atomic_t active;
static atomic_t epoch;
static struct park_work {
struct delayed_work work;
struct rcu_head rcu;
unsigned long flags;
#define PENDING 0
int epoch;
} park;
static void i915_globals_shrink(void)
{
struct i915_global *global;
/*
* kmem_cache_shrink() discards empty slabs and reorders partially
* filled slabs to prioritise allocating from the mostly full slabs,
* with the aim of reducing fragmentation.
*/
list_for_each_entry(global, &globals, link)
global->shrink();
}
static void __i915_globals_grace(struct rcu_head *rcu)
{
/* Ratelimit parking as shrinking is quite slow */
schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
}
static void __i915_globals_queue_rcu(void)
{
park.epoch = atomic_inc_return(&epoch);
if (!atomic_read(&active)) {
init_rcu_head(&park.rcu);
call_rcu(&park.rcu, __i915_globals_grace);
}
}
static void __i915_globals_park(struct work_struct *work)
{
destroy_rcu_head(&park.rcu);
/* Confirm nothing woke up in the last grace period */
if (park.epoch != atomic_read(&epoch)) {
__i915_globals_queue_rcu();
return;
}
clear_bit(PENDING, &park.flags);
i915_globals_shrink();
}
void __init i915_global_register(struct i915_global *global)
{
GEM_BUG_ON(!global->shrink);
GEM_BUG_ON(!global->exit);
list_add_tail(&global->link, &globals);
}
static void __i915_globals_cleanup(void)
{
struct i915_global *global, *next;
list_for_each_entry_safe_reverse(global, next, &globals, link)
global->exit();
}
static __initconst int (* const initfn[])(void) = {
i915_global_active_init,
i915_global_context_init,
i915_global_gem_context_init,
i915_global_objects_init,
i915_global_request_init,
i915_global_scheduler_init,
i915_global_vma_init,
};
int __init i915_globals_init(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(initfn); i++) {
int err;
err = initfn[i]();
if (err) {
__i915_globals_cleanup();
return err;
}
}
INIT_DELAYED_WORK(&park.work, __i915_globals_park);
return 0;
}
void i915_globals_park(void)
{
/*
* Defer shrinking the global slab caches (and other work) until
* after a RCU grace period has completed with no activity. This
* is to try and reduce the latency impact on the consumers caused
* by us shrinking the caches the same time as they are trying to
* allocate, with the assumption being that if we idle long enough
* for an RCU grace period to elapse since the last use, it is likely
* to be longer until we need the caches again.
*/
if (!atomic_dec_and_test(&active))
return;
/* Queue cleanup after the next RCU grace period has freed slabs */
if (!test_and_set_bit(PENDING, &park.flags))
__i915_globals_queue_rcu();
}
void i915_globals_unpark(void)
{
atomic_inc(&epoch);
atomic_inc(&active);
}
static void __exit __i915_globals_flush(void)
{
atomic_inc(&active); /* skip shrinking */
rcu_barrier(); /* wait for the work to be queued */
flush_delayed_work(&park.work);
atomic_dec(&active);
}
void __exit i915_globals_exit(void)
{
GEM_BUG_ON(atomic_read(&active));
__i915_globals_flush();
__i915_globals_cleanup();
/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
rcu_barrier();
}
|
