From 29e44f4535faa71a70827af3639b5e6762d8f02a Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 17 Aug 2020 11:07:28 +0100 Subject: watch_queue: Limit the number of watches a user can hold Impose a limit on the number of watches that a user can hold so that they can't use this mechanism to fill up all the available memory. This is done by putting a counter in user_struct that's incremented when a watch is allocated and decreased when it is released. If the number exceeds the RLIMIT_NOFILE limit, the watch is rejected with EAGAIN. This can be tested by the following means: (1) Create a watch queue and attach it to fd 5 in the program given - in this case, bash: keyctl watch_session /tmp/nlog /tmp/gclog 5 bash (2) In the shell, set the maximum number of files to, say, 99: ulimit -n 99 (3) Add 200 keyrings: for ((i=0; i<200; i++)); do keyctl newring a$i @s || break; done (4) Try to watch all of the keyrings: for ((i=0; i<200; i++)); do echo $i; keyctl watch_add 5 %:a$i || break; done This should fail when the number of watches belonging to the user hits 99. (5) Remove all the keyrings and all of those watches should go away: for ((i=0; i<200; i++)); do keyctl unlink %:a$i; done (6) Kill off the watch queue by exiting the shell spawned by watch_session. Fixes: c73be61cede5 ("pipe: Add general notification queue support") Reported-by: Linus Torvalds Signed-off-by: David Howells Reviewed-by: Jarkko Sakkinen Signed-off-by: Linus Torvalds --- include/linux/sched/user.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/sched') diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 917d88edb7b9..a8ec3b6093fc 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -36,6 +36,9 @@ struct user_struct { defined(CONFIG_NET) || defined(CONFIG_IO_URING) atomic_long_t locked_vm; #endif +#ifdef CONFIG_WATCH_QUEUE + atomic_t nr_watches; /* The number of watches this user currently has */ +#endif /* Miscellaneous per-user rate limit */ struct ratelimit_state ratelimit; -- cgit v1.3-18-gd494 From cfe7ddcbd72dc67ce5749cc6f451a2b0c6aec5b5 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:47 +0100 Subject: ARM, sched/topology: Remove SD_SHARE_POWERDOMAIN This flag was introduced in 2014 by commit: d77b3ed5c9f8 ("sched: Add a new SD_SHARE_POWERDOMAIN for sched_domain") but AFAIA it was never leveraged by the scheduler. The closest thing I can think of is EAS caring about frequency domains, and it does that by leveraging performance domains. Remove the flag. No change in functionality is expected. Suggested-by: Morten Rasmussen Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Reviewed-by: Dietmar Eggemann Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-2-valentin.schneider@arm.com --- arch/arm/kernel/topology.c | 2 +- include/linux/sched/topology.h | 13 ++++++------- kernel/sched/topology.c | 10 +++------- 3 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include/linux/sched') diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index b5adaf744630..353f3ee660e4 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -243,7 +243,7 @@ topology_populated: static inline int cpu_corepower_flags(void) { - return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN; + return SD_SHARE_PKG_RESOURCES; } static struct sched_domain_topology_level arm_topology[] = { diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 820511289857..6ec7d7c1d1e3 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -18,13 +18,12 @@ #define SD_WAKE_AFFINE 0x0010 /* Wake task to waking CPU */ #define SD_ASYM_CPUCAPACITY 0x0020 /* Domain members have different CPU capacities */ #define SD_SHARE_CPUCAPACITY 0x0040 /* Domain members share CPU capacity */ -#define SD_SHARE_POWERDOMAIN 0x0080 /* Domain members share power domain */ -#define SD_SHARE_PKG_RESOURCES 0x0100 /* Domain members share CPU pkg resources */ -#define SD_SERIALIZE 0x0200 /* Only a single load balancing instance */ -#define SD_ASYM_PACKING 0x0400 /* Place busy groups earlier in the domain */ -#define SD_PREFER_SIBLING 0x0800 /* Prefer to place tasks in a sibling domain */ -#define SD_OVERLAP 0x1000 /* sched_domains of this level overlap */ -#define SD_NUMA 0x2000 /* cross-node balancing */ +#define SD_SHARE_PKG_RESOURCES 0x0080 /* Domain members share CPU pkg resources */ +#define SD_SERIALIZE 0x0100 /* Only a single load balancing instance */ +#define SD_ASYM_PACKING 0x0200 /* Place busy groups earlier in the domain */ +#define SD_PREFER_SIBLING 0x0400 /* Prefer to place tasks in a sibling domain */ +#define SD_OVERLAP 0x0800 /* sched_domains of this level overlap */ +#define SD_NUMA 0x1000 /* cross-node balancing */ #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 007b0a6b0152..fe0396978fea 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -148,8 +148,7 @@ static int sd_degenerate(struct sched_domain *sd) SD_BALANCE_EXEC | SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY | - SD_SHARE_PKG_RESOURCES | - SD_SHARE_POWERDOMAIN)) { + SD_SHARE_PKG_RESOURCES)) { if (sd->groups != sd->groups->next) return 0; } @@ -180,8 +179,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) SD_ASYM_CPUCAPACITY | SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES | - SD_PREFER_SIBLING | - SD_SHARE_POWERDOMAIN); + SD_PREFER_SIBLING); if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } @@ -1292,7 +1290,6 @@ int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; * SD_SHARE_CPUCAPACITY - describes SMT topologies * SD_SHARE_PKG_RESOURCES - describes shared caches * SD_NUMA - describes NUMA topologies - * SD_SHARE_POWERDOMAIN - describes shared power domain * * Odd one out, which beside describing the topology has a quirk also * prescribes the desired behaviour that goes along with it: @@ -1303,8 +1300,7 @@ int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; (SD_SHARE_CPUCAPACITY | \ SD_SHARE_PKG_RESOURCES | \ SD_NUMA | \ - SD_ASYM_PACKING | \ - SD_SHARE_POWERDOMAIN) + SD_ASYM_PACKING) static struct sched_domain * sd_init(struct sched_domain_topology_level *tl, -- cgit v1.3-18-gd494 From d54a9658a75633b839af7a2c6c758807678b8064 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:49 +0100 Subject: sched/topology: Split out SD_* flags declaration to its own file To associate the SD flags with some metadata, we need some more structure in the way they are declared. Rather than shove that in a free-standing macro list, move the declaration in a separate file that can be re-imported with different SD_FLAG definitions. This is inspired by what is done with the syscall table (see uapi/asm/unistd.h and sys_call_table). The value assigned to a given SD flag now depends on the order it appears in sd_flags.h. No change in functionality. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Reviewed-by: Dietmar Eggemann Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-4-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 35 +++++++++++++++++++++++++++++++++++ include/linux/sched/topology.h | 26 +++++++++++++------------- 2 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 include/linux/sched/sd_flags.h (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h new file mode 100644 index 000000000000..373dc45c024e --- /dev/null +++ b/include/linux/sched/sd_flags.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sched-domains (multiprocessor balancing) flag declarations. + */ + +#ifndef SD_FLAG +# error "Incorrect import of SD flags definitions" +#endif + +/* Balance when about to become idle */ +SD_FLAG(SD_BALANCE_NEWIDLE) +/* Balance on exec */ +SD_FLAG(SD_BALANCE_EXEC) +/* Balance on fork, clone */ +SD_FLAG(SD_BALANCE_FORK) +/* Balance on wakeup */ +SD_FLAG(SD_BALANCE_WAKE) +/* Wake task to waking CPU */ +SD_FLAG(SD_WAKE_AFFINE) +/* Domain members have different CPU capacities */ +SD_FLAG(SD_ASYM_CPUCAPACITY) +/* Domain members share CPU capacity */ +SD_FLAG(SD_SHARE_CPUCAPACITY) +/* Domain members share CPU pkg resources */ +SD_FLAG(SD_SHARE_PKG_RESOURCES) +/* Only a single load balancing instance */ +SD_FLAG(SD_SERIALIZE) +/* Place busy groups earlier in the domain */ +SD_FLAG(SD_ASYM_PACKING) +/* Prefer to place tasks in a sibling domain */ +SD_FLAG(SD_PREFER_SIBLING) +/* sched_domains of this level overlap */ +SD_FLAG(SD_OVERLAP) +/* cross-node balancing */ +SD_FLAG(SD_NUMA) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 6ec7d7c1d1e3..3e41c0401b5f 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -11,19 +11,19 @@ */ #ifdef CONFIG_SMP -#define SD_BALANCE_NEWIDLE 0x0001 /* Balance when about to become idle */ -#define SD_BALANCE_EXEC 0x0002 /* Balance on exec */ -#define SD_BALANCE_FORK 0x0004 /* Balance on fork, clone */ -#define SD_BALANCE_WAKE 0x0008 /* Balance on wakeup */ -#define SD_WAKE_AFFINE 0x0010 /* Wake task to waking CPU */ -#define SD_ASYM_CPUCAPACITY 0x0020 /* Domain members have different CPU capacities */ -#define SD_SHARE_CPUCAPACITY 0x0040 /* Domain members share CPU capacity */ -#define SD_SHARE_PKG_RESOURCES 0x0080 /* Domain members share CPU pkg resources */ -#define SD_SERIALIZE 0x0100 /* Only a single load balancing instance */ -#define SD_ASYM_PACKING 0x0200 /* Place busy groups earlier in the domain */ -#define SD_PREFER_SIBLING 0x0400 /* Prefer to place tasks in a sibling domain */ -#define SD_OVERLAP 0x0800 /* sched_domains of this level overlap */ -#define SD_NUMA 0x1000 /* cross-node balancing */ +/* Generate SD flag indexes */ +#define SD_FLAG(name) __##name, +enum { + #include + __SD_FLAG_CNT, +}; +#undef SD_FLAG +/* Generate SD flag bits */ +#define SD_FLAG(name) name = 1 << __##name, +enum { + #include +}; +#undef SD_FLAG #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) -- cgit v1.3-18-gd494 From b6e862f386722e0de6c37f85f1cf438a0efa7f93 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:50 +0100 Subject: sched/topology: Define and assign sched_domain flag metadata There are some expectations regarding how sched domain flags should be laid out, but none of them are checked or asserted in sched_domain_debug_one(). After staring at said flags for a while, I've come to realize there's two repeating patterns: - Shared with children: those flags are set from the base CPU domain upwards. Any domain that has it set will have it set in its children. It hints at "some property holds true / some behaviour is enabled until this level". - Shared with parents: those flags are set from the topmost domain downwards. Any domain that has it set will have it set in its parents. It hints at "some property isn't visible / some behaviour is disabled until this level". There are two outliers that (currently) do not map to either of these: o SD_PREFER_SIBLING, which is cleared below levels with SD_ASYM_CPUCAPACITY. The change was introduced by commit: 9c63e84db29b ("sched/core: Disable SD_PREFER_SIBLING on asymmetric CPU capacity domains") as it could break misfit migration on some systems. In light of this, we might want to change it back to make it fit one of the two categories and fix the issue another way. o SD_ASYM_CPUCAPACITY, which gets set on a single level and isn't propagated up nor down. From a topology description point of view, it really wants to be SDF_SHARED_PARENT; this will be rectified in a later patch. Tweak the sched_domain flag declaration to assign each flag an expected layout, and include the rationale for each flag "meta type" assignment as a comment. Consolidate the flag metadata into an array; the index of a flag's metadata can easily be found with log2(flag), IOW __ffs(flag). Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Reviewed-by: Dietmar Eggemann Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-5-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 147 +++++++++++++++++++++++++++++++++-------- include/linux/sched/topology.h | 15 ++++- 2 files changed, 134 insertions(+), 28 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 373dc45c024e..c24a45b05fbb 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -7,29 +7,124 @@ # error "Incorrect import of SD flags definitions" #endif -/* Balance when about to become idle */ -SD_FLAG(SD_BALANCE_NEWIDLE) -/* Balance on exec */ -SD_FLAG(SD_BALANCE_EXEC) -/* Balance on fork, clone */ -SD_FLAG(SD_BALANCE_FORK) -/* Balance on wakeup */ -SD_FLAG(SD_BALANCE_WAKE) -/* Wake task to waking CPU */ -SD_FLAG(SD_WAKE_AFFINE) -/* Domain members have different CPU capacities */ -SD_FLAG(SD_ASYM_CPUCAPACITY) -/* Domain members share CPU capacity */ -SD_FLAG(SD_SHARE_CPUCAPACITY) -/* Domain members share CPU pkg resources */ -SD_FLAG(SD_SHARE_PKG_RESOURCES) -/* Only a single load balancing instance */ -SD_FLAG(SD_SERIALIZE) -/* Place busy groups earlier in the domain */ -SD_FLAG(SD_ASYM_PACKING) -/* Prefer to place tasks in a sibling domain */ -SD_FLAG(SD_PREFER_SIBLING) -/* sched_domains of this level overlap */ -SD_FLAG(SD_OVERLAP) -/* cross-node balancing */ -SD_FLAG(SD_NUMA) +/* + * Expected flag uses + * + * SHARED_CHILD: These flags are meant to be set from the base domain upwards. + * If a domain has this flag set, all of its children should have it set. This + * is usually because the flag describes some shared resource (all CPUs in that + * domain share the same resource), or because they are tied to a scheduling + * behaviour that we want to disable at some point in the hierarchy for + * scalability reasons. + * + * In those cases it doesn't make sense to have the flag set for a domain but + * not have it in (some of) its children: sched domains ALWAYS span their child + * domains, so operations done with parent domains will cover CPUs in the lower + * child domains. + * + * + * SHARED_PARENT: These flags are meant to be set from the highest domain + * downwards. If a domain has this flag set, all of its parents should have it + * set. This is usually for topology properties that start to appear above a + * certain level (e.g. domain starts spanning CPUs outside of the base CPU's + * socket). + */ +#define SDF_SHARED_CHILD 0x1 +#define SDF_SHARED_PARENT 0x2 + +/* + * Balance when about to become idle + * + * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level. + */ +SD_FLAG(SD_BALANCE_NEWIDLE, SDF_SHARED_CHILD) + +/* + * Balance on exec + * + * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level. + */ +SD_FLAG(SD_BALANCE_EXEC, SDF_SHARED_CHILD) + +/* + * Balance on fork, clone + * + * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level. + */ +SD_FLAG(SD_BALANCE_FORK, SDF_SHARED_CHILD) + +/* + * Balance on wakeup + * + * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level. + */ +SD_FLAG(SD_BALANCE_WAKE, SDF_SHARED_CHILD) + +/* + * Consider waking task on waking CPU. + * + * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level. + */ +SD_FLAG(SD_WAKE_AFFINE, SDF_SHARED_CHILD) + +/* + * Domain members have different CPU capacities + */ +SD_FLAG(SD_ASYM_CPUCAPACITY, 0) + +/* + * Domain members share CPU capacity (i.e. SMT) + * + * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share + * CPU capacity. + */ +SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD) + +/* + * Domain members share CPU package resources (i.e. caches) + * + * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share + * the same cache(s). + */ +SD_FLAG(SD_SHARE_PKG_RESOURCES, SDF_SHARED_CHILD) + +/* + * Only a single load balancing instance + * + * SHARED_PARENT: Set for all NUMA levels above NODE. Could be set from a + * different level upwards, but it doesn't change that if a domain has this flag + * set, then all of its parents need to have it too (otherwise the serialization + * doesn't make sense). + */ +SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT) + +/* + * Place busy tasks earlier in the domain + * + * SHARED_CHILD: Usually set on the SMT level. Technically could be set further + * up, but currently assumed to be set from the base domain upwards (see + * update_top_cache_domain()). + */ +SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD) + +/* + * Prefer to place tasks in a sibling domain + * + * Set up until domains start spanning NUMA nodes. Close to being a SHARED_CHILD + * flag, but cleared below domains with SD_ASYM_CPUCAPACITY. + */ +SD_FLAG(SD_PREFER_SIBLING, 0) + +/* + * sched_groups of this level overlap + * + * SHARED_PARENT: Set for all NUMA levels above NODE. + */ +SD_FLAG(SD_OVERLAP, SDF_SHARED_PARENT) + +/* + * Cross-node balancing + * + * SHARED_PARENT: Set for all NUMA levels above NODE. + */ +SD_FLAG(SD_NUMA, SDF_SHARED_PARENT) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 3e41c0401b5f..32f602ff37a0 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -12,19 +12,30 @@ #ifdef CONFIG_SMP /* Generate SD flag indexes */ -#define SD_FLAG(name) __##name, +#define SD_FLAG(name, mflags) __##name, enum { #include __SD_FLAG_CNT, }; #undef SD_FLAG /* Generate SD flag bits */ -#define SD_FLAG(name) name = 1 << __##name, +#define SD_FLAG(name, mflags) name = 1 << __##name, enum { #include }; #undef SD_FLAG +#ifdef CONFIG_SCHED_DEBUG +#define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name }, +static const struct { + unsigned int meta_flags; + char *name; +} sd_flag_debug[] = { +#include +}; +#undef SD_FLAG +#endif + #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) { -- cgit v1.3-18-gd494 From 4ee4ea443a5dc3fc4d8ae338199676eae9d8ef02 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:53 +0100 Subject: sched/topology: Introduce SD metaflag for flags needing > 1 groups In preparation of cleaning up the sd_degenerate*() functions, mark flags used in sd_degenerate() with the new SDF_NEEDS_GROUPS flag. With this, build a compile-time mask of those SD flags. Note that sd_parent_degenerate() uses an extra flag in its mask, SD_PREFER_SIBLING, which remains singled out for now. Suggested-by: Peter Zijlstra Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Reviewed-by: Dietmar Eggemann Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-8-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 39 ++++++++++++++++++++++++++++----------- include/linux/sched/topology.h | 7 +++++++ 2 files changed, 35 insertions(+), 11 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index c24a45b05fbb..ee5cbfc7189f 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -8,7 +8,7 @@ #endif /* - * Expected flag uses + * Hierarchical metaflags * * SHARED_CHILD: These flags are meant to be set from the base domain upwards. * If a domain has this flag set, all of its children should have it set. This @@ -29,29 +29,42 @@ * certain level (e.g. domain starts spanning CPUs outside of the base CPU's * socket). */ -#define SDF_SHARED_CHILD 0x1 -#define SDF_SHARED_PARENT 0x2 +#define SDF_SHARED_CHILD 0x1 +#define SDF_SHARED_PARENT 0x2 + +/* + * Behavioural metaflags + * + * NEEDS_GROUPS: These flags are only relevant if the domain they are set on has + * more than one group. This is usually for balancing flags (load balancing + * involves equalizing a metric between groups), or for flags describing some + * shared resource (which would be shared between groups). + */ +#define SDF_NEEDS_GROUPS 0x4 /* * Balance when about to become idle * * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level. + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_BALANCE_NEWIDLE, SDF_SHARED_CHILD) +SD_FLAG(SD_BALANCE_NEWIDLE, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Balance on exec * * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level. + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_BALANCE_EXEC, SDF_SHARED_CHILD) +SD_FLAG(SD_BALANCE_EXEC, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Balance on fork, clone * * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level. + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_BALANCE_FORK, SDF_SHARED_CHILD) +SD_FLAG(SD_BALANCE_FORK, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Balance on wakeup @@ -69,24 +82,28 @@ SD_FLAG(SD_WAKE_AFFINE, SDF_SHARED_CHILD) /* * Domain members have different CPU capacities + * + * NEEDS_GROUPS: Per-CPU capacity is asymmetric between groups. */ -SD_FLAG(SD_ASYM_CPUCAPACITY, 0) +SD_FLAG(SD_ASYM_CPUCAPACITY, SDF_NEEDS_GROUPS) /* * Domain members share CPU capacity (i.e. SMT) * * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share - * CPU capacity. + * CPU capacity. + * NEEDS_GROUPS: Capacity is shared between groups. */ -SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD) +SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Domain members share CPU package resources (i.e. caches) * * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share - * the same cache(s). + * the same cache(s). + * NEEDS_GROUPS: Caches are shared between groups. */ -SD_FLAG(SD_SHARE_PKG_RESOURCES, SDF_SHARED_CHILD) +SD_FLAG(SD_SHARE_PKG_RESOURCES, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Only a single load balancing instance diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 32f602ff37a0..2d59ca77103e 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -25,6 +25,13 @@ enum { }; #undef SD_FLAG +/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */ +#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) | +static const unsigned int SD_DEGENERATE_GROUPS_MASK = +#include +0; +#undef SD_FLAG + #ifdef CONFIG_SCHED_DEBUG #define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name }, static const struct { -- cgit v1.3-18-gd494 From c200191d4c2c1aa2ffb62a984b756ac1f02dc55c Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:56 +0100 Subject: sched/topology: Propagate SD_ASYM_CPUCAPACITY upwards We currently set this flag *only* on domains whose topology level exactly match the level where we detect asymmetry (as returned by asym_cpu_capacity_level()). This is rather problematic. Say there are two clusters in the system, one with a lone big CPU and the other with a mix of big and LITTLE CPUs (as is allowed by DynamIQ): DIE [ ] MC [ ][ ] 0 1 2 3 4 L L B B B asym_cpu_capacity_level() will figure out that the MC level is the one where all CPUs can see a CPU of max capacity, and we will thus set SD_ASYM_CPUCAPACITY at MC level for all CPUs. That lone big CPU will degenerate its MC domain, since it would be alone in there, and will end up with just a DIE domain. Since the flag was only set at MC, this CPU ends up not seeing any SD with the flag set, which is broken. Rather than clearing dflags at every topology level, clear it before entering the topology level loop. This will properly propagate upwards flags that are set starting from a certain level. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Reviewed-by: Quentin Perret Reviewed-by: Dietmar Eggemann Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-11-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 4 +++- kernel/sched/topology.c | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index ee5cbfc7189f..40ad0d5c32e3 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -83,9 +83,11 @@ SD_FLAG(SD_WAKE_AFFINE, SDF_SHARED_CHILD) /* * Domain members have different CPU capacities * + * SHARED_PARENT: Set from the topmost domain down to the first domain where + * asymmetry is detected. * NEEDS_GROUPS: Per-CPU capacity is asymmetric between groups. */ -SD_FLAG(SD_ASYM_CPUCAPACITY, SDF_NEEDS_GROUPS) +SD_FLAG(SD_ASYM_CPUCAPACITY, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) /* * Domain members share CPU capacity (i.e. SMT) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index c662c53736e2..f36ed96f3197 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1988,11 +1988,10 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att /* Set up domains for CPUs specified by the cpu_map: */ for_each_cpu(i, cpu_map) { struct sched_domain_topology_level *tl; + int dflags = 0; sd = NULL; for_each_sd_topology(tl) { - int dflags = 0; - if (tl == tl_asym) { dflags |= SD_ASYM_CPUCAPACITY; has_asym = true; -- cgit v1.3-18-gd494 From 3a6712c7685352a5d71eaf459a4fddfc3589f018 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:57 +0100 Subject: sched/topology: Mark SD_PREFER_SIBLING as SDF_NEEDS_GROUPS SD_PREFER_SIBLING is currently considered in sd_parent_degenerate() but not in sd_degenerate(). It too hinges on load balancing, and thus won't have any effect when set on a domain with a single group. Add it to SD_DEGENERATE_GROUPS_MASK. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-12-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 4 +++- kernel/sched/topology.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 40ad0d5c32e3..d28fe67c3098 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -131,8 +131,10 @@ SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD) * * Set up until domains start spanning NUMA nodes. Close to being a SHARED_CHILD * flag, but cleared below domains with SD_ASYM_CPUCAPACITY. + * + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_PREFER_SIBLING, 0) +SD_FLAG(SD_PREFER_SIBLING, SDF_NEEDS_GROUPS) /* * sched_groups of this level overlap diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index f36ed96f3197..c674aaab312c 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -184,7 +184,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) /* Flags needing groups don't count if only 1 group in parent */ if (parent->groups == parent->groups->next) - pflags &= ~(SD_DEGENERATE_GROUPS_MASK | SD_PREFER_SIBLING); + pflags &= ~SD_DEGENERATE_GROUPS_MASK; if (~cflags & pflags) return 0; -- cgit v1.3-18-gd494 From 94b858fea1f2246a2fb7f7af21840fd14ced028f Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:58 +0100 Subject: sched/topology: Mark SD_BALANCE_WAKE as SDF_NEEDS_GROUPS Even if no mainline topology uses this flag, it is a load balancing flag just like SD_BALANCE_FORK and requires 2+ groups to have any effect. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-13-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index d28fe67c3098..729510a291b2 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -70,8 +70,9 @@ SD_FLAG(SD_BALANCE_FORK, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) * Balance on wakeup * * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level. + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_BALANCE_WAKE, SDF_SHARED_CHILD) +SD_FLAG(SD_BALANCE_WAKE, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Consider waking task on waking CPU. -- cgit v1.3-18-gd494 From bdb7c802cc0a7e21f5223dc3ce41b7ac220c576e Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:29:59 +0100 Subject: sched/topology: Mark SD_SERIALIZE as SDF_NEEDS_GROUPS There would be no point in preserving a sched_domain with a single group just because it has this flag set. Add it to SD_DEGENERATE_GROUPS_MASK. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-14-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 729510a291b2..b7f4d80e338e 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -112,11 +112,12 @@ SD_FLAG(SD_SHARE_PKG_RESOURCES, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) * Only a single load balancing instance * * SHARED_PARENT: Set for all NUMA levels above NODE. Could be set from a - * different level upwards, but it doesn't change that if a domain has this flag - * set, then all of its parents need to have it too (otherwise the serialization - * doesn't make sense). + * different level upwards, but it doesn't change that if a + * domain has this flag set, then all of its parents need to have + * it too (otherwise the serialization doesn't make sense). + * NEEDS_GROUPS: No point in preserving domain if it has a single group. */ -SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT) +SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) /* * Place busy tasks earlier in the domain -- cgit v1.3-18-gd494 From 33199b0143daf4778d6301f966cb914d75f122eb Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:30:00 +0100 Subject: sched/topology: Mark SD_ASYM_PACKING as SDF_NEEDS_GROUPS Being a load-balancing flag, it requires 2+ groups to have any effect. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-15-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index b7f4d80e338e..2998ece2c18d 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -123,10 +123,11 @@ SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) * Place busy tasks earlier in the domain * * SHARED_CHILD: Usually set on the SMT level. Technically could be set further - * up, but currently assumed to be set from the base domain upwards (see - * update_top_cache_domain()). + * up, but currently assumed to be set from the base domain + * upwards (see update_top_cache_domain()). + * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD) +SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) /* * Prefer to place tasks in a sibling domain -- cgit v1.3-18-gd494 From 3551e954f5d95faf3dbc340d422da7624658c230 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:30:01 +0100 Subject: sched/topology: Mark SD_OVERLAP as SDF_NEEDS_GROUPS A sched_domain can only have overlapping sched_groups if it has more than one group. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-16-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 2998ece2c18d..29af5f032861 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -143,8 +143,9 @@ SD_FLAG(SD_PREFER_SIBLING, SDF_NEEDS_GROUPS) * sched_groups of this level overlap * * SHARED_PARENT: Set for all NUMA levels above NODE. + * NEEDS_GROUPS: Overlaps can only exist with more than one group. */ -SD_FLAG(SD_OVERLAP, SDF_SHARED_PARENT) +SD_FLAG(SD_OVERLAP, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) /* * Cross-node balancing -- cgit v1.3-18-gd494 From 5f4a1c4ea44728aa80be21dbf3a0469b5ca81d88 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Mon, 17 Aug 2020 12:30:02 +0100 Subject: sched/topology: Mark SD_NUMA as SDF_NEEDS_GROUPS There would be no point in preserving a sched_domain with a single group just because it has this flag set. Add it to SD_DEGENERATE_GROUPS_MASK. Signed-off-by: Valentin Schneider Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20200817113003.20802-17-valentin.schneider@arm.com --- include/linux/sched/sd_flags.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 29af5f032861..34b21e971d77 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -151,5 +151,6 @@ SD_FLAG(SD_OVERLAP, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) * Cross-node balancing * * SHARED_PARENT: Set for all NUMA levels above NODE. + * NEEDS_GROUPS: No point in preserving domain if it has a single group. */ -SD_FLAG(SD_NUMA, SDF_SHARED_PARENT) +SD_FLAG(SD_NUMA, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) -- cgit v1.3-18-gd494 From cad6967ac10843a70842cd39c7b53412901dd21f Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Wed, 19 Aug 2020 12:46:45 +0200 Subject: fork: introduce kernel_clone() The old _do_fork() helper doesn't follow naming conventions of in-kernel helpers for syscalls. The process creation cleanup in [1] didn't change the name to something more reasonable mainly because _do_fork() was used in quite a few places. So sending this as a separate series seemed the better strategy. This commit does two things: 1. renames _do_fork() to kernel_clone() but keeps _do_fork() as a simple static inline wrapper around kernel_clone(). 2. Changes the return type from long to pid_t. This aligns kernel_thread() and kernel_clone(). Also, the return value from kernel_clone that is surfaced in fork(), vfork(), clone(), and clone3() is taken from pid_vrn() which returns a pid_t too. Follow-up patches will switch each caller of _do_fork() and each place where it is referenced over to kernel_clone(). After all these changes are done, we can remove _do_fork() completely and will only be left with kernel_clone(). [1]: 9ba27414f2ec ("Merge tag 'fork-v5.9' of git://git.kernel.org/pub/scm/linux/kernel/git/brauner/linux") Signed-off-by: Christian Brauner Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Matthew Wilcox (Oracle) Cc: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20200819104655.436656-2-christian.brauner@ubuntu.com --- include/linux/sched/task.h | 6 +++++- kernel/fork.c | 16 ++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index a98965007eef..d4428039c3c1 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -83,7 +83,11 @@ extern void do_group_exit(int); extern void exit_files(struct task_struct *); extern void exit_itimers(struct signal_struct *); -extern long _do_fork(struct kernel_clone_args *kargs); +extern pid_t kernel_clone(struct kernel_clone_args *kargs); +static inline long _do_fork(struct kernel_clone_args *kargs) +{ + return kernel_clone(kargs); +} struct task_struct *fork_idle(int); struct mm_struct *copy_init_mm(void); extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); diff --git a/kernel/fork.c b/kernel/fork.c index 4d32190861bd..d822c7a4b9c4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2384,14 +2384,14 @@ struct mm_struct *copy_init_mm(void) * * args->exit_signal is expected to be checked for sanity by the caller. */ -long _do_fork(struct kernel_clone_args *args) +pid_t kernel_clone(struct kernel_clone_args *args) { u64 clone_flags = args->flags; struct completion vfork; struct pid *pid; struct task_struct *p; int trace = 0; - long nr; + pid_t nr; /* * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument @@ -2477,7 +2477,7 @@ pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) .stack_size = (unsigned long)arg, }; - return _do_fork(&args); + return kernel_clone(&args); } #ifdef __ARCH_WANT_SYS_FORK @@ -2488,7 +2488,7 @@ SYSCALL_DEFINE0(fork) .exit_signal = SIGCHLD, }; - return _do_fork(&args); + return kernel_clone(&args); #else /* can not support in nommu mode */ return -EINVAL; @@ -2504,7 +2504,7 @@ SYSCALL_DEFINE0(vfork) .exit_signal = SIGCHLD, }; - return _do_fork(&args); + return kernel_clone(&args); } #endif @@ -2542,7 +2542,7 @@ SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, .tls = tls, }; - return _do_fork(&args); + return kernel_clone(&args); } #endif @@ -2700,7 +2700,7 @@ SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size) if (!clone3_args_valid(&kargs)) return -EINVAL; - return _do_fork(&kargs); + return kernel_clone(&kargs); } #endif @@ -2863,7 +2863,7 @@ int unshare_fd(unsigned long unshare_flags, unsigned int max_fds, /* * unshare allows a process to 'unshare' part of the process * context which was originally shared using clone. copy_* - * functions used by _do_fork() cannot be used here directly + * functions used by kernel_clone() cannot be used here directly * because they modify an inactive task_struct that is being * constructed. Here we are modifying the current, active, * task_struct. -- cgit v1.3-18-gd494 From 06fe45634942dc96c316bbb789049a4b0b692542 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Wed, 19 Aug 2020 12:46:55 +0200 Subject: sched: remove _do_fork() Now that all callers of _do_fork() have been switched to kernel_clone() remove the _do_fork() helper. Signed-off-by: Christian Brauner Link: https://lore.kernel.org/r/20200819104655.436656-12-christian.brauner@ubuntu.com --- include/linux/sched/task.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index d4428039c3c1..85fb2f34c59b 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -84,10 +84,6 @@ extern void exit_files(struct task_struct *); extern void exit_itimers(struct signal_struct *); extern pid_t kernel_clone(struct kernel_clone_args *kargs); -static inline long _do_fork(struct kernel_clone_args *kargs) -{ - return kernel_clone(kargs); -} struct task_struct *fork_idle(int); struct mm_struct *copy_init_mm(void); extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); -- cgit v1.3-18-gd494 From 8fca9494d4b4d6b57b1398cd473feb308df656db Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Tue, 25 Aug 2020 14:32:15 +0100 Subject: sched/topology: Move sd_flag_debug out of linux/sched/topology.h Defining an array in a header imported all over the place clearly is a daft idea, that still didn't stop me from doing it. Leave a declaration of sd_flag_debug in topology.h and move its definition to sched/debug.c. Fixes: b6e862f38672 ("sched/topology: Define and assign sched_domain flag metadata") Reported-by: Andy Shevchenko Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200825133216.9163-1-valentin.schneider@arm.com --- include/linux/sched/topology.h | 9 ++++----- kernel/sched/debug.c | 6 ++++++ 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 2d59ca77103e..b9b0dab4d067 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -33,14 +33,13 @@ static const unsigned int SD_DEGENERATE_GROUPS_MASK = #undef SD_FLAG #ifdef CONFIG_SCHED_DEBUG -#define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name }, -static const struct { + +struct sd_flag_debug { unsigned int meta_flags; char *name; -} sd_flag_debug[] = { -#include }; -#undef SD_FLAG +extern const struct sd_flag_debug sd_flag_debug[]; + #endif #ifdef CONFIG_SCHED_SMT diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 0655524700d2..0d7896d2a0b2 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -245,6 +245,12 @@ set_table_entry(struct ctl_table *entry, entry->proc_handler = proc_handler; } +#define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name }, +const struct sd_flag_debug sd_flag_debug[] = { +#include +}; +#undef SD_FLAG + static int sd_ctl_doflags(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { -- cgit v1.3-18-gd494 From 4fc472f1214ef75e5450f207e23ff13af6eecad4 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Tue, 25 Aug 2020 14:32:16 +0100 Subject: sched/topology: Move SD_DEGENERATE_GROUPS_MASK out of linux/sched/topology.h SD_DEGENERATE_GROUPS_MASK is only useful for sched/topology.c, but still gets defined for anyone who imports topology.h, leading to a flurry of unused variable warnings. Move it out of the header and place it next to the SD degeneration functions in sched/topology.c. Fixes: 4ee4ea443a5d ("sched/topology: Introduce SD metaflag for flags needing > 1 groups") Reported-by: Andy Shevchenko Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200825133216.9163-2-valentin.schneider@arm.com --- include/linux/sched/topology.h | 7 ------- kernel/sched/topology.c | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index b9b0dab4d067..9ef7bf686a9f 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -25,13 +25,6 @@ enum { }; #undef SD_FLAG -/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */ -#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) | -static const unsigned int SD_DEGENERATE_GROUPS_MASK = -#include -0; -#undef SD_FLAG - #ifdef CONFIG_SCHED_DEBUG struct sd_flag_debug { diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index c674aaab312c..aa1676abc544 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -154,6 +154,13 @@ static inline bool sched_debug(void) } #endif /* CONFIG_SCHED_DEBUG */ +/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */ +#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) | +static const unsigned int SD_DEGENERATE_GROUPS_MASK = +#include +0; +#undef SD_FLAG + static int sd_degenerate(struct sched_domain *sd) { if (cpumask_weight(sched_domain_span(sd)) == 1) -- cgit v1.3-18-gd494 From 2a36ab717e8fe678d98f81c14a0b124712719840 Mon Sep 17 00:00:00 2001 From: Peter Oskolkov Date: Wed, 23 Sep 2020 16:36:16 -0700 Subject: rseq/membarrier: Add MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ This patchset is based on Google-internal RSEQ work done by Paul Turner and Andrew Hunter. When working with per-CPU RSEQ-based memory allocations, it is sometimes important to make sure that a global memory location is no longer accessed from RSEQ critical sections. For example, there can be two per-CPU lists, one is "active" and accessed per-CPU, while another one is inactive and worked on asynchronously "off CPU" (e.g. garbage collection is performed). Then at some point the two lists are swapped, and a fast RCU-like mechanism is required to make sure that the previously active list is no longer accessed. This patch introduces such a mechanism: in short, membarrier() syscall issues an IPI to a CPU, restarting a potentially active RSEQ critical section on the CPU. Signed-off-by: Peter Oskolkov Signed-off-by: Peter Zijlstra (Intel) Acked-by: Mathieu Desnoyers Link: https://lkml.kernel.org/r/20200923233618.2572849-1-posk@google.com --- include/linux/sched/mm.h | 3 + include/linux/syscalls.h | 2 +- include/uapi/linux/membarrier.h | 26 ++++++++ kernel/sched/membarrier.c | 136 +++++++++++++++++++++++++++++++--------- 4 files changed, 136 insertions(+), 31 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index f889e332912f..15bfb06f2884 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -348,10 +348,13 @@ enum { MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4), MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5), + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = (1U << 6), + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = (1U << 7), }; enum { MEMBARRIER_FLAG_SYNC_CORE = (1U << 0), + MEMBARRIER_FLAG_RSEQ = (1U << 1), }; #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 75ac7f8ae93c..06db09875aa4 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -974,7 +974,7 @@ asmlinkage long sys_execveat(int dfd, const char __user *filename, const char __user *const __user *argv, const char __user *const __user *envp, int flags); asmlinkage long sys_userfaultfd(int flags); -asmlinkage long sys_membarrier(int cmd, int flags); +asmlinkage long sys_membarrier(int cmd, unsigned int flags, int cpu_id); asmlinkage long sys_mlock2(unsigned long start, size_t len, int flags); asmlinkage long sys_copy_file_range(int fd_in, loff_t __user *off_in, int fd_out, loff_t __user *off_out, diff --git a/include/uapi/linux/membarrier.h b/include/uapi/linux/membarrier.h index 5891d7614c8c..737605897f36 100644 --- a/include/uapi/linux/membarrier.h +++ b/include/uapi/linux/membarrier.h @@ -114,6 +114,26 @@ * If this command is not implemented by an * architecture, -EINVAL is returned. * Returns 0 on success. + * @MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: + * Ensure the caller thread, upon return from + * system call, that all its running thread + * siblings have any currently running rseq + * critical sections restarted if @flags + * parameter is 0; if @flags parameter is + * MEMBARRIER_CMD_FLAG_CPU, + * then this operation is performed only + * on CPU indicated by @cpu_id. If this command is + * not implemented by an architecture, -EINVAL + * is returned. A process needs to register its + * intent to use the private expedited rseq + * command prior to using it, otherwise + * this command returns -EPERM. + * @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ: + * Register the process intent to use + * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ. + * If this command is not implemented by an + * architecture, -EINVAL is returned. + * Returns 0 on success. * @MEMBARRIER_CMD_SHARED: * Alias to MEMBARRIER_CMD_GLOBAL. Provided for * header backward compatibility. @@ -131,9 +151,15 @@ enum membarrier_cmd { MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4), MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 5), MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 6), + MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ = (1 << 7), + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ = (1 << 8), /* Alias for header backward compatibility. */ MEMBARRIER_CMD_SHARED = MEMBARRIER_CMD_GLOBAL, }; +enum membarrier_cmd_flag { + MEMBARRIER_CMD_FLAG_CPU = (1 << 0), +}; + #endif /* _UAPI_LINUX_MEMBARRIER_H */ diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 168479a7d61b..e23e74d52db5 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -18,6 +18,14 @@ #define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK 0 #endif +#ifdef CONFIG_RSEQ +#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \ + (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \ + | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK) +#else +#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0 +#endif + #define MEMBARRIER_CMD_BITMASK \ (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \ | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \ @@ -30,6 +38,11 @@ static void ipi_mb(void *info) smp_mb(); /* IPIs should be serializing but paranoid. */ } +static void ipi_rseq(void *info) +{ + rseq_preempt(current); +} + static void ipi_sync_rq_state(void *info) { struct mm_struct *mm = (struct mm_struct *) info; @@ -129,19 +142,27 @@ static int membarrier_global_expedited(void) return 0; } -static int membarrier_private_expedited(int flags) +static int membarrier_private_expedited(int flags, int cpu_id) { - int cpu; cpumask_var_t tmpmask; struct mm_struct *mm = current->mm; + smp_call_func_t ipi_func = ipi_mb; - if (flags & MEMBARRIER_FLAG_SYNC_CORE) { + if (flags == MEMBARRIER_FLAG_SYNC_CORE) { if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE)) return -EINVAL; if (!(atomic_read(&mm->membarrier_state) & MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY)) return -EPERM; + } else if (flags == MEMBARRIER_FLAG_RSEQ) { + if (!IS_ENABLED(CONFIG_RSEQ)) + return -EINVAL; + if (!(atomic_read(&mm->membarrier_state) & + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY)) + return -EPERM; + ipi_func = ipi_rseq; } else { + WARN_ON_ONCE(flags); if (!(atomic_read(&mm->membarrier_state) & MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY)) return -EPERM; @@ -156,35 +177,59 @@ static int membarrier_private_expedited(int flags) */ smp_mb(); /* system call entry is not a mb. */ - if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) + if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) return -ENOMEM; cpus_read_lock(); - rcu_read_lock(); - for_each_online_cpu(cpu) { + + if (cpu_id >= 0) { struct task_struct *p; - /* - * Skipping the current CPU is OK even through we can be - * migrated at any point. The current CPU, at the point - * where we read raw_smp_processor_id(), is ensured to - * be in program order with respect to the caller - * thread. Therefore, we can skip this CPU from the - * iteration. - */ - if (cpu == raw_smp_processor_id()) - continue; - p = rcu_dereference(cpu_rq(cpu)->curr); - if (p && p->mm == mm) - __cpumask_set_cpu(cpu, tmpmask); + if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id)) + goto out; + if (cpu_id == raw_smp_processor_id()) + goto out; + rcu_read_lock(); + p = rcu_dereference(cpu_rq(cpu_id)->curr); + if (!p || p->mm != mm) { + rcu_read_unlock(); + goto out; + } + rcu_read_unlock(); + } else { + int cpu; + + rcu_read_lock(); + for_each_online_cpu(cpu) { + struct task_struct *p; + + /* + * Skipping the current CPU is OK even through we can be + * migrated at any point. The current CPU, at the point + * where we read raw_smp_processor_id(), is ensured to + * be in program order with respect to the caller + * thread. Therefore, we can skip this CPU from the + * iteration. + */ + if (cpu == raw_smp_processor_id()) + continue; + p = rcu_dereference(cpu_rq(cpu)->curr); + if (p && p->mm == mm) + __cpumask_set_cpu(cpu, tmpmask); + } + rcu_read_unlock(); } - rcu_read_unlock(); preempt_disable(); - smp_call_function_many(tmpmask, ipi_mb, NULL, 1); + if (cpu_id >= 0) + smp_call_function_single(cpu_id, ipi_func, NULL, 1); + else + smp_call_function_many(tmpmask, ipi_func, NULL, 1); preempt_enable(); - free_cpumask_var(tmpmask); +out: + if (cpu_id < 0) + free_cpumask_var(tmpmask); cpus_read_unlock(); /* @@ -283,11 +328,18 @@ static int membarrier_register_private_expedited(int flags) set_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED, ret; - if (flags & MEMBARRIER_FLAG_SYNC_CORE) { + if (flags == MEMBARRIER_FLAG_SYNC_CORE) { if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE)) return -EINVAL; ready_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY; + } else if (flags == MEMBARRIER_FLAG_RSEQ) { + if (!IS_ENABLED(CONFIG_RSEQ)) + return -EINVAL; + ready_state = + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY; + } else { + WARN_ON_ONCE(flags); } /* @@ -299,6 +351,8 @@ static int membarrier_register_private_expedited(int flags) return 0; if (flags & MEMBARRIER_FLAG_SYNC_CORE) set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE; + if (flags & MEMBARRIER_FLAG_RSEQ) + set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ; atomic_or(set_state, &mm->membarrier_state); ret = sync_runqueues_membarrier_state(mm); if (ret) @@ -310,8 +364,15 @@ static int membarrier_register_private_expedited(int flags) /** * sys_membarrier - issue memory barriers on a set of threads - * @cmd: Takes command values defined in enum membarrier_cmd. - * @flags: Currently needs to be 0. For future extensions. + * @cmd: Takes command values defined in enum membarrier_cmd. + * @flags: Currently needs to be 0 for all commands other than + * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: in the latter + * case it can be MEMBARRIER_CMD_FLAG_CPU, indicating that @cpu_id + * contains the CPU on which to interrupt (= restart) + * the RSEQ critical section. + * @cpu_id: if @flags == MEMBARRIER_CMD_FLAG_CPU, indicates the cpu on which + * RSEQ CS should be interrupted (@cmd must be + * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ). * * If this system call is not implemented, -ENOSYS is returned. If the * command specified does not exist, not available on the running @@ -337,10 +398,21 @@ static int membarrier_register_private_expedited(int flags) * smp_mb() X O O * sys_membarrier() O O O */ -SYSCALL_DEFINE2(membarrier, int, cmd, int, flags) +SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id) { - if (unlikely(flags)) - return -EINVAL; + switch (cmd) { + case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: + if (unlikely(flags && flags != MEMBARRIER_CMD_FLAG_CPU)) + return -EINVAL; + break; + default: + if (unlikely(flags)) + return -EINVAL; + } + + if (!(flags & MEMBARRIER_CMD_FLAG_CPU)) + cpu_id = -1; + switch (cmd) { case MEMBARRIER_CMD_QUERY: { @@ -362,13 +434,17 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags) case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED: return membarrier_register_global_expedited(); case MEMBARRIER_CMD_PRIVATE_EXPEDITED: - return membarrier_private_expedited(0); + return membarrier_private_expedited(0, cpu_id); case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED: return membarrier_register_private_expedited(0); case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: - return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE); + return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, cpu_id); case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE); + case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: + return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, cpu_id); + case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ: + return membarrier_register_private_expedited(MEMBARRIER_FLAG_RSEQ); default: return -EINVAL; } -- cgit v1.3-18-gd494 From 67197a4f28d28d0b073ab0427b03cb2ee5382578 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Tue, 13 Oct 2020 16:58:35 -0700 Subject: mm, oom_adj: don't loop through tasks in __set_oom_adj when not necessary Currently __set_oom_adj loops through all processes in the system to keep oom_score_adj and oom_score_adj_min in sync between processes sharing their mm. This is done for any task with more that one mm_users, which includes processes with multiple threads (sharing mm and signals). However for such processes the loop is unnecessary because their signal structure is shared as well. Android updates oom_score_adj whenever a tasks changes its role (background/foreground/...) or binds to/unbinds from a service, making it more/less important. Such operation can happen frequently. We noticed that updates to oom_score_adj became more expensive and after further investigation found out that the patch mentioned in "Fixes" introduced a regression. Using Pixel 4 with a typical Android workload, write time to oom_score_adj increased from ~3.57us to ~362us. Moreover this regression linearly depends on the number of multi-threaded processes running on the system. Mark the mm with a new MMF_MULTIPROCESS flag bit when task is created with (CLONE_VM && !CLONE_THREAD && !CLONE_VFORK). Change __set_oom_adj to use MMF_MULTIPROCESS instead of mm_users to decide whether oom_score_adj update should be synchronized between multiple processes. To prevent races between clone() and __set_oom_adj(), when oom_score_adj of the process being cloned might be modified from userspace, we use oom_adj_mutex. Its scope is changed to global. The combination of (CLONE_VM && !CLONE_THREAD) is rarely used except for the case of vfork(). To prevent performance regressions of vfork(), we skip taking oom_adj_mutex and setting MMF_MULTIPROCESS when CLONE_VFORK is specified. Clearing the MMF_MULTIPROCESS flag (when the last process sharing the mm exits) is left out of this patch to keep it simple and because it is believed that this threading model is rare. Should there ever be a need for optimizing that case as well, it can be done by hooking into the exit path, likely following the mm_update_next_owner pattern. With the combination of (CLONE_VM && !CLONE_THREAD && !CLONE_VFORK) being quite rare, the regression is gone after the change is applied. [surenb@google.com: v3] Link: https://lkml.kernel.org/r/20200902012558.2335613-1-surenb@google.com Fixes: 44a70adec910 ("mm, oom_adj: make sure processes sharing mm have same view of oom_score_adj") Reported-by: Tim Murray Suggested-by: Michal Hocko Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Acked-by: Christian Brauner Acked-by: Michal Hocko Acked-by: Oleg Nesterov Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Eugene Syromiatnikov Cc: Christian Kellner Cc: Adrian Reber Cc: Shakeel Butt Cc: Aleksa Sarai Cc: Alexey Dobriyan Cc: "Eric W. Biederman" Cc: Alexey Gladkov Cc: Michel Lespinasse Cc: Daniel Jordan Cc: Andrei Vagin Cc: Bernd Edlinger Cc: John Johansen Cc: Yafang Shao Link: https://lkml.kernel.org/r/20200824153036.3201505-1-surenb@google.com Debugged-by: Minchan Kim Signed-off-by: Linus Torvalds --- fs/proc/base.c | 3 +-- include/linux/oom.h | 1 + include/linux/sched/coredump.h | 1 + kernel/fork.c | 21 +++++++++++++++++++++ mm/oom_kill.c | 2 ++ 5 files changed, 26 insertions(+), 2 deletions(-) (limited to 'include/linux/sched') diff --git a/fs/proc/base.c b/fs/proc/base.c index 617db4e0faa0..aa69c35d904c 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1055,7 +1055,6 @@ static ssize_t oom_adj_read(struct file *file, char __user *buf, size_t count, static int __set_oom_adj(struct file *file, int oom_adj, bool legacy) { - static DEFINE_MUTEX(oom_adj_mutex); struct mm_struct *mm = NULL; struct task_struct *task; int err = 0; @@ -1095,7 +1094,7 @@ static int __set_oom_adj(struct file *file, int oom_adj, bool legacy) struct task_struct *p = find_lock_task_mm(task); if (p) { - if (atomic_read(&p->mm->mm_users) > 1) { + if (test_bit(MMF_MULTIPROCESS, &p->mm->flags)) { mm = p->mm; mmgrab(mm); } diff --git a/include/linux/oom.h b/include/linux/oom.h index f022f581ac29..2db9a1432511 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -55,6 +55,7 @@ struct oom_control { }; extern struct mutex oom_lock; +extern struct mutex oom_adj_mutex; static inline void set_current_oom_origin(void) { diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index ecdc6542070f..dfd82eab2902 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -72,6 +72,7 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_DISABLE_THP 24 /* disable THP for all VMAs */ #define MMF_OOM_VICTIM 25 /* mm is the oom victim */ #define MMF_OOM_REAP_QUEUED 26 /* mm was queued for oom_reaper */ +#define MMF_MULTIPROCESS 27 /* mm is shared between processes */ #define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ diff --git a/kernel/fork.c b/kernel/fork.c index ede26e5a6097..50c90d368117 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1812,6 +1812,25 @@ static __always_inline void delayed_free_task(struct task_struct *tsk) free_task(tsk); } +static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk) +{ + /* Skip if kernel thread */ + if (!tsk->mm) + return; + + /* Skip if spawning a thread or using vfork */ + if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM) + return; + + /* We need to synchronize with __set_oom_adj */ + mutex_lock(&oom_adj_mutex); + set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); + /* Update the values in case they were changed after copy_signal */ + tsk->signal->oom_score_adj = current->signal->oom_score_adj; + tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; + mutex_unlock(&oom_adj_mutex); +} + /* * This creates a new process as a copy of the old one, * but does not actually start it yet. @@ -2288,6 +2307,8 @@ static __latent_entropy struct task_struct *copy_process( trace_task_newtask(p, clone_flags); uprobe_copy_process(p, clone_flags); + copy_oom_score_adj(clone_flags, p); + return p; bad_fork_cancel_cgroup: diff --git a/mm/oom_kill.c b/mm/oom_kill.c index e90f25d6385d..8b84661a6410 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -64,6 +64,8 @@ int sysctl_oom_dump_tasks = 1; * and mark_oom_victim */ DEFINE_MUTEX(oom_lock); +/* Serializes oom_score_adj and oom_score_adj_min updates */ +DEFINE_MUTEX(oom_adj_mutex); static inline bool is_memcg_oom(struct oom_control *oc) { -- cgit v1.3-18-gd494 From 4d45e75a9955ade5c2f49bd96fc4173b2cec9a72 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Thu, 15 Oct 2020 20:13:00 -0700 Subject: mm: remove the now-unnecessary mmget_still_valid() hack The preceding patches have ensured that core dumping properly takes the mmap_lock. Thanks to that, we can now remove mmget_still_valid() and all its users. Signed-off-by: Jann Horn Signed-off-by: Andrew Morton Acked-by: Linus Torvalds Cc: Christoph Hellwig Cc: Alexander Viro Cc: "Eric W . Biederman" Cc: Oleg Nesterov Cc: Hugh Dickins Link: http://lkml.kernel.org/r/20200827114932.3572699-8-jannh@google.com Signed-off-by: Linus Torvalds --- drivers/infiniband/core/uverbs_main.c | 3 --- drivers/vfio/pci/vfio_pci.c | 38 +++++++++++++++++------------------ fs/proc/task_mmu.c | 18 ----------------- fs/userfaultfd.c | 28 +++++++++----------------- include/linux/sched/mm.h | 25 ----------------------- mm/khugepaged.c | 2 +- mm/madvise.c | 17 ---------------- mm/mmap.c | 5 +---- 8 files changed, 29 insertions(+), 107 deletions(-) (limited to 'include/linux/sched') diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 37794d88b1f3..a4ba0b87d6de 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -845,8 +845,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) * will only be one mm, so no big deal. */ mmap_read_lock(mm); - if (!mmget_still_valid(mm)) - goto skip_mm; mutex_lock(&ufile->umap_lock); list_for_each_entry_safe (priv, next_priv, &ufile->umaps, list) { @@ -865,7 +863,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) } } mutex_unlock(&ufile->umap_lock); - skip_mm: mmap_read_unlock(mm); mmput(mm); } diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 1ab1f5cda4ac..b0f4b92a87ed 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -1480,31 +1480,29 @@ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) } else { mmap_read_lock(mm); } - if (mmget_still_valid(mm)) { - if (try) { - if (!mutex_trylock(&vdev->vma_lock)) { - mmap_read_unlock(mm); - mmput(mm); - return 0; - } - } else { - mutex_lock(&vdev->vma_lock); + if (try) { + if (!mutex_trylock(&vdev->vma_lock)) { + mmap_read_unlock(mm); + mmput(mm); + return 0; } - list_for_each_entry_safe(mmap_vma, tmp, - &vdev->vma_list, vma_next) { - struct vm_area_struct *vma = mmap_vma->vma; + } else { + mutex_lock(&vdev->vma_lock); + } + list_for_each_entry_safe(mmap_vma, tmp, + &vdev->vma_list, vma_next) { + struct vm_area_struct *vma = mmap_vma->vma; - if (vma->vm_mm != mm) - continue; + if (vma->vm_mm != mm) + continue; - list_del(&mmap_vma->vma_next); - kfree(mmap_vma); + list_del(&mmap_vma->vma_next); + kfree(mmap_vma); - zap_vma_ptes(vma, vma->vm_start, - vma->vm_end - vma->vm_start); - } - mutex_unlock(&vdev->vma_lock); + zap_vma_ptes(vma, vma->vm_start, + vma->vm_end - vma->vm_start); } + mutex_unlock(&vdev->vma_lock); mmap_read_unlock(mm); mmput(mm); } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 846d43df3fdf..217aa2705d5d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1244,24 +1244,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, count = -EINTR; goto out_mm; } - /* - * Avoid to modify vma->vm_flags - * without locked ops while the - * coredump reads the vm_flags. - */ - if (!mmget_still_valid(mm)) { - /* - * Silently return "count" - * like if get_task_mm() - * failed. FIXME: should this - * function have returned - * -ESRCH if get_task_mm() - * failed like if - * get_proc_task() fails? - */ - mmap_write_unlock(mm); - goto out_mm; - } for (vma = mm->mmap; vma; vma = vma->vm_next) { vma->vm_flags &= ~VM_SOFTDIRTY; vma_set_page_prot(vma); diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 0e4a3837da52..000b457ad087 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -601,8 +601,6 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, /* the various vma->vm_userfaultfd_ctx still points to it */ mmap_write_lock(mm); - /* no task can run (and in turn coredump) yet */ - VM_WARN_ON(!mmget_still_valid(mm)); for (vma = mm->mmap; vma; vma = vma->vm_next) if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; @@ -842,7 +840,6 @@ static int userfaultfd_release(struct inode *inode, struct file *file) /* len == 0 means wake all */ struct userfaultfd_wake_range range = { .len = 0, }; unsigned long new_flags; - bool still_valid; WRITE_ONCE(ctx->released, true); @@ -858,7 +855,6 @@ static int userfaultfd_release(struct inode *inode, struct file *file) * taking the mmap_lock for writing. */ mmap_write_lock(mm); - still_valid = mmget_still_valid(mm); prev = NULL; for (vma = mm->mmap; vma; vma = vma->vm_next) { cond_resched(); @@ -869,17 +865,15 @@ static int userfaultfd_release(struct inode *inode, struct file *file) continue; } new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); - if (still_valid) { - prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, - new_flags, vma->anon_vma, - vma->vm_file, vma->vm_pgoff, - vma_policy(vma), - NULL_VM_UFFD_CTX); - if (prev) - vma = prev; - else - prev = vma; - } + prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, + new_flags, vma->anon_vma, + vma->vm_file, vma->vm_pgoff, + vma_policy(vma), + NULL_VM_UFFD_CTX); + if (prev) + vma = prev; + else + prev = vma; vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; } @@ -1309,8 +1303,6 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out; mmap_write_lock(mm); - if (!mmget_still_valid(mm)) - goto out_unlock; vma = find_vma_prev(mm, start, &prev); if (!vma) goto out_unlock; @@ -1511,8 +1503,6 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, goto out; mmap_write_lock(mm); - if (!mmget_still_valid(mm)) - goto out_unlock; vma = find_vma_prev(mm, start, &prev); if (!vma) goto out_unlock; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 15bfb06f2884..981e34cb1409 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -49,31 +49,6 @@ static inline void mmdrop(struct mm_struct *mm) __mmdrop(mm); } -/* - * This has to be called after a get_task_mm()/mmget_not_zero() - * followed by taking the mmap_lock for writing before modifying the - * vmas or anything the coredump pretends not to change from under it. - * - * It also has to be called when mmgrab() is used in the context of - * the process, but then the mm_count refcount is transferred outside - * the context of the process to run down_write() on that pinned mm. - * - * NOTE: find_extend_vma() called from GUP context is the only place - * that can modify the "mm" (notably the vm_start/end) under mmap_lock - * for reading and outside the context of the process, so it is also - * the only case that holds the mmap_lock for reading that must call - * this function. Generally if the mmap_lock is hold for reading - * there's no need of this check after get_task_mm()/mmget_not_zero(). - * - * This function can be obsoleted and the check can be removed, after - * the coredump code will hold the mmap_lock for writing before - * invoking the ->core_dump methods. - */ -static inline bool mmget_still_valid(struct mm_struct *mm) -{ - return likely(!mm->core_state); -} - /** * mmget() - Pin the address space associated with a &struct mm_struct. * @mm: The address space to pin. diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 58b0d9c502a1..4e3dff13eb70 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -434,7 +434,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, static inline int khugepaged_test_exit(struct mm_struct *mm) { - return atomic_read(&mm->mm_users) == 0 || !mmget_still_valid(mm); + return atomic_read(&mm->mm_users) == 0; } static bool hugepage_vma_check(struct vm_area_struct *vma, diff --git a/mm/madvise.c b/mm/madvise.c index 68b761c359d0..fd1f448b4e1d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1085,23 +1085,6 @@ int do_madvise(unsigned long start, size_t len_in, int behavior) if (write) { if (mmap_write_lock_killable(current->mm)) return -EINTR; - - /* - * We may have stolen the mm from another process - * that is undergoing core dumping. - * - * Right now that's io_ring, in the future it may - * be remote process management and not "current" - * at all. - * - * We need to fix core dumping to not do this, - * but for now we have the mmget_still_valid() - * model. - */ - if (!mmget_still_valid(current->mm)) { - mmap_write_unlock(current->mm); - return -EINTR; - } } else { mmap_read_lock(current->mm); } diff --git a/mm/mmap.c b/mm/mmap.c index 9fb9f8a233c7..ebb92f5515a1 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2562,7 +2562,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) if (vma && (vma->vm_start <= addr)) return vma; /* don't alter vm_end if the coredump is running */ - if (!prev || !mmget_still_valid(mm) || expand_stack(prev, addr)) + if (!prev || expand_stack(prev, addr)) return NULL; if (prev->vm_flags & VM_LOCKED) populate_vma_page_range(prev, addr, prev->vm_end, NULL); @@ -2588,9 +2588,6 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) return vma; if (!(vma->vm_flags & VM_GROWSDOWN)) return NULL; - /* don't alter vm_start if the coredump is running */ - if (!mmget_still_valid(mm)) - return NULL; start = vma->vm_start; if (expand_stack(vma, addr)) return NULL; -- cgit v1.3-18-gd494 From b87d8cefe43c7f22e8aa13919c1dfa2b4b4b4e01 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Sat, 17 Oct 2020 16:13:40 -0700 Subject: mm, memcg: rework remote charging API to support nesting Currently the remote memcg charging API consists of two functions: memalloc_use_memcg() and memalloc_unuse_memcg(), which set and clear the memcg value, which overwrites the memcg of the current task. memalloc_use_memcg(target_memcg); <...> memalloc_unuse_memcg(); It works perfectly for allocations performed from a normal context, however an attempt to call it from an interrupt context or just nest two remote charging blocks will lead to an incorrect accounting. On exit from the inner block the active memcg will be cleared instead of being restored. memalloc_use_memcg(target_memcg); memalloc_use_memcg(target_memcg_2); <...> memalloc_unuse_memcg(); Error: allocation here are charged to the memcg of the current process instead of target_memcg. memalloc_unuse_memcg(); This patch extends the remote charging API by switching to a single function: struct mem_cgroup *set_active_memcg(struct mem_cgroup *memcg), which sets the new value and returns the old one. So a remote charging block will look like: old_memcg = set_active_memcg(target_memcg); <...> set_active_memcg(old_memcg); This patch is heavily based on the patch by Johannes Weiner, which can be found here: https://lkml.org/lkml/2020/5/28/806 . Signed-off-by: Roman Gushchin Signed-off-by: Andrew Morton Reviewed-by: Shakeel Butt Cc: Johannes Weiner Cc: Dan Schatzberg Link: https://lkml.kernel.org/r/20200821212056.3769116-1-guro@fb.com Signed-off-by: Linus Torvalds --- fs/buffer.c | 6 +++--- fs/notify/fanotify/fanotify.c | 5 +++-- fs/notify/inotify/inotify_fsnotify.c | 5 +++-- include/linux/sched/mm.h | 30 ++++++++++-------------------- mm/memcontrol.c | 6 +++--- 5 files changed, 22 insertions(+), 30 deletions(-) (limited to 'include/linux/sched') diff --git a/fs/buffer.c b/fs/buffer.c index 5a28a6aa7f16..23f645657488 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -842,13 +842,13 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, struct buffer_head *bh, *head; gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; long offset; - struct mem_cgroup *memcg; + struct mem_cgroup *memcg, *old_memcg; if (retry) gfp |= __GFP_NOFAIL; memcg = get_mem_cgroup_from_page(page); - memalloc_use_memcg(memcg); + old_memcg = set_active_memcg(memcg); head = NULL; offset = PAGE_SIZE; @@ -867,7 +867,7 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, set_bh_page(bh, page, offset); } out: - memalloc_unuse_memcg(); + set_active_memcg(old_memcg); mem_cgroup_put(memcg); return head; /* diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index c942910a8649..9167884a61ec 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -531,6 +531,7 @@ static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group, struct inode *dirid = fanotify_dfid_inode(mask, data, data_type, dir); const struct path *path = fsnotify_data_path(data, data_type); unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); + struct mem_cgroup *old_memcg; struct inode *child = NULL; bool name_event = false; @@ -580,7 +581,7 @@ static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group, gfp |= __GFP_RETRY_MAYFAIL; /* Whoever is interested in the event, pays for the allocation. */ - memalloc_use_memcg(group->memcg); + old_memcg = set_active_memcg(group->memcg); if (fanotify_is_perm_event(mask)) { event = fanotify_alloc_perm_event(path, gfp); @@ -608,7 +609,7 @@ static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group, event->pid = get_pid(task_tgid(current)); out: - memalloc_unuse_memcg(); + set_active_memcg(old_memcg); return event; } diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c index a65cf8c9f600..9ddcbadc98e2 100644 --- a/fs/notify/inotify/inotify_fsnotify.c +++ b/fs/notify/inotify/inotify_fsnotify.c @@ -66,6 +66,7 @@ static int inotify_one_event(struct fsnotify_group *group, u32 mask, int ret; int len = 0; int alloc_len = sizeof(struct inotify_event_info); + struct mem_cgroup *old_memcg; if ((inode_mark->mask & FS_EXCL_UNLINK) && path && d_unlinked(path->dentry)) @@ -87,9 +88,9 @@ static int inotify_one_event(struct fsnotify_group *group, u32 mask, * trigger OOM killer in the target monitoring memcg as it may have * security repercussion. */ - memalloc_use_memcg(group->memcg); + old_memcg = set_active_memcg(group->memcg); event = kmalloc(alloc_len, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); - memalloc_unuse_memcg(); + set_active_memcg(old_memcg); if (unlikely(!event)) { /* diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 981e34cb1409..1a80fb128e74 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -280,38 +280,28 @@ static inline void memalloc_nocma_restore(unsigned int flags) #ifdef CONFIG_MEMCG /** - * memalloc_use_memcg - Starts the remote memcg charging scope. + * set_active_memcg - Starts the remote memcg charging scope. * @memcg: memcg to charge. * * This function marks the beginning of the remote memcg charging scope. All the * __GFP_ACCOUNT allocations till the end of the scope will be charged to the * given memcg. * - * NOTE: This function is not nesting safe. + * NOTE: This function can nest. Users must save the return value and + * reset the previous value after their own charging scope is over. */ -static inline void memalloc_use_memcg(struct mem_cgroup *memcg) +static inline struct mem_cgroup * +set_active_memcg(struct mem_cgroup *memcg) { - WARN_ON_ONCE(current->active_memcg); + struct mem_cgroup *old = current->active_memcg; current->active_memcg = memcg; -} - -/** - * memalloc_unuse_memcg - Ends the remote memcg charging scope. - * - * This function marks the end of the remote memcg charging scope started by - * memalloc_use_memcg(). - */ -static inline void memalloc_unuse_memcg(void) -{ - current->active_memcg = NULL; + return old; } #else -static inline void memalloc_use_memcg(struct mem_cgroup *memcg) -{ -} - -static inline void memalloc_unuse_memcg(void) +static inline struct mem_cgroup * +set_active_memcg(struct mem_cgroup *memcg) { + return NULL; } #endif diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7f74a158cfa8..4c741248198b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5290,12 +5290,12 @@ static struct cgroup_subsys_state * __ref mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) { struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); - struct mem_cgroup *memcg; + struct mem_cgroup *memcg, *old_memcg; long error = -ENOMEM; - memalloc_use_memcg(parent); + old_memcg = set_active_memcg(parent); memcg = mem_cgroup_alloc(); - memalloc_unuse_memcg(); + set_active_memcg(old_memcg); if (IS_ERR(memcg)) return ERR_CAST(memcg); -- cgit v1.3-18-gd494 From 37d5985c003daab138a72dd4af9853b396d91c26 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Sat, 17 Oct 2020 16:13:50 -0700 Subject: mm: kmem: prepare remote memcg charging infra for interrupt contexts Remote memcg charging API uses current->active_memcg to store the currently active memory cgroup, which overwrites the memory cgroup of the current process. It works well for normal contexts, but doesn't work for interrupt contexts: indeed, if an interrupt occurs during the execution of a section with an active memcg set, all allocations inside the interrupt will be charged to the active memcg set (given that we'll enable accounting for allocations from an interrupt context). But because the interrupt might have no relation to the active memcg set outside, it's obviously wrong from the accounting prospective. To resolve this problem, let's add a global percpu int_active_memcg variable, which will be used to store an active memory cgroup which will be used from interrupt contexts. set_active_memcg() will transparently use current->active_memcg or int_active_memcg depending on the context. To make the read part simple and transparent for the caller, let's introduce two new functions: - struct mem_cgroup *active_memcg(void), - struct mem_cgroup *get_active_memcg(void). They are returning the active memcg if it's set, hiding all implementation details: where to get it depending on the current context. Signed-off-by: Roman Gushchin Signed-off-by: Andrew Morton Reviewed-by: Shakeel Butt Cc: Johannes Weiner Cc: Michal Hocko Link: http://lkml.kernel.org/r/20200827225843.1270629-4-guro@fb.com Signed-off-by: Linus Torvalds --- include/linux/sched/mm.h | 13 +++++++++++-- mm/memcontrol.c | 48 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 16 deletions(-) (limited to 'include/linux/sched') diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 1a80fb128e74..d5ece7a9a403 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -279,6 +279,7 @@ static inline void memalloc_nocma_restore(unsigned int flags) #endif #ifdef CONFIG_MEMCG +DECLARE_PER_CPU(struct mem_cgroup *, int_active_memcg); /** * set_active_memcg - Starts the remote memcg charging scope. * @memcg: memcg to charge. @@ -293,8 +294,16 @@ static inline void memalloc_nocma_restore(unsigned int flags) static inline struct mem_cgroup * set_active_memcg(struct mem_cgroup *memcg) { - struct mem_cgroup *old = current->active_memcg; - current->active_memcg = memcg; + struct mem_cgroup *old; + + if (in_interrupt()) { + old = this_cpu_read(int_active_memcg); + this_cpu_write(int_active_memcg, memcg); + } else { + old = current->active_memcg; + current->active_memcg = memcg; + } + return old; } #else diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 51b1698bf06c..a3318b66e41e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -73,6 +73,9 @@ EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly; +/* Active memory cgroup to use from an interrupt context */ +DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); + /* Socket memory accounting disabled? */ static bool cgroup_memory_nosocket; @@ -1061,26 +1064,43 @@ struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) } EXPORT_SYMBOL(get_mem_cgroup_from_page); -/** - * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg. - */ -static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void) +static __always_inline struct mem_cgroup *active_memcg(void) { - if (memcg_kmem_bypass()) - return NULL; + if (in_interrupt()) + return this_cpu_read(int_active_memcg); + else + return current->active_memcg; +} - if (unlikely(current->active_memcg)) { - struct mem_cgroup *memcg; +static __always_inline struct mem_cgroup *get_active_memcg(void) +{ + struct mem_cgroup *memcg; - rcu_read_lock(); + rcu_read_lock(); + memcg = active_memcg(); + if (memcg) { /* current->active_memcg must hold a ref. */ - if (WARN_ON_ONCE(!css_tryget(¤t->active_memcg->css))) + if (WARN_ON_ONCE(!css_tryget(&memcg->css))) memcg = root_mem_cgroup; else memcg = current->active_memcg; - rcu_read_unlock(); - return memcg; } + rcu_read_unlock(); + + return memcg; +} + +/** + * If active memcg is set, do not fallback to current->mm->memcg. + */ +static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void) +{ + if (memcg_kmem_bypass()) + return NULL; + + if (unlikely(active_memcg())) + return get_active_memcg(); + return get_mem_cgroup_from_mm(current->mm); } @@ -2940,8 +2960,8 @@ __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void) return NULL; rcu_read_lock(); - if (unlikely(current->active_memcg)) - memcg = rcu_dereference(current->active_memcg); + if (unlikely(active_memcg())) + memcg = active_memcg(); else memcg = mem_cgroup_from_task(current); -- cgit v1.3-18-gd494 From 33def8498fdde180023444b08e12b72a9efed41d Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Wed, 21 Oct 2020 19:36:07 -0700 Subject: treewide: Convert macro and uses of __section(foo) to __section("foo") Use a more generic form for __section that requires quotes to avoid complications with clang and gcc differences. Remove the quote operator # from compiler_attributes.h __section macro. Convert all unquoted __section(foo) uses to quoted __section("foo"). Also convert __attribute__((section("foo"))) uses to __section("foo") even if the __attribute__ has multiple list entry forms. Conversion done using the script at: https://lore.kernel.org/lkml/75393e5ddc272dc7403de74d645e6c6e0f4e70eb.camel@perches.com/2-convert_section.pl Signed-off-by: Joe Perches Reviewed-by: Nick Desaulniers Reviewed-by: Miguel Ojeda Signed-off-by: Linus Torvalds --- arch/arc/include/asm/linkage.h | 8 ++++---- arch/arc/include/asm/mach_desc.h | 2 +- arch/arc/plat-hsdk/platform.c | 2 +- arch/arm/include/asm/cache.h | 2 +- arch/arm/include/asm/cpuidle.h | 2 +- arch/arm/include/asm/idmap.h | 2 +- arch/arm/include/asm/mach/arch.h | 4 ++-- arch/arm/include/asm/setup.h | 2 +- arch/arm/include/asm/smp.h | 2 +- arch/arm/include/asm/tcm.h | 8 ++++---- arch/arm/kernel/cpuidle.c | 2 +- arch/arm/kernel/devtree.c | 2 +- arch/arm64/include/asm/cache.h | 2 +- arch/arm64/kernel/efi.c | 2 +- arch/arm64/kernel/smp_spin_table.c | 2 +- arch/arm64/mm/mmu.c | 2 +- arch/csky/include/asm/tcm.h | 8 ++++---- arch/ia64/include/asm/cache.h | 2 +- arch/microblaze/kernel/setup.c | 2 +- arch/mips/include/asm/cache.h | 2 +- arch/mips/include/asm/machine.h | 2 +- arch/mips/kernel/setup.c | 2 +- arch/mips/mm/init.c | 2 +- arch/parisc/include/asm/cache.h | 2 +- arch/parisc/include/asm/ldcw.h | 2 +- arch/parisc/kernel/ftrace.c | 2 +- arch/parisc/mm/init.c | 6 +++--- arch/powerpc/include/asm/cache.h | 2 +- arch/powerpc/include/asm/machdep.h | 2 +- arch/powerpc/kernel/btext.c | 2 +- arch/powerpc/kernel/prom_init.c | 2 +- arch/powerpc/kvm/book3s_64_vio_hv.c | 2 +- arch/riscv/include/asm/soc.h | 4 ++-- arch/riscv/kernel/cpu_ops.c | 4 ++-- arch/riscv/kernel/setup.c | 4 ++-- arch/s390/boot/startup.c | 2 +- arch/s390/include/asm/cache.h | 2 +- arch/s390/include/asm/sections.h | 4 ++-- arch/s390/mm/init.c | 2 +- arch/sh/boards/of-generic.c | 2 +- arch/sh/include/asm/cache.h | 2 +- arch/sh/include/asm/machvec.h | 2 +- arch/sh/include/asm/smp.h | 2 +- arch/sparc/include/asm/cache.h | 2 +- arch/sparc/kernel/btext.c | 2 +- arch/um/include/shared/init.h | 22 +++++++++++----------- arch/um/kernel/skas/clone.c | 2 +- arch/um/kernel/um_arch.c | 2 +- arch/x86/boot/compressed/pgtable_64.c | 8 ++++---- arch/x86/boot/tty.c | 8 ++++---- arch/x86/boot/video.h | 2 +- arch/x86/include/asm/apic.h | 4 ++-- arch/x86/include/asm/cache.h | 2 +- arch/x86/include/asm/intel-mid.h | 2 +- arch/x86/include/asm/irqflags.h | 2 +- arch/x86/include/asm/mem_encrypt.h | 2 +- arch/x86/include/asm/setup.h | 2 +- arch/x86/kernel/cpu/cpu.h | 2 +- arch/x86/kernel/head64.c | 2 +- arch/x86/mm/mem_encrypt.c | 6 +++--- arch/x86/mm/mem_encrypt_identity.c | 2 +- arch/x86/platform/pvh/enlighten.c | 4 ++-- arch/x86/purgatory/purgatory.c | 4 ++-- arch/x86/um/stub_segv.c | 2 +- arch/x86/xen/enlighten.c | 2 +- arch/x86/xen/enlighten_pvh.c | 2 +- arch/xtensa/kernel/setup.c | 2 +- drivers/clk/clk.c | 2 +- drivers/clocksource/timer-probe.c | 2 +- drivers/irqchip/irqchip.c | 2 +- drivers/of/of_reserved_mem.c | 2 +- drivers/thermal/thermal_core.h | 2 +- fs/xfs/xfs_message.h | 2 +- include/asm-generic/bug.h | 6 +++--- include/asm-generic/error-injection.h | 2 +- include/asm-generic/kprobes.h | 4 ++-- include/kunit/test.h | 2 +- include/linux/acpi.h | 4 ++-- include/linux/cache.h | 2 +- include/linux/compiler.h | 8 ++++---- include/linux/compiler_attributes.h | 2 +- include/linux/cpu.h | 2 +- include/linux/dynamic_debug.h | 2 +- include/linux/export.h | 2 +- include/linux/firmware.h | 2 +- include/linux/init.h | 34 +++++++++++++++++----------------- include/linux/init_task.h | 4 ++-- include/linux/interrupt.h | 4 ++-- include/linux/kernel.h | 6 +++--- include/linux/linkage.h | 4 ++-- include/linux/lsm_hooks.h | 4 ++-- include/linux/module.h | 2 +- include/linux/moduleparam.h | 4 ++-- include/linux/mtd/xip.h | 2 +- include/linux/objtool.h | 2 +- include/linux/of.h | 2 +- include/linux/percpu-defs.h | 2 +- include/linux/printk.h | 4 ++-- include/linux/rcupdate.h | 2 +- include/linux/sched/debug.h | 2 +- include/linux/serial_core.h | 2 +- include/linux/spinlock.h | 2 +- include/linux/syscalls.h | 6 +++--- include/linux/trace_events.h | 2 +- include/linux/tracepoint.h | 8 ++++---- include/trace/bpf_probe.h | 2 +- include/trace/trace_events.h | 10 +++++----- kernel/kallsyms.c | 4 ++-- kernel/sched/deadline.c | 2 +- kernel/sched/fair.c | 2 +- kernel/sched/idle.c | 2 +- kernel/sched/rt.c | 2 +- kernel/sched/stop_task.c | 2 +- kernel/trace/trace.h | 2 +- kernel/trace/trace_export.c | 2 +- scripts/mod/modpost.c | 4 ++-- tools/include/linux/objtool.h | 2 +- 117 files changed, 196 insertions(+), 196 deletions(-) (limited to 'include/linux/sched') diff --git a/arch/arc/include/asm/linkage.h b/arch/arc/include/asm/linkage.h index fe19f1d412e7..c9434ff3aa4c 100644 --- a/arch/arc/include/asm/linkage.h +++ b/arch/arc/include/asm/linkage.h @@ -64,15 +64,15 @@ #else /* !__ASSEMBLY__ */ #ifdef CONFIG_ARC_HAS_ICCM -#define __arcfp_code __section(.text.arcfp) +#define __arcfp_code __section(".text.arcfp") #else -#define __arcfp_code __section(.text) +#define __arcfp_code __section(".text") #endif #ifdef CONFIG_ARC_HAS_DCCM -#define __arcfp_data __section(.data.arcfp) +#define __arcfp_data __section(".data.arcfp") #else -#define __arcfp_data __section(.data) +#define __arcfp_data __section(".data") #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/arc/include/asm/mach_desc.h b/arch/arc/include/asm/mach_desc.h index 73746ed5b834..c4e197059379 100644 --- a/arch/arc/include/asm/mach_desc.h +++ b/arch/arc/include/asm/mach_desc.h @@ -53,7 +53,7 @@ extern const struct machine_desc __arch_info_begin[], __arch_info_end[]; */ #define MACHINE_START(_type, _name) \ static const struct machine_desc __mach_desc_##_type \ -__used __section(.arch.info.init) = { \ +__used __section(".arch.info.init") = { \ .name = _name, #define MACHINE_END \ diff --git a/arch/arc/plat-hsdk/platform.c b/arch/arc/plat-hsdk/platform.c index 0b961a2a10b8..0b63fc095b99 100644 --- a/arch/arc/plat-hsdk/platform.c +++ b/arch/arc/plat-hsdk/platform.c @@ -13,7 +13,7 @@ #include #include -int arc_hsdk_axi_dmac_coherent __section(.data) = 0; +int arc_hsdk_axi_dmac_coherent __section(".data") = 0; #define ARC_CCM_UNUSED_ADDR 0x60000000 diff --git a/arch/arm/include/asm/cache.h b/arch/arm/include/asm/cache.h index 1d65ed3a2755..e3ea34558ada 100644 --- a/arch/arm/include/asm/cache.h +++ b/arch/arm/include/asm/cache.h @@ -24,6 +24,6 @@ #define ARCH_SLAB_MINALIGN 8 #endif -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #endif diff --git a/arch/arm/include/asm/cpuidle.h b/arch/arm/include/asm/cpuidle.h index 6b2ff7243b4b..0d67ed682e07 100644 --- a/arch/arm/include/asm/cpuidle.h +++ b/arch/arm/include/asm/cpuidle.h @@ -42,7 +42,7 @@ struct of_cpuidle_method { #define CPUIDLE_METHOD_OF_DECLARE(name, _method, _ops) \ static const struct of_cpuidle_method __cpuidle_method_of_table_##name \ - __used __section(__cpuidle_method_of_table) \ + __used __section("__cpuidle_method_of_table") \ = { .method = _method, .ops = _ops } extern int arm_cpuidle_suspend(int index); diff --git a/arch/arm/include/asm/idmap.h b/arch/arm/include/asm/idmap.h index aab7e8358e6a..baebb67b3512 100644 --- a/arch/arm/include/asm/idmap.h +++ b/arch/arm/include/asm/idmap.h @@ -6,7 +6,7 @@ #include /* Tag a function as requiring to be executed via an identity mapping. */ -#define __idmap __section(.idmap.text) noinline notrace +#define __idmap __section(".idmap.text") noinline notrace extern pgd_t *idmap_pgd; diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index e7df5a822cab..eec0c0bda766 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -81,7 +81,7 @@ extern const struct machine_desc __arch_info_begin[], __arch_info_end[]; #define MACHINE_START(_type,_name) \ static const struct machine_desc __mach_desc_##_type \ __used \ - __attribute__((__section__(".arch.info.init"))) = { \ + __section(".arch.info.init") = { \ .nr = MACH_TYPE_##_type, \ .name = _name, @@ -91,7 +91,7 @@ static const struct machine_desc __mach_desc_##_type \ #define DT_MACHINE_START(_name, _namestr) \ static const struct machine_desc __mach_desc_##_name \ __used \ - __attribute__((__section__(".arch.info.init"))) = { \ + __section(".arch.info.init") = { \ .nr = ~0, \ .name = _namestr, diff --git a/arch/arm/include/asm/setup.h b/arch/arm/include/asm/setup.h index 67d20712cb48..3ae68a1b3de6 100644 --- a/arch/arm/include/asm/setup.h +++ b/arch/arm/include/asm/setup.h @@ -14,7 +14,7 @@ #include -#define __tag __used __attribute__((__section__(".taglist.init"))) +#define __tag __used __section(".taglist.init") #define __tagtable(tag, fn) \ static const struct tagtable __tagtable_##fn __tag = { tag, fn } diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 0ca55a607d0a..5d508f5d56c4 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -112,7 +112,7 @@ struct of_cpu_method { #define CPU_METHOD_OF_DECLARE(name, _method, _ops) \ static const struct of_cpu_method __cpu_method_of_table_##name \ - __used __section(__cpu_method_of_table) \ + __used __section("__cpu_method_of_table") \ = { .method = _method, .ops = _ops } /* * set platform specific SMP operations diff --git a/arch/arm/include/asm/tcm.h b/arch/arm/include/asm/tcm.h index b845b10fe29a..d8bd8a4b0ede 100644 --- a/arch/arm/include/asm/tcm.h +++ b/arch/arm/include/asm/tcm.h @@ -16,13 +16,13 @@ #include /* Tag variables with this */ -#define __tcmdata __section(.tcm.data) +#define __tcmdata __section(".tcm.data") /* Tag constants with this */ -#define __tcmconst __section(.tcm.rodata) +#define __tcmconst __section(".tcm.rodata") /* Tag functions inside TCM called from outside TCM with this */ -#define __tcmfunc __attribute__((long_call)) __section(.tcm.text) noinline +#define __tcmfunc __attribute__((long_call)) __section(".tcm.text") noinline /* Tag function inside TCM called from inside TCM with this */ -#define __tcmlocalfunc __section(.tcm.text) +#define __tcmlocalfunc __section(".tcm.text") void *tcm_alloc(size_t len); void tcm_free(void *addr, size_t len); diff --git a/arch/arm/kernel/cpuidle.c b/arch/arm/kernel/cpuidle.c index 093368e0d020..e1684623e1b2 100644 --- a/arch/arm/kernel/cpuidle.c +++ b/arch/arm/kernel/cpuidle.c @@ -11,7 +11,7 @@ extern struct of_cpuidle_method __cpuidle_method_of_table[]; static const struct of_cpuidle_method __cpuidle_method_of_table_sentinel - __used __section(__cpuidle_method_of_table_end); + __used __section("__cpuidle_method_of_table_end"); static struct cpuidle_ops cpuidle_ops[NR_CPUS] __ro_after_init; diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 39c978698406..7f0745a97e20 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -29,7 +29,7 @@ extern struct of_cpu_method __cpu_method_of_table[]; static const struct of_cpu_method __cpu_method_of_table_sentinel - __used __section(__cpu_method_of_table_end); + __used __section("__cpu_method_of_table_end"); static int __init set_smp_ops_by_method(struct device_node *node) diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index a4d1b5f771f6..0ac3e06a2118 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -79,7 +79,7 @@ static inline u32 cache_type_cwg(void) return (read_cpuid_cachetype() >> CTR_CWG_SHIFT) & CTR_CWG_MASK; } -#define __read_mostly __section(.data..read_mostly) +#define __read_mostly __section(".data..read_mostly") static inline int cache_line_size_of_cpu(void) { diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index d0cf596db82c..fa02efb28e88 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -54,7 +54,7 @@ static __init pteval_t create_mapping_protection(efi_memory_desc_t *md) } /* we will fill this structure from the stub, so don't put it in .bss */ -struct screen_info screen_info __section(.data); +struct screen_info screen_info __section(".data"); int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 5892e79fa429..056772c26098 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -19,7 +19,7 @@ #include extern void secondary_holding_pen(void); -volatile unsigned long __section(.mmuoff.data.read) +volatile unsigned long __section(".mmuoff.data.read") secondary_holding_pen_release = INVALID_HWID; static phys_addr_t cpu_release_addr[NR_CPUS]; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index beff3ad8c7f8..1c0f3e02f731 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -43,7 +43,7 @@ u64 idmap_t0sz = TCR_T0SZ(VA_BITS); u64 idmap_ptrs_per_pgd = PTRS_PER_PGD; -u64 __section(.mmuoff.data.write) vabits_actual; +u64 __section(".mmuoff.data.write") vabits_actual; EXPORT_SYMBOL(vabits_actual); u64 kimage_voffset __ro_after_init; diff --git a/arch/csky/include/asm/tcm.h b/arch/csky/include/asm/tcm.h index 2b135cefb73f..bd1e662ecdfa 100644 --- a/arch/csky/include/asm/tcm.h +++ b/arch/csky/include/asm/tcm.h @@ -10,13 +10,13 @@ #include /* Tag variables with this */ -#define __tcmdata __section(.tcm.data) +#define __tcmdata __section(".tcm.data") /* Tag constants with this */ -#define __tcmconst __section(.tcm.rodata) +#define __tcmconst __section(".tcm.rodata") /* Tag functions inside TCM called from outside TCM with this */ -#define __tcmfunc __section(.tcm.text) noinline +#define __tcmfunc __section(".tcm.text") noinline /* Tag function inside TCM called from inside TCM with this */ -#define __tcmlocalfunc __section(.tcm.text) +#define __tcmlocalfunc __section(".tcm.text") void *tcm_alloc(size_t len); void tcm_free(void *addr, size_t len); diff --git a/arch/ia64/include/asm/cache.h b/arch/ia64/include/asm/cache.h index 4eb6f742d14f..2f1c70647068 100644 --- a/arch/ia64/include/asm/cache.h +++ b/arch/ia64/include/asm/cache.h @@ -25,6 +25,6 @@ # define SMP_CACHE_BYTES (1 << 3) #endif -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #endif /* _ASM_IA64_CACHE_H */ diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c index 2310daff1f8a..333b09658ca8 100644 --- a/arch/microblaze/kernel/setup.c +++ b/arch/microblaze/kernel/setup.c @@ -46,7 +46,7 @@ DEFINE_PER_CPU(unsigned int, CURRENT_SAVE); /* Saved current pointer */ * ASM code. Default position is BSS section which is cleared * in machine_early_init(). */ -char cmd_line[COMMAND_LINE_SIZE] __attribute__ ((section(".data"))); +char cmd_line[COMMAND_LINE_SIZE] __section(".data"); void __init setup_arch(char **cmdline_p) { diff --git a/arch/mips/include/asm/cache.h b/arch/mips/include/asm/cache.h index 8b14c2706aa5..29187e12b861 100644 --- a/arch/mips/include/asm/cache.h +++ b/arch/mips/include/asm/cache.h @@ -14,6 +14,6 @@ #define L1_CACHE_SHIFT CONFIG_MIPS_L1_CACHE_SHIFT #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #endif /* _ASM_CACHE_H */ diff --git a/arch/mips/include/asm/machine.h b/arch/mips/include/asm/machine.h index 29ca344a8cab..fc64cce270f0 100644 --- a/arch/mips/include/asm/machine.h +++ b/arch/mips/include/asm/machine.h @@ -23,7 +23,7 @@ extern long __mips_machines_end; #define MIPS_MACHINE(name) \ static const struct mips_machine __mips_mach_##name \ - __used __section(.mips.machines.init) + __used __section(".mips.machines.init") #define for_each_mips_machine(mach) \ for ((mach) = (struct mips_machine *)&__mips_machines_start; \ diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index fccdbe2e7c2b..0d4253208bde 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -44,7 +44,7 @@ #include #ifdef CONFIG_MIPS_ELF_APPENDED_DTB -const char __section(.appended_dtb) __appended_dtb[0x100000]; +const char __section(".appended_dtb") __appended_dtb[0x100000]; #endif /* CONFIG_MIPS_ELF_APPENDED_DTB */ struct cpuinfo_mips cpu_data[NR_CPUS] __read_mostly; diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 6c7bbfe35ba3..07e84a774938 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -569,7 +569,7 @@ unsigned long pgd_current[NR_CPUS]; * size, and waste space. So we place it in its own section and align * it in the linker script. */ -pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir); +pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); #ifndef __PAGETABLE_PUD_FOLDED pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss; #endif diff --git a/arch/parisc/include/asm/cache.h b/arch/parisc/include/asm/cache.h index e5de3f897633..d53e9e27dba0 100644 --- a/arch/parisc/include/asm/cache.h +++ b/arch/parisc/include/asm/cache.h @@ -22,7 +22,7 @@ #define ARCH_DMA_MINALIGN L1_CACHE_BYTES -#define __read_mostly __section(.data..read_mostly) +#define __read_mostly __section(".data..read_mostly") void parisc_cache_init(void); /* initializes cache-flushing */ void disable_sr_hashing_asm(int); /* low level support for above */ diff --git a/arch/parisc/include/asm/ldcw.h b/arch/parisc/include/asm/ldcw.h index e080143e79a3..6d28b5514699 100644 --- a/arch/parisc/include/asm/ldcw.h +++ b/arch/parisc/include/asm/ldcw.h @@ -52,7 +52,7 @@ }) #ifdef CONFIG_SMP -# define __lock_aligned __section(.data..lock_aligned) +# define __lock_aligned __section(".data..lock_aligned") #endif #endif /* __PARISC_LDCW_H */ diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c index 4bab21c71055..63e3ecb9da81 100644 --- a/arch/parisc/kernel/ftrace.c +++ b/arch/parisc/kernel/ftrace.c @@ -21,7 +21,7 @@ #include #include -#define __hot __attribute__ ((__section__ (".text.hot"))) +#define __hot __section(".text.hot") #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index 4381b65ae1e0..3ec633b11b54 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -42,11 +42,11 @@ extern void parisc_kernel_start(void); /* Kernel entry point in head.S */ * guarantee that global objects will be laid out in memory in the same order * as the order of declaration, so put these in different sections and use * the linker script to order them. */ -pmd_t pmd0[PTRS_PER_PMD] __attribute__ ((__section__ (".data..vm0.pmd"), aligned(PAGE_SIZE))); +pmd_t pmd0[PTRS_PER_PMD] __section(".data..vm0.pmd") __attribute__ ((aligned(PAGE_SIZE))); #endif -pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__ ((__section__ (".data..vm0.pgd"), aligned(PAGE_SIZE))); -pte_t pg0[PT_INITIAL * PTRS_PER_PTE] __attribute__ ((__section__ (".data..vm0.pte"), aligned(PAGE_SIZE))); +pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".data..vm0.pgd") __attribute__ ((aligned(PAGE_SIZE))); +pte_t pg0[PT_INITIAL * PTRS_PER_PTE] __section(".data..vm0.pte") __attribute__ ((aligned(PAGE_SIZE))); static struct resource data_resource = { .name = "Kernel data", diff --git a/arch/powerpc/include/asm/cache.h b/arch/powerpc/include/asm/cache.h index 2124b7090db9..ae0a68a838e8 100644 --- a/arch/powerpc/include/asm/cache.h +++ b/arch/powerpc/include/asm/cache.h @@ -97,7 +97,7 @@ static inline u32 l1_icache_bytes(void) #endif -#define __read_mostly __section(.data..read_mostly) +#define __read_mostly __section(".data..read_mostly") #ifdef CONFIG_PPC_BOOK3S_32 extern long _get_L2CR(void); diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 95081078aa8a..475687f24f4a 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -232,7 +232,7 @@ extern void book3e_idle(void); extern struct machdep_calls ppc_md; extern struct machdep_calls *machine_id; -#define __machine_desc __attribute__ ((__section__ (".machine.desc"))) +#define __machine_desc __section(".machine.desc") #define define_machine(name) \ extern struct machdep_calls mach_##name; \ diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c index c22a8e0dbc93..803c2a45b22a 100644 --- a/arch/powerpc/kernel/btext.c +++ b/arch/powerpc/kernel/btext.c @@ -26,7 +26,7 @@ static void scrollscreen(void); #endif -#define __force_data __section(.data) +#define __force_data __section(".data") static int g_loc_X __force_data; static int g_loc_Y __force_data; diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 5090a5ab54e5..38ae5933d917 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -45,7 +45,7 @@ #include /* All of prom_init bss lives here */ -#define __prombss __section(.bss.prominit) +#define __prombss __section(".bss.prominit") /* * Eventually bump that one up diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index 470e7c518a10..083a4e037718 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -32,7 +32,7 @@ #ifdef CONFIG_BUG #define WARN_ON_ONCE_RM(condition) ({ \ - static bool __section(.data.unlikely) __warned; \ + static bool __section(".data.unlikely") __warned; \ int __ret_warn_once = !!(condition); \ \ if (unlikely(__ret_warn_once && !__warned)) { \ diff --git a/arch/riscv/include/asm/soc.h b/arch/riscv/include/asm/soc.h index 136a442ef876..6c8363b1f327 100644 --- a/arch/riscv/include/asm/soc.h +++ b/arch/riscv/include/asm/soc.h @@ -13,7 +13,7 @@ #define SOC_EARLY_INIT_DECLARE(name, compat, fn) \ static const struct of_device_id __soc_early_init__##name \ - __used __section(__soc_early_init_table) \ + __used __section("__soc_early_init_table") \ = { .compatible = compat, .data = fn } void soc_early_init(void); @@ -46,7 +46,7 @@ struct soc_builtin_dtb { } \ \ static const struct soc_builtin_dtb __soc_builtin_dtb__##name \ - __used __section(__soc_builtin_dtb_table) = \ + __used __section("__soc_builtin_dtb_table") = \ { \ .vendor_id = vendor, \ .arch_id = arch, \ diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c index 0ec22354018c..1985884fe829 100644 --- a/arch/riscv/kernel/cpu_ops.c +++ b/arch/riscv/kernel/cpu_ops.c @@ -15,8 +15,8 @@ const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; -void *__cpu_up_stack_pointer[NR_CPUS] __section(.data); -void *__cpu_up_task_pointer[NR_CPUS] __section(.data); +void *__cpu_up_stack_pointer[NR_CPUS] __section(".data"); +void *__cpu_up_task_pointer[NR_CPUS] __section(".data"); extern const struct cpu_operations cpu_ops_sbi; extern const struct cpu_operations cpu_ops_spinwait; diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 4c96ac198e14..c424cc6dd833 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -32,7 +32,7 @@ #include "head.h" #if defined(CONFIG_DUMMY_CONSOLE) || defined(CONFIG_EFI) -struct screen_info screen_info __section(.data) = { +struct screen_info screen_info __section(".data") = { .orig_video_lines = 30, .orig_video_cols = 80, .orig_video_mode = 0, @@ -47,7 +47,7 @@ struct screen_info screen_info __section(.data) = { * This is used before the kernel initializes the BSS so it can't be in the * BSS. */ -atomic_t hart_lottery __section(.sdata); +atomic_t hart_lottery __section(".sdata"); unsigned long boot_cpu_hartid; static DEFINE_PER_CPU(struct cpu, cpu_devices); diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 90842936545b..cc96b04cc0ba 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -46,7 +46,7 @@ struct diag_ops __bootdata_preserved(diag_dma_ops) = { .diag0c = _diag0c_dma, .diag308_reset = _diag308_reset_dma }; -static struct diag210 _diag210_tmp_dma __section(.dma.data); +static struct diag210 _diag210_tmp_dma __section(".dma.data"); struct diag210 *__bootdata_preserved(__diag210_tmp_dma) = &_diag210_tmp_dma; void error(char *x) diff --git a/arch/s390/include/asm/cache.h b/arch/s390/include/asm/cache.h index d5e22e837416..00128174c025 100644 --- a/arch/s390/include/asm/cache.h +++ b/arch/s390/include/asm/cache.h @@ -14,6 +14,6 @@ #define L1_CACHE_SHIFT 8 #define NET_SKB_PAD 32 -#define __read_mostly __section(.data..read_mostly) +#define __read_mostly __section(".data..read_mostly") #endif diff --git a/arch/s390/include/asm/sections.h b/arch/s390/include/asm/sections.h index 42de04ad9c07..a996d3990a02 100644 --- a/arch/s390/include/asm/sections.h +++ b/arch/s390/include/asm/sections.h @@ -26,14 +26,14 @@ static inline int arch_is_kernel_initmem_freed(unsigned long addr) * final .boot.data section, which should be identical in the decompressor and * the decompressed kernel (that is checked during the build). */ -#define __bootdata(var) __section(.boot.data.var) var +#define __bootdata(var) __section(".boot.data.var") var /* * .boot.preserved.data is similar to .boot.data, but it is not part of the * .init section and thus will be preserved for later use in the decompressed * kernel. */ -#define __bootdata_preserved(var) __section(.boot.preserved.data.var) var +#define __bootdata_preserved(var) __section(".boot.preserved.data.var") var extern unsigned long __sdma, __edma; extern unsigned long __stext_dma, __etext_dma; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 284939f9661c..77767850d0d0 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -48,7 +48,7 @@ #include #include -pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir); +pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); diff --git a/arch/sh/boards/of-generic.c b/arch/sh/boards/of-generic.c index d91065e81a4e..bffbe69b2236 100644 --- a/arch/sh/boards/of-generic.c +++ b/arch/sh/boards/of-generic.c @@ -49,7 +49,7 @@ static struct plat_smp_ops dummy_smp_ops = { extern const struct of_cpu_method __cpu_method_of_table[]; const struct of_cpu_method __cpu_method_of_table_sentinel - __section(__cpu_method_of_table_end); + __section("__cpu_method_of_table_end"); static void sh_of_smp_probe(void) { diff --git a/arch/sh/include/asm/cache.h b/arch/sh/include/asm/cache.h index a293343456af..32dfa6b82ec6 100644 --- a/arch/sh/include/asm/cache.h +++ b/arch/sh/include/asm/cache.h @@ -14,7 +14,7 @@ #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #ifndef __ASSEMBLY__ struct cache_info { diff --git a/arch/sh/include/asm/machvec.h b/arch/sh/include/asm/machvec.h index f7d05546beca..2b4b085e8f21 100644 --- a/arch/sh/include/asm/machvec.h +++ b/arch/sh/include/asm/machvec.h @@ -36,6 +36,6 @@ extern struct sh_machine_vector sh_mv; #define get_system_type() sh_mv.mv_name #define __initmv \ - __used __section(.machvec.init) + __used __section(".machvec.init") #endif /* _ASM_SH_MACHVEC_H */ diff --git a/arch/sh/include/asm/smp.h b/arch/sh/include/asm/smp.h index 100bf241340b..199381f77293 100644 --- a/arch/sh/include/asm/smp.h +++ b/arch/sh/include/asm/smp.h @@ -71,7 +71,7 @@ struct of_cpu_method { #define CPU_METHOD_OF_DECLARE(name, _method, _ops) \ static const struct of_cpu_method __cpu_method_of_table_##name \ - __used __section(__cpu_method_of_table) \ + __used __section("__cpu_method_of_table") \ = { .method = _method, .ops = _ops } #else diff --git a/arch/sparc/include/asm/cache.h b/arch/sparc/include/asm/cache.h index dcfd58118c11..e62fd0e72606 100644 --- a/arch/sparc/include/asm/cache.h +++ b/arch/sparc/include/asm/cache.h @@ -21,6 +21,6 @@ #define SMP_CACHE_BYTES (1 << SMP_CACHE_BYTES_SHIFT) -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #endif /* !(_SPARC_CACHE_H) */ diff --git a/arch/sparc/kernel/btext.c b/arch/sparc/kernel/btext.c index 5869773f3dc4..e2d3f0d2971f 100644 --- a/arch/sparc/kernel/btext.c +++ b/arch/sparc/kernel/btext.c @@ -24,7 +24,7 @@ static void draw_byte_32(unsigned char *bits, unsigned int *base, int rb); static void draw_byte_16(unsigned char *bits, unsigned int *base, int rb); static void draw_byte_8(unsigned char *bits, unsigned int *base, int rb); -#define __force_data __attribute__((__section__(".data"))) +#define __force_data __section(".data") static int g_loc_X __force_data; static int g_loc_Y __force_data; diff --git a/arch/um/include/shared/init.h b/arch/um/include/shared/init.h index c66de434a983..1a659e2e8cc3 100644 --- a/arch/um/include/shared/init.h +++ b/arch/um/include/shared/init.h @@ -45,15 +45,15 @@ typedef void (*exitcall_t)(void); /* These are for everybody (although not all archs will actually discard it in modules) */ -#define __init __section(.init.text) -#define __initdata __section(.init.data) -#define __exitdata __section(.exit.data) -#define __exit_call __used __section(.exitcall.exit) +#define __init __section(".init.text") +#define __initdata __section(".init.data") +#define __exitdata __section(".exit.data") +#define __exit_call __used __section(".exitcall.exit") #ifdef MODULE -#define __exit __section(.exit.text) +#define __exit __section(".exit.text") #else -#define __exit __used __section(.exit.text) +#define __exit __used __section(".exit.text") #endif #endif @@ -102,10 +102,10 @@ extern struct uml_param __uml_setup_start, __uml_setup_end; * Mark functions and data as being only used at initialization * or exit time. */ -#define __uml_init_setup __used __section(.uml.setup.init) -#define __uml_setup_help __used __section(.uml.help.init) -#define __uml_postsetup_call __used __section(.uml.postsetup.init) -#define __uml_exit_call __used __section(.uml.exitcall.exit) +#define __uml_init_setup __used __section(".uml.setup.init") +#define __uml_setup_help __used __section(".uml.help.init") +#define __uml_postsetup_call __used __section(".uml.postsetup.init") +#define __uml_exit_call __used __section(".uml.exitcall.exit") #ifdef __UM_HOST__ @@ -120,7 +120,7 @@ extern struct uml_param __uml_setup_start, __uml_setup_end; #define __exitcall(fn) static exitcall_t __exitcall_##fn __exit_call = fn -#define __init_call __used __section(.initcall.init) +#define __init_call __used __section(".initcall.init") #endif diff --git a/arch/um/kernel/skas/clone.c b/arch/um/kernel/skas/clone.c index bfb70c456b30..95c355181dcd 100644 --- a/arch/um/kernel/skas/clone.c +++ b/arch/um/kernel/skas/clone.c @@ -21,7 +21,7 @@ * on some systems. */ -void __attribute__ ((__section__ (".__syscall_stub"))) +void __section(".__syscall_stub") stub_clone_handler(void) { struct stub_data *data = (struct stub_data *) STUB_DATA; diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 00141e70de56..76b37297b7d4 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -52,7 +52,7 @@ struct cpuinfo_um boot_cpu_data = { }; union thread_union cpu0_irqstack - __attribute__((__section__(".data..init_irqstack"))) = + __section(".data..init_irqstack") = { .thread_info = INIT_THREAD_INFO(init_task) }; /* Changed in setup_arch, which is called in early boot */ diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 5def1674d6f1..2a78746f5a4c 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -10,9 +10,9 @@ #ifdef CONFIG_X86_5LEVEL /* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */ -unsigned int __section(.data) __pgtable_l5_enabled; -unsigned int __section(.data) pgdir_shift = 39; -unsigned int __section(.data) ptrs_per_p4d = 1; +unsigned int __section(".data") __pgtable_l5_enabled; +unsigned int __section(".data") pgdir_shift = 39; +unsigned int __section(".data") ptrs_per_p4d = 1; #endif struct paging_config { @@ -30,7 +30,7 @@ static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; * Avoid putting the pointer into .bss as it will be cleared between * paging_prepare() and extract_kernel(). */ -unsigned long *trampoline_32bit __section(.data); +unsigned long *trampoline_32bit __section(".data"); extern struct boot_params *boot_params; int cmdline_find_option_bool(const char *option); diff --git a/arch/x86/boot/tty.c b/arch/x86/boot/tty.c index 1fedabdb95ad..f7eb976b0a4b 100644 --- a/arch/x86/boot/tty.c +++ b/arch/x86/boot/tty.c @@ -25,7 +25,7 @@ int early_serial_base; * error during initialization. */ -static void __attribute__((section(".inittext"))) serial_putchar(int ch) +static void __section(".inittext") serial_putchar(int ch) { unsigned timeout = 0xffff; @@ -35,7 +35,7 @@ static void __attribute__((section(".inittext"))) serial_putchar(int ch) outb(ch, early_serial_base + TXR); } -static void __attribute__((section(".inittext"))) bios_putchar(int ch) +static void __section(".inittext") bios_putchar(int ch) { struct biosregs ireg; @@ -47,7 +47,7 @@ static void __attribute__((section(".inittext"))) bios_putchar(int ch) intcall(0x10, &ireg, NULL); } -void __attribute__((section(".inittext"))) putchar(int ch) +void __section(".inittext") putchar(int ch) { if (ch == '\n') putchar('\r'); /* \n -> \r\n */ @@ -58,7 +58,7 @@ void __attribute__((section(".inittext"))) putchar(int ch) serial_putchar(ch); } -void __attribute__((section(".inittext"))) puts(const char *str) +void __section(".inittext") puts(const char *str) { while (*str) putchar(*str++); diff --git a/arch/x86/boot/video.h b/arch/x86/boot/video.h index cbf7fed22441..04bde0bb2003 100644 --- a/arch/x86/boot/video.h +++ b/arch/x86/boot/video.h @@ -78,7 +78,7 @@ struct card_info { u16 xmode_n; /* Size of unprobed mode range */ }; -#define __videocard struct card_info __attribute__((used,section(".videocards"))) +#define __videocard struct card_info __section(".videocards") __attribute__((used)) extern struct card_info video_cards[], video_cards_end[]; int mode_defined(u16 mode); /* video.c */ diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 1c129abb7f09..4e3099d9ae62 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -374,12 +374,12 @@ extern struct apic *apic; #define apic_driver(sym) \ static const struct apic *__apicdrivers_##sym __used \ __aligned(sizeof(struct apic *)) \ - __section(.apicdrivers) = { &sym } + __section(".apicdrivers") = { &sym } #define apic_drivers(sym1, sym2) \ static struct apic *__apicdrivers_##sym1##sym2[2] __used \ __aligned(sizeof(struct apic *)) \ - __section(.apicdrivers) = { &sym1, &sym2 } + __section(".apicdrivers") = { &sym1, &sym2 } extern struct apic *__apicdrivers[], *__apicdrivers_end[]; diff --git a/arch/x86/include/asm/cache.h b/arch/x86/include/asm/cache.h index abe08690a887..69404eae9983 100644 --- a/arch/x86/include/asm/cache.h +++ b/arch/x86/include/asm/cache.h @@ -8,7 +8,7 @@ #define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT) #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #define INTERNODE_CACHE_SHIFT CONFIG_X86_INTERNODE_CACHE_SHIFT #define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT) diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index de58391bdee0..cf0e25f45422 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -43,7 +43,7 @@ struct devs_id { #define sfi_device(i) \ static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ - __attribute__((__section__(".x86_intel_mid_dev.init"))) = &i + __section(".x86_intel_mid_dev.init") = &i /** * struct mid_sd_board_info - template for SD device creation diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 02a0cf547d7b..2dfc8d380dab 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -9,7 +9,7 @@ #include /* Provide __cpuidle; we can't safely include */ -#define __cpuidle __attribute__((__section__(".cpuidle.text"))) +#define __cpuidle __section(".cpuidle.text") /* * Interrupt control: diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index c9f5df0a1c10..2f62bbdd9d12 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -54,7 +54,7 @@ bool sme_active(void); bool sev_active(void); bool sev_es_active(void); -#define __bss_decrypted __attribute__((__section__(".bss..decrypted"))) +#define __bss_decrypted __section(".bss..decrypted") #else /* !CONFIG_AMD_MEM_ENCRYPT */ diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 7d7a064af6ff..389d851a02c4 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -119,7 +119,7 @@ void *extend_brk(size_t size, size_t align); * executable.) */ #define RESERVE_BRK(name,sz) \ - static void __section(.discard.text) __used notrace \ + static void __section(".discard.text") __used notrace \ __brk_reservation_fn_##name##__(void) { \ asm volatile ( \ ".pushsection .brk_reservation,\"aw\",@nobits;" \ diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 9d033693519a..67944128876d 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -38,7 +38,7 @@ struct _tlb_table { #define cpu_dev_register(cpu_devX) \ static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \ - __attribute__((__section__(".x86_cpu_dev.init"))) = \ + __section(".x86_cpu_dev.init") = \ &cpu_devX; extern const struct cpu_dev *const __x86_cpu_dev_start[], diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 4199f25c0063..05e117137b45 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -84,7 +84,7 @@ static struct desc_ptr startup_gdt_descr = { .address = 0, }; -#define __head __section(.head.text) +#define __head __section(".head.text") static void __head *fixup_pointer(void *ptr, unsigned long physaddr) { diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index ebb7edc8bc0a..efbb3de472df 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -37,13 +37,13 @@ * reside in the .data section so as not to be zeroed out when the .bss * section is later cleared. */ -u64 sme_me_mask __section(.data) = 0; -u64 sev_status __section(.data) = 0; +u64 sme_me_mask __section(".data") = 0; +u64 sev_status __section(".data") = 0; EXPORT_SYMBOL(sme_me_mask); DEFINE_STATIC_KEY_FALSE(sev_enable_key); EXPORT_SYMBOL_GPL(sev_enable_key); -bool sev_enabled __section(.data); +bool sev_enabled __section(".data"); /* Buffer used for early in-place encryption by BSP, no locking needed */ static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE); diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 68d75379e06a..733b983f3a89 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -81,7 +81,7 @@ struct sme_populate_pgd_data { * section is 2MB aligned to allow for simple pagetable setup using only * PMD entries (see vmlinux.lds.S). */ -static char sme_workarea[2 * PMD_PAGE_SIZE] __section(.init.scratch); +static char sme_workarea[2 * PMD_PAGE_SIZE] __section(".init.scratch"); static char sme_cmdline_arg[] __initdata = "mem_encrypt"; static char sme_cmdline_on[] __initdata = "on"; diff --git a/arch/x86/platform/pvh/enlighten.c b/arch/x86/platform/pvh/enlighten.c index c0a502f7e3a7..9ac7457f52a3 100644 --- a/arch/x86/platform/pvh/enlighten.c +++ b/arch/x86/platform/pvh/enlighten.c @@ -19,8 +19,8 @@ * pvh_bootparams and pvh_start_info need to live in the data segment since * they are used after startup_{32|64}, which clear .bss, are invoked. */ -struct boot_params pvh_bootparams __attribute__((section(".data"))); -struct hvm_start_info pvh_start_info __attribute__((section(".data"))); +struct boot_params pvh_bootparams __section(".data"); +struct hvm_start_info pvh_start_info __section(".data"); unsigned int pvh_start_info_sz = sizeof(pvh_start_info); diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index 2961234d0795..7b37a412f829 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -14,9 +14,9 @@ #include "../boot/string.h" -u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE] __section(.kexec-purgatory); +u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE] __section(".kexec-purgatory"); -struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(.kexec-purgatory); +struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(".kexec-purgatory"); static int verify_sha256_digest(void) { diff --git a/arch/x86/um/stub_segv.c b/arch/x86/um/stub_segv.c index 27361cbb7ca9..fdcd58af707a 100644 --- a/arch/x86/um/stub_segv.c +++ b/arch/x86/um/stub_segv.c @@ -8,7 +8,7 @@ #include #include -void __attribute__ ((__section__ (".__syscall_stub"))) +void __section(".__syscall_stub") stub_segv_handler(int sig, siginfo_t *info, void *p) { ucontext_t *uc = p; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 205b1176084f..aa9f50fccc5d 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -71,7 +71,7 @@ EXPORT_SYMBOL_GPL(xen_have_vector_callback); * NB: needs to live in .data because it's used by xen_prepare_pvh which runs * before clearing the bss. */ -uint32_t xen_start_flags __attribute__((section(".data"))) = 0; +uint32_t xen_start_flags __section(".data") = 0; EXPORT_SYMBOL(xen_start_flags); /* diff --git a/arch/x86/xen/enlighten_pvh.c b/arch/x86/xen/enlighten_pvh.c index 80a79db72fcf..0d5e34b9e6f9 100644 --- a/arch/x86/xen/enlighten_pvh.c +++ b/arch/x86/xen/enlighten_pvh.c @@ -21,7 +21,7 @@ * The variable xen_pvh needs to live in the data segment since it is used * after startup_{32|64} is invoked, which will clear the .bss segment. */ -bool xen_pvh __attribute__((section(".data"))) = 0; +bool xen_pvh __section(".data") = 0; void __init xen_pvh_init(struct boot_params *boot_params) { diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c index be2c78f71695..ed184106e4cf 100644 --- a/arch/xtensa/kernel/setup.c +++ b/arch/xtensa/kernel/setup.c @@ -93,7 +93,7 @@ typedef struct tagtable { } tagtable_t; #define __tagtable(tag, fn) static tagtable_t __tagtable_##fn \ - __attribute__((used, section(".taglist"))) = { tag, fn } + __section(".taglist") __attribute__((used)) = { tag, fn } /* parse current tag */ diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 0a9261a099bd..f83dac54ed85 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -4363,7 +4363,7 @@ struct of_clk_provider { extern struct of_device_id __clk_of_table; static const struct of_device_id __clk_of_table_sentinel - __used __section(__clk_of_table_end); + __used __section("__clk_of_table_end"); static LIST_HEAD(of_clk_providers); static DEFINE_MUTEX(of_clk_mutex); diff --git a/drivers/clocksource/timer-probe.c b/drivers/clocksource/timer-probe.c index ee9574da53c0..b7860bc0db4b 100644 --- a/drivers/clocksource/timer-probe.c +++ b/drivers/clocksource/timer-probe.c @@ -11,7 +11,7 @@ extern struct of_device_id __timer_of_table[]; static const struct of_device_id __timer_of_table_sentinel - __used __section(__timer_of_table_end); + __used __section("__timer_of_table_end"); void __init timer_probe(void) { diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index d2341153e181..3570f0a588c4 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -22,7 +22,7 @@ * special section. */ static const struct of_device_id -irqchip_of_match_end __used __section(__irqchip_of_table_end); +irqchip_of_match_end __used __section("__irqchip_of_table_end"); extern struct of_device_id __irqchip_of_table[]; diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 46b9371c8a33..bcd154485972 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -162,7 +162,7 @@ static int __init __reserved_mem_alloc_size(unsigned long node, } static const struct of_device_id __rmem_of_table_sentinel - __used __section(__reservedmem_of_table_end); + __used __section("__reservedmem_of_table_end"); /** * __reserved_mem_init_node() - call region specific reserved memory init code diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index 764c2de31771..681209db42a8 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -34,7 +34,7 @@ extern struct thermal_governor *__governor_thermal_table_end[]; #define THERMAL_TABLE_ENTRY(table, name) \ static typeof(name) *__thermal_table_entry_##name \ - __used __section(__##table##_thermal_table) = &name + __used __section("__" #table "_thermal_table") = &name #define THERMAL_GOVERNOR_DECLARE(name) THERMAL_TABLE_ENTRY(governor, name) diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h index 4d9bd6bb63ca..3c392b1512ac 100644 --- a/fs/xfs/xfs_message.h +++ b/fs/xfs/xfs_message.h @@ -42,7 +42,7 @@ do { \ #define xfs_printk_once(func, dev, fmt, ...) \ ({ \ - static bool __section(.data.once) __print_once; \ + static bool __section(".data.once") __print_once; \ bool __ret_print_once = !__print_once; \ \ if (!__print_once) { \ diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 18b0f4eee8cb..76a10e0dca9f 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -141,7 +141,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #ifndef WARN_ON_ONCE #define WARN_ON_ONCE(condition) ({ \ - static bool __section(.data.once) __warned; \ + static bool __section(".data.once") __warned; \ int __ret_warn_once = !!(condition); \ \ if (unlikely(__ret_warn_once && !__warned)) { \ @@ -153,7 +153,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #endif #define WARN_ONCE(condition, format...) ({ \ - static bool __section(.data.once) __warned; \ + static bool __section(".data.once") __warned; \ int __ret_warn_once = !!(condition); \ \ if (unlikely(__ret_warn_once && !__warned)) { \ @@ -164,7 +164,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, }) #define WARN_TAINT_ONCE(condition, taint, format...) ({ \ - static bool __section(.data.once) __warned; \ + static bool __section(".data.once") __warned; \ int __ret_warn_once = !!(condition); \ \ if (unlikely(__ret_warn_once && !__warned)) { \ diff --git a/include/asm-generic/error-injection.h b/include/asm-generic/error-injection.h index 80ca61058dd2..7ddd9dc10ce9 100644 --- a/include/asm-generic/error-injection.h +++ b/include/asm-generic/error-injection.h @@ -25,7 +25,7 @@ struct pt_regs; */ #define ALLOW_ERROR_INJECTION(fname, _etype) \ static struct error_injection_entry __used \ - __attribute__((__section__("_error_injection_whitelist"))) \ + __section("_error_injection_whitelist") \ _eil_addr_##fname = { \ .addr = (unsigned long)fname, \ .etype = EI_ETYPE_##_etype, \ diff --git a/include/asm-generic/kprobes.h b/include/asm-generic/kprobes.h index 4a982089c95c..060eab094e5a 100644 --- a/include/asm-generic/kprobes.h +++ b/include/asm-generic/kprobes.h @@ -10,11 +10,11 @@ */ # define __NOKPROBE_SYMBOL(fname) \ static unsigned long __used \ - __attribute__((__section__("_kprobe_blacklist"))) \ + __section("_kprobe_blacklist") \ _kbl_addr_##fname = (unsigned long)fname; # define NOKPROBE_SYMBOL(fname) __NOKPROBE_SYMBOL(fname) /* Use this to forbid a kprobes attach on very low level functions */ -# define __kprobes __attribute__((__section__(".kprobes.text"))) +# define __kprobes __section(".kprobes.text") # define nokprobe_inline __always_inline #else # define NOKPROBE_SYMBOL(fname) diff --git a/include/kunit/test.h b/include/kunit/test.h index a423fffefea0..9197da792336 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -288,7 +288,7 @@ static inline int kunit_run_all_tests(void) static struct kunit_suite *unique_array[] = { __VA_ARGS__, NULL }; \ kunit_test_suites_for_module(unique_array); \ static struct kunit_suite **unique_suites \ - __used __section(.kunit_test_suites) = unique_array + __used __section(".kunit_test_suites") = unique_array /** * kunit_test_suites() - used to register one or more &struct kunit_suite diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 143c6ffce2db..39263c6b52e1 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1153,7 +1153,7 @@ struct acpi_probe_entry { #define ACPI_DECLARE_PROBE_ENTRY(table, name, table_id, subtable, \ valid, data, fn) \ static const struct acpi_probe_entry __acpi_probe_##name \ - __used __section(__##table##_acpi_probe_table) = { \ + __used __section("__" #table "_acpi_probe_table") = { \ .id = table_id, \ .type = subtable, \ .subtable_valid = valid, \ @@ -1164,7 +1164,7 @@ struct acpi_probe_entry { #define ACPI_DECLARE_SUBTABLE_PROBE_ENTRY(table, name, table_id, \ subtable, valid, data, fn) \ static const struct acpi_probe_entry __acpi_probe_##name \ - __used __section(__##table##_acpi_probe_table) = { \ + __used __section("__" #table "_acpi_probe_table") = { \ .id = table_id, \ .type = subtable, \ .subtable_valid = valid, \ diff --git a/include/linux/cache.h b/include/linux/cache.h index 1aa8009f6d06..d742c57eaee5 100644 --- a/include/linux/cache.h +++ b/include/linux/cache.h @@ -34,7 +34,7 @@ * but may get written to during init, so can't live in .rodata (via "const"). */ #ifndef __ro_after_init -#define __ro_after_init __attribute__((__section__(".data..ro_after_init"))) +#define __ro_after_init __section(".data..ro_after_init") #endif #ifndef ____cacheline_aligned diff --git a/include/linux/compiler.h b/include/linux/compiler.h index ac45f6d40d39..e512f5505dad 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -24,7 +24,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, long ______r; \ static struct ftrace_likely_data \ __aligned(4) \ - __section(_ftrace_annotated_branch) \ + __section("_ftrace_annotated_branch") \ ______f = { \ .data.func = __func__, \ .data.file = __FILE__, \ @@ -60,7 +60,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #define __trace_if_value(cond) ({ \ static struct ftrace_branch_data \ __aligned(4) \ - __section(_ftrace_branch) \ + __section("_ftrace_branch") \ __if_trace = { \ .func = __func__, \ .file = __FILE__, \ @@ -118,7 +118,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, ".popsection\n\t" /* Annotate a C jump table to allow objtool to follow the code flow */ -#define __annotate_jump_table __section(.rodata..c_jump_table) +#define __annotate_jump_table __section(".rodata..c_jump_table") #else #define annotate_reachable() @@ -206,7 +206,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, * visible to the compiler. */ #define __ADDRESSABLE(sym) \ - static void * __section(.discard.addressable) __used \ + static void * __section(".discard.addressable") __used \ __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym; /** diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index ea7b756b1c8f..b2a3f4f641a7 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -254,7 +254,7 @@ * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-section-variable-attribute * clang: https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate */ -#define __section(S) __attribute__((__section__(#S))) +#define __section(section) __attribute__((__section__(section))) /* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-unused-function-attribute diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 8aa84c052fdf..d6428aaf67e7 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -173,7 +173,7 @@ void cpu_startup_entry(enum cpuhp_state state); void cpu_idle_poll_ctrl(bool enable); /* Attach to any functions which should be considered cpuidle. */ -#define __cpuidle __attribute__((__section__(".cpuidle.text"))) +#define __cpuidle __section(".cpuidle.text") bool cpu_in_idle(unsigned long pc); diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index 8aa0c7c2608c..a57ee75342cf 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -84,7 +84,7 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) \ static struct _ddebug __aligned(8) \ - __section(__dyndbg) name = { \ + __section("__dyndbg") name = { \ .modname = KBUILD_MODNAME, \ .function = __func__, \ .filename = __FILE__, \ diff --git a/include/linux/export.h b/include/linux/export.h index 8933ff6ad23a..fceb5e855717 100644 --- a/include/linux/export.h +++ b/include/linux/export.h @@ -130,7 +130,7 @@ struct kernel_symbol { * discarded in the final link stage. */ #define __ksym_marker(sym) \ - static int __ksym_marker_##sym[0] __section(.discard.ksym) __used + static int __ksym_marker_##sym[0] __section(".discard.ksym") __used #define __EXPORT_SYMBOL(sym, sec, ns) \ __ksym_marker(sym); \ diff --git a/include/linux/firmware.h b/include/linux/firmware.h index c15acadc6cf4..84e346ae766e 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -36,7 +36,7 @@ struct builtin_fw { #define DECLARE_BUILTIN_FIRMWARE_SIZE(name, blob, size) \ static const struct builtin_fw __fw_concat(__builtin_fw,__COUNTER__) \ - __used __section(.builtin_fw) = { name, blob, size } + __used __section(".builtin_fw") = { name, blob, size } #if defined(CONFIG_FW_LOADER) || (defined(CONFIG_FW_LOADER_MODULE) && defined(MODULE)) int request_firmware(const struct firmware **fw, const char *name, diff --git a/include/linux/init.h b/include/linux/init.h index 212fc9e2f691..7b53cb3092ee 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,11 +47,11 @@ /* These are for everybody (although not all archs will actually discard it in modules) */ -#define __init __section(.init.text) __cold __latent_entropy __noinitretpoline -#define __initdata __section(.init.data) -#define __initconst __section(.init.rodata) -#define __exitdata __section(.exit.data) -#define __exit_call __used __section(.exitcall.exit) +#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline +#define __initdata __section(".init.data") +#define __initconst __section(".init.rodata") +#define __exitdata __section(".exit.data") +#define __exit_call __used __section(".exitcall.exit") /* * modpost check for section mismatches during the kernel build. @@ -70,9 +70,9 @@ * * The markers follow same syntax rules as __init / __initdata. */ -#define __ref __section(.ref.text) noinline -#define __refdata __section(.ref.data) -#define __refconst __section(.ref.rodata) +#define __ref __section(".ref.text") noinline +#define __refdata __section(".ref.data") +#define __refconst __section(".ref.rodata") #ifdef MODULE #define __exitused @@ -80,16 +80,16 @@ #define __exitused __used #endif -#define __exit __section(.exit.text) __exitused __cold notrace +#define __exit __section(".exit.text") __exitused __cold notrace /* Used for MEMORY_HOTPLUG */ -#define __meminit __section(.meminit.text) __cold notrace \ +#define __meminit __section(".meminit.text") __cold notrace \ __latent_entropy -#define __meminitdata __section(.meminit.data) -#define __meminitconst __section(.meminit.rodata) -#define __memexit __section(.memexit.text) __exitused __cold notrace -#define __memexitdata __section(.memexit.data) -#define __memexitconst __section(.memexit.rodata) +#define __meminitdata __section(".meminit.data") +#define __meminitconst __section(".meminit.rodata") +#define __memexit __section(".memexit.text") __exitused __cold notrace +#define __memexitdata __section(".memexit.data") +#define __memexitconst __section(".memexit.rodata") /* For assembly routines */ #define __HEAD .section ".head.text","ax" @@ -254,7 +254,7 @@ struct obs_kernel_param { static const char __setup_str_##unique_id[] __initconst \ __aligned(1) = str; \ static struct obs_kernel_param __setup_##unique_id \ - __used __section(.init.setup) \ + __used __section(".init.setup") \ __attribute__((aligned((sizeof(long))))) \ = { __setup_str_##unique_id, fn, early } @@ -298,7 +298,7 @@ void __init parse_early_options(char *cmdline); #endif /* Data marked not to be saved by software suspend */ -#define __nosavedata __section(.data..nosave) +#define __nosavedata __section(".data..nosave") #ifdef MODULE #define __exit_p(x) x diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 2c620d7ac432..b2412b4d4c20 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -40,12 +40,12 @@ extern struct cred init_cred; /* Attach to the init_task data structure for proper alignment */ #ifdef CONFIG_ARCH_TASK_STRUCT_ON_STACK -#define __init_task_data __attribute__((__section__(".data..init_task"))) +#define __init_task_data __section(".data..init_task") #else #define __init_task_data /**/ #endif /* Attach to the thread_info data structure for proper alignment */ -#define __init_thread_info __attribute__((__section__(".data..init_thread_info"))) +#define __init_thread_info __section(".data..init_thread_info") #endif diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index f9aee3538461..ee8299eb1f52 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -792,9 +792,9 @@ extern int arch_early_irq_init(void); * We want to know which function is an entrypoint of a hardirq or a softirq. */ #ifndef __irq_entry -# define __irq_entry __attribute__((__section__(".irqentry.text"))) +# define __irq_entry __section(".irqentry.text") #endif -#define __softirq_entry __attribute__((__section__(".softirqentry.text"))) +#define __softirq_entry __section(".softirqentry.text") #endif diff --git a/include/linux/kernel.h b/include/linux/kernel.h index c629215fdad9..2f05e9128201 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -729,7 +729,7 @@ do { \ #define do_trace_printk(fmt, args...) \ do { \ static const char *trace_printk_fmt __used \ - __attribute__((section("__trace_printk_fmt"))) = \ + __section("__trace_printk_fmt") = \ __builtin_constant_p(fmt) ? fmt : NULL; \ \ __trace_printk_check_format(fmt, ##args); \ @@ -773,7 +773,7 @@ int __trace_printk(unsigned long ip, const char *fmt, ...); #define trace_puts(str) ({ \ static const char *trace_printk_fmt __used \ - __attribute__((section("__trace_printk_fmt"))) = \ + __section("__trace_printk_fmt") = \ __builtin_constant_p(str) ? str : NULL; \ \ if (__builtin_constant_p(str)) \ @@ -795,7 +795,7 @@ extern void trace_dump_stack(int skip); do { \ if (__builtin_constant_p(fmt)) { \ static const char *trace_printk_fmt __used \ - __attribute__((section("__trace_printk_fmt"))) = \ + __section("__trace_printk_fmt") = \ __builtin_constant_p(fmt) ? fmt : NULL; \ \ __ftrace_vbprintk(_THIS_IP_, trace_printk_fmt, vargs); \ diff --git a/include/linux/linkage.h b/include/linux/linkage.h index d796ec20d114..5bcfbd972e97 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -36,8 +36,8 @@ __stringify(name)) #endif -#define __page_aligned_data __section(.data..page_aligned) __aligned(PAGE_SIZE) -#define __page_aligned_bss __section(.bss..page_aligned) __aligned(PAGE_SIZE) +#define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) +#define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) /* * For assembly routines. diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 8814e3d5952d..c503f7ab8afb 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1611,12 +1611,12 @@ extern struct lsm_info __start_early_lsm_info[], __end_early_lsm_info[]; #define DEFINE_LSM(lsm) \ static struct lsm_info __lsm_##lsm \ - __used __section(.lsm_info.init) \ + __used __section(".lsm_info.init") \ __aligned(sizeof(unsigned long)) #define DEFINE_EARLY_LSM(lsm) \ static struct lsm_info __early_lsm_##lsm \ - __used __section(.early_lsm_info.init) \ + __used __section(".early_lsm_info.init") \ __aligned(sizeof(unsigned long)) #ifdef CONFIG_SECURITY_SELINUX_DISABLE diff --git a/include/linux/module.h b/include/linux/module.h index a29187f7c360..7ccdf87f376f 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -278,7 +278,7 @@ extern typeof(name) __mod_##type##__##name##_device_table \ .version = _version, \ }; \ static const struct module_version_attribute \ - __used __attribute__ ((__section__ ("__modver"))) \ + __used __section("__modver") \ * __moduleparam_const __modver_attr = &___modver_attr #endif diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 47879fc7f75e..6388eb9734a5 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -22,7 +22,7 @@ #define __MODULE_INFO(tag, name, info) \ static const char __UNIQUE_ID(name)[] \ - __used __attribute__((section(".modinfo"), unused, aligned(1))) \ + __used __section(".modinfo") __attribute__((unused, aligned(1))) \ = __MODULE_INFO_PREFIX __stringify(tag) "=" info #define __MODULE_PARM_TYPE(name, _type) \ @@ -289,7 +289,7 @@ struct kparam_array static const char __param_str_##name[] = prefix #name; \ static struct kernel_param __moduleparam_const __param_##name \ __used \ - __attribute__ ((unused,__section__ ("__param"),aligned(sizeof(void *)))) \ + __section("__param") __attribute__ ((unused, aligned(sizeof(void *)))) \ = { __param_str_##name, THIS_MODULE, ops, \ VERIFY_OCTAL_PERMISSIONS(perm), level, flags, { arg } } diff --git a/include/linux/mtd/xip.h b/include/linux/mtd/xip.h index a4e352b1dfe6..3cac9360588f 100644 --- a/include/linux/mtd/xip.h +++ b/include/linux/mtd/xip.h @@ -28,7 +28,7 @@ * those functions so they get relocated to ram. */ #ifdef CONFIG_XIP_KERNEL -#define __xipram noinline __attribute__ ((__section__ (".xiptext"))) +#define __xipram noinline __section(".xiptext") #endif /* diff --git a/include/linux/objtool.h b/include/linux/objtool.h index ab82c793c897..577f51436cf9 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -60,7 +60,7 @@ struct unwind_hint { * For more information, see tools/objtool/Documentation/stack-validation.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ - static void __used __section(.discard.func_stack_frame_non_standard) \ + static void __used __section(".discard.func_stack_frame_non_standard") \ *__func_stack_frame_non_standard_##func = func #else /* __ASSEMBLY__ */ diff --git a/include/linux/of.h b/include/linux/of.h index 481ec0467285..5d51891cbf1a 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -1299,7 +1299,7 @@ static inline int of_get_available_child_count(const struct device_node *np) #if defined(CONFIG_OF) && !defined(MODULE) #define _OF_DECLARE(table, name, compat, fn, fn_type) \ static const struct of_device_id __of_table_##name \ - __used __section(__##table##_of_table) \ + __used __section("__" #table "_of_table") \ = { .compatible = compat, \ .data = (fn == (fn_type)NULL) ? fn : fn } #else diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 176bfbd52d97..dff7040f629a 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -51,7 +51,7 @@ PER_CPU_ATTRIBUTES #define __PCPU_DUMMY_ATTRS \ - __attribute__((section(".discard"), unused)) + __section(".discard") __attribute__((unused)) /* * s390 and alpha modules require percpu variables to be defined as diff --git a/include/linux/printk.h b/include/linux/printk.h index 78479633ccfc..fe7eb2351610 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -437,7 +437,7 @@ extern int kptr_restrict; #ifdef CONFIG_PRINTK #define printk_once(fmt, ...) \ ({ \ - static bool __section(.data.once) __print_once; \ + static bool __section(".data.once") __print_once; \ bool __ret_print_once = !__print_once; \ \ if (!__print_once) { \ @@ -448,7 +448,7 @@ extern int kptr_restrict; }) #define printk_deferred_once(fmt, ...) \ ({ \ - static bool __section(.data.once) __print_once; \ + static bool __section(".data.once") __print_once; \ bool __ret_print_once = !__print_once; \ \ if (!__print_once) { \ diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 7c1ceff02852..6cdd0152c253 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -299,7 +299,7 @@ static inline int rcu_read_lock_any_held(void) */ #define RCU_LOCKDEP_WARN(c, s) \ do { \ - static bool __section(.data.unlikely) __warned; \ + static bool __section(".data.unlikely") __warned; \ if (debug_lockdep_rcu_enabled() && !__warned && (c)) { \ __warned = true; \ lockdep_rcu_suspicious(__FILE__, __LINE__, s); \ diff --git a/include/linux/sched/debug.h b/include/linux/sched/debug.h index 00c45a0e6abe..ae51f4529fc9 100644 --- a/include/linux/sched/debug.h +++ b/include/linux/sched/debug.h @@ -43,7 +43,7 @@ extern void proc_sched_set_task(struct task_struct *p); #endif /* Attach to any functions which should be ignored in wchan output. */ -#define __sched __attribute__((__section__(".sched.text"))) +#define __sched __section(".sched.text") /* Linker adds these: start and end of __sched functions */ extern char __sched_text_start[], __sched_text_end[]; diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 8a99279a579b..ff63c2963359 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -373,7 +373,7 @@ extern const struct earlycon_id *__earlycon_table_end[]; .compatible = compat, \ .setup = fn }; \ static const struct earlycon_id EARLYCON_USED_OR_UNUSED \ - __section(__earlycon_table) \ + __section("__earlycon_table") \ * const __PASTE(__p, unique_id) = &unique_id #define OF_EARLYCON_DECLARE(_name, compat, fn) \ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index f2f12d746dbd..79897841a2cc 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -76,7 +76,7 @@ #define LOCK_SECTION_END \ ".previous\n\t" -#define __lockfunc __attribute__((section(".spinlock.text"))) +#define __lockfunc __section(".spinlock.text") /* * Pull the arch_spinlock_t and arch_rwlock_t definitions: diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 2eda7678fe1d..37bea07c12f2 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -144,7 +144,7 @@ extern struct trace_event_functions exit_syscall_print_funcs; .flags = TRACE_EVENT_FL_CAP_ANY, \ }; \ static struct trace_event_call __used \ - __attribute__((section("_ftrace_events"))) \ + __section("_ftrace_events") \ *__event_enter_##sname = &event_enter_##sname; #define SYSCALL_TRACE_EXIT_EVENT(sname) \ @@ -160,7 +160,7 @@ extern struct trace_event_functions exit_syscall_print_funcs; .flags = TRACE_EVENT_FL_CAP_ANY, \ }; \ static struct trace_event_call __used \ - __attribute__((section("_ftrace_events"))) \ + __section("_ftrace_events") \ *__event_exit_##sname = &event_exit_##sname; #define SYSCALL_METADATA(sname, nb, ...) \ @@ -184,7 +184,7 @@ extern struct trace_event_functions exit_syscall_print_funcs; .enter_fields = LIST_HEAD_INIT(__syscall_meta_##sname.enter_fields), \ }; \ static struct syscall_metadata __used \ - __attribute__((section("__syscalls_metadata"))) \ + __section("__syscalls_metadata") \ *__p_syscall_meta_##sname = &__syscall_meta_##sname; static inline int is_syscall_trace_event(struct trace_event_call *tp_event) diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 5c6943354049..d321fe5ad1a1 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -709,7 +709,7 @@ do { \ tracing_record_cmdline(current); \ if (__builtin_constant_p(fmt)) { \ static const char *trace_printk_fmt \ - __attribute__((section("__trace_printk_fmt"))) = \ + __section("__trace_printk_fmt") = \ __builtin_constant_p(fmt) ? fmt : NULL; \ \ __trace_bprintk(ip, trace_printk_fmt, ##args); \ diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 81fa0b2f271e..0f21617f1a66 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -119,7 +119,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) #define __TRACEPOINT_ENTRY(name) \ static tracepoint_ptr_t __tracepoint_ptr_##name __used \ - __section(__tracepoints_ptrs) = &__tracepoint_##name + __section("__tracepoints_ptrs") = &__tracepoint_##name #endif #endif /* _LINUX_TRACEPOINT_H */ @@ -286,11 +286,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) */ #define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \ static const char __tpstrtab_##_name[] \ - __section(__tracepoints_strings) = #_name; \ + __section("__tracepoints_strings") = #_name; \ extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \ int __traceiter_##_name(void *__data, proto); \ struct tracepoint __tracepoint_##_name __used \ - __section(__tracepoints) = { \ + __section("__tracepoints") = { \ .name = __tpstrtab_##_name, \ .key = STATIC_KEY_INIT_FALSE, \ .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \ @@ -396,7 +396,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) static const char *___tp_str __tracepoint_string = str; \ ___tp_str; \ }) -#define __tracepoint_string __used __section(__tracepoint_str) +#define __tracepoint_string __used __section("__tracepoint_str") #else /* * tracepoint_string() is used to save the string address for userspace diff --git a/include/trace/bpf_probe.h b/include/trace/bpf_probe.h index 1ce3be63add1..cd74bffed5c6 100644 --- a/include/trace/bpf_probe.h +++ b/include/trace/bpf_probe.h @@ -79,7 +79,7 @@ static union { \ struct bpf_raw_event_map event; \ btf_trace_##call handler; \ } __bpf_trace_tp_map_##call __used \ -__attribute__((section("__bpf_raw_tp_map"))) = { \ +__section("__bpf_raw_tp_map") = { \ .event = { \ .tp = &__tracepoint_##call, \ .bpf_func = __bpf_trace_##template, \ diff --git a/include/trace/trace_events.h b/include/trace/trace_events.h index 1bc3e7bba9a4..7785961d82ba 100644 --- a/include/trace/trace_events.h +++ b/include/trace/trace_events.h @@ -45,7 +45,7 @@ TRACE_MAKE_SYSTEM_STR(); .eval_value = a \ }; \ static struct trace_eval_map __used \ - __attribute__((section("_ftrace_eval_map"))) \ + __section("_ftrace_eval_map") \ *TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a #undef TRACE_DEFINE_SIZEOF @@ -58,7 +58,7 @@ TRACE_MAKE_SYSTEM_STR(); .eval_value = sizeof(a) \ }; \ static struct trace_eval_map __used \ - __attribute__((section("_ftrace_eval_map"))) \ + __section("_ftrace_eval_map") \ *TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a /* @@ -607,7 +607,7 @@ static inline notrace int trace_event_get_offsets_##call( \ * // its only safe to use pointers when doing linker tricks to * // create an array. * static struct trace_event_call __used - * __attribute__((section("_ftrace_events"))) *__event_ = &event_; + * __section("_ftrace_events") *__event_ = &event_; * */ @@ -755,7 +755,7 @@ static struct trace_event_call __used event_##call = { \ .flags = TRACE_EVENT_FL_TRACEPOINT, \ }; \ static struct trace_event_call __used \ -__attribute__((section("_ftrace_events"))) *__event_##call = &event_##call +__section("_ftrace_events") *__event_##call = &event_##call #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ @@ -772,6 +772,6 @@ static struct trace_event_call __used event_##call = { \ .flags = TRACE_EVENT_FL_TRACEPOINT, \ }; \ static struct trace_event_call __used \ -__attribute__((section("_ftrace_events"))) *__event_##call = &event_##call +__section("_ftrace_events") *__event_##call = &event_##call #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 4fb15fa96734..fe9de067771c 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -40,10 +40,10 @@ extern const u8 kallsyms_names[] __weak; * has one (eg: FRV). */ extern const unsigned int kallsyms_num_syms -__attribute__((weak, section(".rodata"))); +__section(".rodata") __attribute__((weak)); extern const unsigned long kallsyms_relative_base -__attribute__((weak, section(".rodata"))); +__section(".rodata") __attribute__((weak)); extern const char kallsyms_token_table[] __weak; extern const u16 kallsyms_token_index[] __weak; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 6d93f4518734..f232305dcefe 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2504,7 +2504,7 @@ static void prio_changed_dl(struct rq *rq, struct task_struct *p, } const struct sched_class dl_sched_class - __attribute__((section("__dl_sched_class"))) = { + __section("__dl_sched_class") = { .enqueue_task = enqueue_task_dl, .dequeue_task = dequeue_task_dl, .yield_task = yield_task_dl, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e17012be4d14..290f9e38378c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11159,7 +11159,7 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task * All the scheduling class methods: */ const struct sched_class fair_sched_class - __attribute__((section("__fair_sched_class"))) = { + __section("__fair_sched_class") = { .enqueue_task = enqueue_task_fair, .dequeue_task = dequeue_task_fair, .yield_task = yield_task_fair, diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index f324dc36fc43..24d0ee26377d 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -458,7 +458,7 @@ static void update_curr_idle(struct rq *rq) * Simple, special scheduling class for the per-CPU idle tasks: */ const struct sched_class idle_sched_class - __attribute__((section("__idle_sched_class"))) = { + __section("__idle_sched_class") = { /* no enqueue/yield_task for idle tasks */ /* dequeue is not valid, we print a debug message there: */ diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index f215eea6a966..49ec096a8aa1 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2430,7 +2430,7 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) } const struct sched_class rt_sched_class - __attribute__((section("__rt_sched_class"))) = { + __section("__rt_sched_class") = { .enqueue_task = enqueue_task_rt, .dequeue_task = dequeue_task_rt, .yield_task = yield_task_rt, diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index 394bc8126a1e..ceb5b6b12561 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -110,7 +110,7 @@ static void update_curr_stop(struct rq *rq) * Simple, special scheduling class for the per-CPU stop tasks: */ const struct sched_class stop_sched_class - __attribute__((section("__stop_sched_class"))) = { + __section("__stop_sched_class") = { .enqueue_task = enqueue_task_stop, .dequeue_task = dequeue_task_stop, diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 34e0c4d5a6e7..f3f5e77123ad 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -99,7 +99,7 @@ enum trace_type { /* Use this for memory failure errors */ #define MEM_FAIL(condition, fmt, ...) ({ \ - static bool __section(.data.once) __warned; \ + static bool __section(".data.once") __warned; \ int __ret_warn_once = !!(condition); \ \ if (unlikely(__ret_warn_once && !__warned)) { \ diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index 70d3d0a09053..90f81d33fa3f 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c @@ -176,7 +176,7 @@ struct trace_event_call __used event_##call = { \ .flags = TRACE_EVENT_FL_IGNORE_ENABLE, \ }; \ static struct trace_event_call __used \ -__attribute__((section("_ftrace_events"))) *__event_##call = &event_##call; +__section("_ftrace_events") *__event_##call = &event_##call; #undef FTRACE_ENTRY #define FTRACE_ENTRY(call, struct_name, etype, tstruct, print) \ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 69341b36f271..f882ce0d9327 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -2254,7 +2254,7 @@ static void add_header(struct buffer *b, struct module *mod) buf_printf(b, "MODULE_INFO(name, KBUILD_MODNAME);\n"); buf_printf(b, "\n"); buf_printf(b, "__visible struct module __this_module\n"); - buf_printf(b, "__section(.gnu.linkonce.this_module) = {\n"); + buf_printf(b, "__section(\".gnu.linkonce.this_module\") = {\n"); buf_printf(b, "\t.name = KBUILD_MODNAME,\n"); if (mod->has_init) buf_printf(b, "\t.init = init_module,\n"); @@ -2308,7 +2308,7 @@ static int add_versions(struct buffer *b, struct module *mod) buf_printf(b, "\n"); buf_printf(b, "static const struct modversion_info ____versions[]\n"); - buf_printf(b, "__used __section(__versions) = {\n"); + buf_printf(b, "__used __section(\"__versions\") = {\n"); for (s = mod->unres; s; s = s->next) { if (!s->module) diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h index ab82c793c897..577f51436cf9 100644 --- a/tools/include/linux/objtool.h +++ b/tools/include/linux/objtool.h @@ -60,7 +60,7 @@ struct unwind_hint { * For more information, see tools/objtool/Documentation/stack-validation.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ - static void __used __section(.discard.func_stack_frame_non_standard) \ + static void __used __section(".discard.func_stack_frame_non_standard") \ *__func_stack_frame_non_standard_##func = func #else /* __ASSEMBLY__ */ -- cgit v1.3-18-gd494