diff options
author | Rafael J. Wysocki <rafael.j.wysocki@intel.com> | 2020-10-05 13:12:02 +0200 |
---|---|---|
committer | Rafael J. Wysocki <rafael.j.wysocki@intel.com> | 2020-10-05 13:12:02 +0200 |
commit | fccd2f0e629b357437bebc015810f7f1957a4643 (patch) | |
tree | 4d0cc002fb0e0d487e66d205f115737f37933b98 | |
parent | cpufreq: intel_pstate: Fix missing return statement (diff) | |
parent | arch_topology, arm, arm64: define arch_scale_freq_invariant() (diff) | |
download | linux-dev-fccd2f0e629b357437bebc015810f7f1957a4643.tar.xz linux-dev-fccd2f0e629b357437bebc015810f7f1957a4643.zip |
Merge back cpufreq material for 5.10.
-rw-r--r-- | arch/arm/include/asm/topology.h | 1 | ||||
-rw-r--r-- | arch/arm64/include/asm/topology.h | 1 | ||||
-rw-r--r-- | arch/arm64/kernel/topology.c | 9 | ||||
-rw-r--r-- | drivers/base/arch_topology.c | 13 | ||||
-rw-r--r-- | drivers/cpufreq/cpufreq-dt.c | 10 | ||||
-rw-r--r-- | drivers/cpufreq/cpufreq.c | 33 | ||||
-rw-r--r-- | drivers/cpufreq/qcom-cpufreq-hw.c | 9 | ||||
-rw-r--r-- | drivers/cpufreq/scmi-cpufreq.c | 12 | ||||
-rw-r--r-- | drivers/cpufreq/scpi-cpufreq.c | 6 | ||||
-rw-r--r-- | drivers/cpufreq/vexpress-spc-cpufreq.c | 12 | ||||
-rw-r--r-- | include/linux/arch_topology.h | 4 | ||||
-rw-r--r-- | include/linux/cpufreq.h | 8 |
12 files changed, 68 insertions, 50 deletions
diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index e0593cf095d0..9219e67befbe 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -9,6 +9,7 @@ /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +#define arch_scale_freq_invariant topology_scale_freq_invariant /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index e042f6527981..7cb519473fbd 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -27,6 +27,7 @@ void topology_scale_freq_tick(void); /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +#define arch_scale_freq_invariant topology_scale_freq_invariant /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 0801a0f3c156..4064d39bb66d 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -246,6 +246,13 @@ static int __init init_amu_fie(void) static_branch_enable(&amu_fie_key); } + /* + * If the system is not fully invariant after AMU init, disable + * partial use of counters for frequency invariance. + */ + if (!topology_scale_freq_invariant()) + static_branch_disable(&amu_fie_key); + free_valid_mask: free_cpumask_var(valid_cpus); @@ -253,7 +260,7 @@ free_valid_mask: } late_initcall_sync(init_amu_fie); -bool arch_freq_counters_available(struct cpumask *cpus) +bool arch_freq_counters_available(const struct cpumask *cpus) { return amu_freq_invariant() && cpumask_subset(cpus, amu_fie_cpus); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 75f72d684294..89cae168b076 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -21,18 +21,27 @@ #include <linux/sched.h> #include <linux/smp.h> -__weak bool arch_freq_counters_available(struct cpumask *cpus) +bool topology_scale_freq_invariant(void) +{ + return cpufreq_supports_freq_invariance() || + arch_freq_counters_available(cpu_online_mask); +} + +__weak bool arch_freq_counters_available(const struct cpumask *cpus) { return false; } DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; -void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, +void arch_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, unsigned long max_freq) { unsigned long scale; int i; + if (WARN_ON_ONCE(!cur_freq || !max_freq)) + return; + /* * If the use of counters for FIE is enabled, just return as we don't * want to update the scale factor with information from CPUFREQ. diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 944d7b45afe9..9fd4ce774f12 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -40,16 +40,8 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) { struct private_data *priv = policy->driver_data; unsigned long freq = policy->freq_table[index].frequency; - int ret; - - ret = dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); - if (!ret) { - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - } - - return ret; + return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); } /* diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 47aa90f9a7c2..2ea245a6c0c0 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -61,6 +61,12 @@ static struct cpufreq_driver *cpufreq_driver; static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); static DEFINE_RWLOCK(cpufreq_driver_lock); +static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); +bool cpufreq_supports_freq_invariance(void) +{ + return static_branch_likely(&cpufreq_freq_invariance); +} + /* Flag to suspend/resume CPUFreq governors */ static bool cpufreq_suspended; @@ -154,8 +160,9 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) } EXPORT_SYMBOL_GPL(get_cpu_idle_time); -__weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, - unsigned long max_freq) +__weak void arch_set_freq_scale(const struct cpumask *cpus, + unsigned long cur_freq, + unsigned long max_freq) { } EXPORT_SYMBOL_GPL(arch_set_freq_scale); @@ -446,6 +453,10 @@ void cpufreq_freq_transition_end(struct cpufreq_policy *policy, cpufreq_notify_post_transition(policy, freqs, transition_failed); + arch_set_freq_scale(policy->related_cpus, + policy->cur, + policy->cpuinfo.max_freq); + policy->transition_ongoing = false; policy->transition_task = NULL; @@ -2056,9 +2067,15 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier); unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { + unsigned int freq; + target_freq = clamp_val(target_freq, policy->min, policy->max); + freq = cpufreq_driver->fast_switch(policy, target_freq); - return cpufreq_driver->fast_switch(policy, target_freq); + arch_set_freq_scale(policy->related_cpus, freq, + policy->cpuinfo.max_freq); + + return freq; } EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); @@ -2710,6 +2727,15 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); + /* + * Mark support for the scheduler's frequency invariance engine for + * drivers that implement target(), target_index() or fast_switch(). + */ + if (!cpufreq_driver->setpolicy) { + static_branch_enable_cpuslocked(&cpufreq_freq_invariance); + pr_debug("supports frequency invariance"); + } + if (driver_data->setpolicy) driver_data->flags |= CPUFREQ_CONST_LOOPS; @@ -2779,6 +2805,7 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) cpus_read_lock(); subsys_interface_unregister(&cpufreq_interface); remove_boost_sysfs_file(); + static_branch_disable_cpuslocked(&cpufreq_freq_invariance); cpuhp_remove_state_nocalls_cpuslocked(hp_online); write_lock_irqsave(&cpufreq_driver_lock, flags); diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 3fb044b907a8..aabe4306d92f 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -85,8 +85,6 @@ static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, if (icc_scaling_enabled) qcom_cpufreq_set_bw(policy, freq); - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); return 0; } @@ -113,16 +111,11 @@ static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, { void __iomem *perf_state_reg = policy->driver_data; unsigned int index; - unsigned long freq; index = policy->cached_resolved_idx; writel_relaxed(index, perf_state_reg); - freq = policy->freq_table[index].frequency; - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - - return freq; + return policy->freq_table[index].frequency; } static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index fb42e3390377..6dd1311660b5 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -48,16 +48,11 @@ static unsigned int scmi_cpufreq_get_rate(unsigned int cpu) static int scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { - int ret; struct scmi_data *priv = policy->driver_data; struct scmi_perf_ops *perf_ops = handle->perf_ops; u64 freq = policy->freq_table[index].frequency; - ret = perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false); - if (!ret) - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - return ret; + return perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false); } static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, @@ -67,11 +62,8 @@ static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, struct scmi_perf_ops *perf_ops = handle->perf_ops; if (!perf_ops->freq_set(handle, priv->domain_id, - target_freq * 1000, true)) { - arch_set_freq_scale(policy->related_cpus, target_freq, - policy->cpuinfo.max_freq); + target_freq * 1000, true)) return target_freq; - } return 0; } diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index b0f5388b8854..43db05b949d9 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -47,9 +47,8 @@ static unsigned int scpi_cpufreq_get_rate(unsigned int cpu) static int scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned long freq = policy->freq_table[index].frequency; + u64 rate = policy->freq_table[index].frequency * 1000; struct scpi_data *priv = policy->driver_data; - u64 rate = freq * 1000; int ret; ret = clk_set_rate(priv->clk, rate); @@ -60,9 +59,6 @@ scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) if (clk_get_rate(priv->clk) != rate) return -EIO; - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - return 0; } diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index 4e8b1dee7c9a..e89b905754d2 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -182,7 +182,6 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy, { u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster; unsigned int freqs_new; - int ret; cur_cluster = cpu_to_cluster(cpu); new_cluster = actual_cluster = per_cpu(physical_cluster, cpu); @@ -197,15 +196,8 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy, new_cluster = A15_CLUSTER; } - ret = ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster, - freqs_new); - - if (!ret) { - arch_set_freq_scale(policy->related_cpus, freqs_new, - policy->cpuinfo.max_freq); - } - - return ret; + return ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster, + freqs_new); } static inline u32 get_table_count(struct cpufreq_frequency_table *table) diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 69b1dabe39dc..083df331a3c9 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -30,7 +30,9 @@ static inline unsigned long topology_get_freq_scale(int cpu) return per_cpu(freq_scale, cpu); } -bool arch_freq_counters_available(struct cpumask *cpus); +bool topology_scale_freq_invariant(void); + +bool arch_freq_counters_available(const struct cpumask *cpus); DECLARE_PER_CPU(unsigned long, thermal_pressure); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index a911e5d06845..9f779fbdbe7b 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -217,6 +217,7 @@ void refresh_frequency_limits(struct cpufreq_policy *policy); void cpufreq_update_policy(unsigned int cpu); void cpufreq_update_limits(unsigned int cpu); bool have_governor_per_policy(void); +bool cpufreq_supports_freq_invariance(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); void cpufreq_enable_fast_switch(struct cpufreq_policy *policy); void cpufreq_disable_fast_switch(struct cpufreq_policy *policy); @@ -237,6 +238,10 @@ static inline unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) { return 0; } +static inline bool cpufreq_supports_freq_invariance(void) +{ + return false; +} static inline void disable_cpufreq(void) { } #endif @@ -1006,7 +1011,8 @@ static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy, extern void arch_freq_prepare_all(void); extern unsigned int arch_freq_get_on_cpu(int cpu); -extern void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, +extern void arch_set_freq_scale(const struct cpumask *cpus, + unsigned long cur_freq, unsigned long max_freq); /* the following are really really optional */ |