From 2c0d278f3293fc59da0d183075415ca1e8c93b40 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 26 Jul 2019 20:30:55 +0200 Subject: KVM: LAPIC: Mark hrtimer to expire in hard interrupt context On PREEMPT_RT enabled kernels unmarked hrtimers are moved into soft interrupt expiry mode by default. While that's not a functional requirement for the KVM local APIC timer emulation, it's a latency issue which can be avoided by marking the timer so hard interrupt context expiry is enforced. No functional change. [ tglx: Split out from larger combo patch. Add changelog. ] Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Paolo Bonzini Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185753.363363474@linutronix.de --- arch/x86/kvm/lapic.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 0aa158657f20..b9e516099d07 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1601,7 +1601,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) likely(ns > apic->lapic_timer.timer_advance_ns)) { expire = ktime_add_ns(now, ns); expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); - hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS); + hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); } else apic_timer_expired(apic); @@ -2302,7 +2302,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) apic->vcpu = vcpu; hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, - HRTIMER_MODE_ABS); + HRTIMER_MODE_ABS_HARD); apic->lapic_timer.timer.function = apic_timer_fn; if (timer_advance_ns == -1) { apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT; @@ -2487,7 +2487,7 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) timer = &vcpu->arch.apic->lapic_timer.timer; if (hrtimer_cancel(timer)) - hrtimer_start_expires(timer, HRTIMER_MODE_ABS); + hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD); } /* -- cgit v1.2.3-59-g8ed1b From adb87ff4f96c9700718e09c97a804124d5cd61ff Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Wed, 14 Aug 2019 20:32:15 +0800 Subject: clocksource/drivers/hyperv: Allocate Hyper-V TSC page statically Prepare to add Hyper-V sched clock callback and move Hyper-V Reference TSC initialization much earlier in the boot process. Earlier initialization is needed so that it happens while the timestamp value is still 0 and no discontinuity in the timestamp will occur when pv_ops.time.sched_clock calculates its offset. The earlier initialization requires that the Hyper-V TSC page be allocated statically instead of with vmalloc(), so fixup the references to the TSC page and the method of getting its physical address. Signed-off-by: Tianyu Lan Signed-off-by: Thomas Gleixner Acked-by: Daniel Lezcano Link: https://lkml.kernel.org/r/20190814123216.32245-2-Tianyu.Lan@microsoft.com --- arch/x86/entry/vdso/vma.c | 2 +- drivers/clocksource/hyperv_timer.c | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 349a61d8bf34..f5937742b290 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -122,7 +122,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK)) return vmf_insert_pfn(vma, vmf->address, - vmalloc_to_pfn(tsc_pg)); + virt_to_phys(tsc_pg) >> PAGE_SHIFT); } return VM_FAULT_SIGBUS; diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index ba2c79e6a0ee..432aa331df04 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -214,17 +214,17 @@ EXPORT_SYMBOL_GPL(hyperv_cs); #ifdef CONFIG_HYPERV_TSCPAGE -static struct ms_hyperv_tsc_page *tsc_pg; +static struct ms_hyperv_tsc_page tsc_pg __aligned(PAGE_SIZE); struct ms_hyperv_tsc_page *hv_get_tsc_page(void) { - return tsc_pg; + return &tsc_pg; } EXPORT_SYMBOL_GPL(hv_get_tsc_page); static u64 notrace read_hv_sched_clock_tsc(void) { - u64 current_tick = hv_read_tsc_page(tsc_pg); + u64 current_tick = hv_read_tsc_page(&tsc_pg); if (current_tick == U64_MAX) hv_get_time_ref_count(current_tick); @@ -280,12 +280,8 @@ static bool __init hv_init_tsc_clocksource(void) if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) return false; - tsc_pg = vmalloc(PAGE_SIZE); - if (!tsc_pg) - return false; - hyperv_cs = &hyperv_cs_tsc; - phys_addr = page_to_phys(vmalloc_to_page(tsc_pg)); + phys_addr = virt_to_phys(&tsc_pg); /* * The Hyper-V TLFS specifies to preserve the value of reserved -- cgit v1.2.3-59-g8ed1b From bd00cd52d5be655a2f217e2ed74b91a71cb2b14f Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Wed, 14 Aug 2019 20:32:16 +0800 Subject: clocksource/drivers/hyperv: Add Hyper-V specific sched clock function Hyper-V guests use the default native_sched_clock() in pv_ops.time.sched_clock on x86. But native_sched_clock() directly uses the raw TSC value, which can be discontinuous in a Hyper-V VM. Add the generic hv_setup_sched_clock() to set the sched clock function appropriately. On x86, this sets pv_ops.time.sched_clock to read the Hyper-V reference TSC value that is scaled and adjusted to be continuous. Also move the Hyper-V reference TSC initialization much earlier in the boot process so no discontinuity is observed when pv_ops.time.sched_clock calculates its offset. [ tglx: Folded build fix ] Signed-off-by: Tianyu Lan Signed-off-by: Thomas Gleixner Reviewed-by: Michael Kelley Link: https://lkml.kernel.org/r/20190814123216.32245-3-Tianyu.Lan@microsoft.com --- arch/x86/hyperv/hv_init.c | 2 -- arch/x86/kernel/cpu/mshyperv.c | 8 ++++++++ drivers/clocksource/hyperv_timer.c | 22 ++++++++++++---------- include/asm-generic/mshyperv.h | 1 + 4 files changed, 21 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 0d258688c8cf..866dfb3dca48 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -301,8 +301,6 @@ void __init hyperv_init(void) x86_init.pci.arch_init = hv_pci_init; - /* Register Hyper-V specific clocksource */ - hv_init_clocksource(); return; remove_cpuhp_state: diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 062f77279ce3..53afd33990eb 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -29,6 +29,7 @@ #include #include #include +#include struct ms_hyperv_info ms_hyperv; EXPORT_SYMBOL_GPL(ms_hyperv); @@ -338,9 +339,16 @@ static void __init ms_hyperv_init_platform(void) x2apic_phys = 1; # endif + /* Register Hyper-V specific clocksource */ + hv_init_clocksource(); #endif } +void hv_setup_sched_clock(void *sched_clock) +{ + pv_ops.time.sched_clock = sched_clock; +} + const __initconst struct hypervisor_x86 x86_hyper_ms_hyperv = { .name = "Microsoft Hyper-V", .detect = ms_hyperv_platform, diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index 432aa331df04..c322ab4d3689 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -22,6 +22,7 @@ #include static struct clock_event_device __percpu *hv_clock_event; +static u64 hv_sched_clock_offset __ro_after_init; /* * If false, we're using the old mechanism for stimer0 interrupts @@ -222,7 +223,7 @@ struct ms_hyperv_tsc_page *hv_get_tsc_page(void) } EXPORT_SYMBOL_GPL(hv_get_tsc_page); -static u64 notrace read_hv_sched_clock_tsc(void) +static u64 notrace read_hv_clock_tsc(struct clocksource *arg) { u64 current_tick = hv_read_tsc_page(&tsc_pg); @@ -232,9 +233,9 @@ static u64 notrace read_hv_sched_clock_tsc(void) return current_tick; } -static u64 read_hv_clock_tsc(struct clocksource *arg) +static u64 read_hv_sched_clock_tsc(void) { - return read_hv_sched_clock_tsc(); + return read_hv_clock_tsc(NULL) - hv_sched_clock_offset; } static struct clocksource hyperv_cs_tsc = { @@ -246,7 +247,7 @@ static struct clocksource hyperv_cs_tsc = { }; #endif -static u64 notrace read_hv_sched_clock_msr(void) +static u64 notrace read_hv_clock_msr(struct clocksource *arg) { u64 current_tick; /* @@ -258,9 +259,9 @@ static u64 notrace read_hv_sched_clock_msr(void) return current_tick; } -static u64 read_hv_clock_msr(struct clocksource *arg) +static u64 read_hv_sched_clock_msr(void) { - return read_hv_sched_clock_msr(); + return read_hv_clock_msr(NULL) - hv_sched_clock_offset; } static struct clocksource hyperv_cs_msr = { @@ -298,8 +299,9 @@ static bool __init hv_init_tsc_clocksource(void) hv_set_clocksource_vdso(hyperv_cs_tsc); clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); - /* sched_clock_register is needed on ARM64 but is a no-op on x86 */ - sched_clock_register(read_hv_sched_clock_tsc, 64, HV_CLOCK_HZ); + hv_sched_clock_offset = hyperv_cs->read(hyperv_cs); + hv_setup_sched_clock(read_hv_sched_clock_tsc); + return true; } #else @@ -329,7 +331,7 @@ void __init hv_init_clocksource(void) hyperv_cs = &hyperv_cs_msr; clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); - /* sched_clock_register is needed on ARM64 but is a no-op on x86 */ - sched_clock_register(read_hv_sched_clock_msr, 64, HV_CLOCK_HZ); + hv_sched_clock_offset = hyperv_cs->read(hyperv_cs); + hv_setup_sched_clock(read_hv_sched_clock_msr); } EXPORT_SYMBOL_GPL(hv_init_clocksource); diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index 0becb7d9704d..18d8e2d8210f 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -167,6 +167,7 @@ void hyperv_report_panic(struct pt_regs *regs, long err); void hyperv_report_panic_msg(phys_addr_t pa, size_t size); bool hv_is_hyperv_initialized(void); void hyperv_cleanup(void); +void hv_setup_sched_clock(void *sched_clock); #else /* CONFIG_HYPERV */ static inline bool hv_is_hyperv_initialized(void) { return false; } static inline void hyperv_cleanup(void) {} -- cgit v1.2.3-59-g8ed1b From 3e2d94535adb2df15f3907e4b4c7cd8a5a4c2b5a Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 22 Aug 2019 10:36:30 +0200 Subject: clocksource/drivers/hyperv: Enable TSC page clocksource on 32bit There is no particular reason to not enable TSC page clocksource on 32-bit. mul_u64_u64_shr() is available and despite the increased computational complexity (compared to 64bit) TSC page is still a huge win compared to MSR-based clocksource. In-kernel reads: MSR based clocksource: 3361 cycles TSC page clocksource: 49 cycles Reads from userspace (utilizing vDSO in case of TSC page): MSR based clocksource: 5664 cycles TSC page clocksource: 131 cycles Enabling TSC page on 32bits allows to get rid of CONFIG_HYPERV_TSCPAGE as it is now not any different from CONFIG_HYPERV_TIMER. Signed-off-by: Vitaly Kuznetsov Signed-off-by: Thomas Gleixner Reviewed-by: Michael Kelley Link: https://lkml.kernel.org/r/20190822083630.17059-1-vkuznets@redhat.com --- arch/x86/include/asm/vdso/gettimeofday.h | 6 +++--- drivers/clocksource/hyperv_timer.c | 11 ----------- drivers/hv/Kconfig | 3 --- include/clocksource/hyperv_timer.h | 8 +++----- 4 files changed, 6 insertions(+), 22 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index ae91429129a6..bcbf901befbe 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -51,7 +51,7 @@ extern struct pvclock_vsyscall_time_info pvclock_page __attribute__((visibility("hidden"))); #endif -#ifdef CONFIG_HYPERV_TSCPAGE +#ifdef CONFIG_HYPERV_TIMER extern struct ms_hyperv_tsc_page hvclock_page __attribute__((visibility("hidden"))); #endif @@ -192,7 +192,7 @@ static u64 vread_pvclock(void) } #endif -#ifdef CONFIG_HYPERV_TSCPAGE +#ifdef CONFIG_HYPERV_TIMER static u64 vread_hvclock(void) { return hv_read_tsc_page(&hvclock_page); @@ -215,7 +215,7 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode) return vread_pvclock(); } #endif -#ifdef CONFIG_HYPERV_TSCPAGE +#ifdef CONFIG_HYPERV_TIMER if (clock_mode == VCLOCK_HVCLOCK) { barrier(); return vread_hvclock(); diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index c322ab4d3689..2317d4e3daaf 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -213,8 +213,6 @@ EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); struct clocksource *hyperv_cs; EXPORT_SYMBOL_GPL(hyperv_cs); -#ifdef CONFIG_HYPERV_TSCPAGE - static struct ms_hyperv_tsc_page tsc_pg __aligned(PAGE_SIZE); struct ms_hyperv_tsc_page *hv_get_tsc_page(void) @@ -245,7 +243,6 @@ static struct clocksource hyperv_cs_tsc = { .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -#endif static u64 notrace read_hv_clock_msr(struct clocksource *arg) { @@ -272,7 +269,6 @@ static struct clocksource hyperv_cs_msr = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -#ifdef CONFIG_HYPERV_TSCPAGE static bool __init hv_init_tsc_clocksource(void) { u64 tsc_msr; @@ -304,13 +300,6 @@ static bool __init hv_init_tsc_clocksource(void) return true; } -#else -static bool __init hv_init_tsc_clocksource(void) -{ - return false; -} -#endif - void __init hv_init_clocksource(void) { diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig index 9a59957922d4..79e5356a737a 100644 --- a/drivers/hv/Kconfig +++ b/drivers/hv/Kconfig @@ -14,9 +14,6 @@ config HYPERV config HYPERV_TIMER def_bool HYPERV -config HYPERV_TSCPAGE - def_bool HYPERV && X86_64 - config HYPERV_UTILS tristate "Microsoft Hyper-V Utilities driver" depends on HYPERV && CONNECTOR && NLS diff --git a/include/clocksource/hyperv_timer.h b/include/clocksource/hyperv_timer.h index a821deb8ecb2..422f5e5237be 100644 --- a/include/clocksource/hyperv_timer.h +++ b/include/clocksource/hyperv_timer.h @@ -28,12 +28,10 @@ extern void hv_stimer_cleanup(unsigned int cpu); extern void hv_stimer_global_cleanup(void); extern void hv_stimer0_isr(void); -#if IS_ENABLED(CONFIG_HYPERV) +#ifdef CONFIG_HYPERV_TIMER extern struct clocksource *hyperv_cs; extern void hv_init_clocksource(void); -#endif /* CONFIG_HYPERV */ -#ifdef CONFIG_HYPERV_TSCPAGE extern struct ms_hyperv_tsc_page *hv_get_tsc_page(void); static inline notrace u64 @@ -91,7 +89,7 @@ hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); } -#else /* CONFIG_HYPERV_TSC_PAGE */ +#else /* CONFIG_HYPERV_TIMER */ static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) { return NULL; @@ -102,6 +100,6 @@ static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, { return U64_MAX; } -#endif /* CONFIG_HYPERV_TSCPAGE */ +#endif /* CONFIG_HYPERV_TIMER */ #endif -- cgit v1.2.3-59-g8ed1b From 5b0221bf7b08163030e6801255ffbaf52775df01 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Thu, 15 Aug 2019 20:38:43 -0400 Subject: arm64: dts: imx8mm: Add system counter node Add i.MX8MM system counter node to enable timer-imx-sysctr broadcast timer driver. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano --- arch/arm64/boot/dts/freescale/imx8mm.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index 232a7412755a..89ef22a8f81e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -510,6 +510,14 @@ #pwm-cells = <2>; status = "disabled"; }; + + system_counter: timer@306a0000 { + compatible = "nxp,sysctr-timer"; + reg = <0x306a0000 0x20000>; + interrupts = ; + clocks = <&osc_24m>; + clock-names = "per"; + }; }; aips3: bus@30800000 { -- cgit v1.2.3-59-g8ed1b From 24e8a5db8ae46bf021d3b4063c005f443282ab4f Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Thu, 15 Aug 2019 20:38:44 -0400 Subject: arm64: dts: imx8mq: Add system counter node Add i.MX8MQ system counter node to enable timer-imx-sysctr broadcast timer driver. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano --- arch/arm64/boot/dts/freescale/imx8mq.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index d09b808eff87..b4529773af51 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -635,6 +635,14 @@ #pwm-cells = <2>; status = "disabled"; }; + + system_counter: timer@306a0000 { + compatible = "nxp,sysctr-timer"; + reg = <0x306a0000 0x20000>; + interrupts = ; + clocks = <&osc_25m>; + clock-names = "per"; + }; }; bus@30800000 { /* AIPS3 */ -- cgit v1.2.3-59-g8ed1b From 41cfe2a2a7f4fad5647031ad3a1da166452b5437 Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Wed, 28 Aug 2019 16:07:47 +0800 Subject: x86/hyperv: Hide pv_ops access for CONFIG_PARAVIRT=n hv_setup_sched_clock() references pv_ops which is only available when CONFIG_PARAVIRT=Y. Wrap it into a #ifdef Signed-off-by: Tianyu Lan Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20190828080747.204419-1-Tianyu.Lan@microsoft.com --- arch/x86/kernel/cpu/mshyperv.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 53afd33990eb..267daad8c036 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -346,7 +346,9 @@ static void __init ms_hyperv_init_platform(void) void hv_setup_sched_clock(void *sched_clock) { +#ifdef CONFIG_PARAVIRT pv_ops.time.sched_clock = sched_clock; +#endif } const __initconst struct hypervisor_x86 x86_hyper_ms_hyperv = { -- cgit v1.2.3-59-g8ed1b