/* * Copyright (C) 2017 Red Hat, Inc. * * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. */ #include #include #include #include #include #include #include #include #include #include "cpu/cpu.h" static DEFINE_MUTEX(spec_ctrl_mutex); static bool noibrs_cmdline __read_mostly; static bool ibp_disabled __read_mostly; static bool unsafe_module __read_mostly; static unsigned int ibrs_mode __read_mostly; /* * The ssbd_userset_key is set to true if the SSDB mode is user settable. */ struct static_key ssbd_userset_key = STATIC_KEY_INIT_FALSE; struct static_key retp_enabled_key = STATIC_KEY_INIT_FALSE; struct static_key ibrs_present_key = STATIC_KEY_INIT_FALSE; struct static_key ibrs_entry_key = STATIC_KEY_INIT_FALSE; struct static_key ibrs_exit_key = STATIC_KEY_INIT_FALSE; struct static_key ibpb_enabled_key = STATIC_KEY_INIT_FALSE; EXPORT_SYMBOL(ssbd_userset_key); EXPORT_SYMBOL(retp_enabled_key); EXPORT_SYMBOL(ibrs_present_key); EXPORT_SYMBOL(ibpb_enabled_key); /* * The vendor and possibly platform specific bits which can be modified in * x86_spec_ctrl_base. * */ u64 __read_mostly x86_spec_ctrl_mask = SPEC_CTRL_IBRS|SPEC_CTRL_SSBD; EXPORT_SYMBOL_GPL(x86_spec_ctrl_mask); /* * The Intel specification for the SPEC_CTRL MSR requires that we * preserve any already set reserved bits at boot time (e.g. for * future additions that this kernel is not currently aware of). * We then set any additional mitigation bits that we want * ourselves and always use this as the base for SPEC_CTRL. * We also use this when handling guest entry/exit as below. * * RHEL note: We do the above to be in sync with upstream, * but in the RHEL case, we have both x86_spec_ctrl_base, * and a PER_CPU spec_ctrl_pcp to track and manage. * * RHEL note: It's actually cleaner to directly export this * and allow all of our assorted IBRS management code to touch * this directly, rather than use the upstream accessors. We * implement them, but we don't use those in the RHEL code. */ /* * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any * writes to SPEC_CTRL contain whatever reserved bits have been set. */ u64 __read_mostly x86_spec_ctrl_base; EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); static bool spec_ctrl_msr_write; /* * AMD specific MSR info for Store Bypass control. x86_amd_ls_cfg_ssbd_mask * is initialized in identify_boot_cpu(). */ u64 __read_mostly x86_amd_ls_cfg_base; u64 __read_mostly x86_amd_ls_cfg_ssbd_mask; static inline bool ssb_is_user_settable(unsigned int mode) { return mode >= SPEC_STORE_BYPASS_PRCTL; } /* * Set the static key to match the given boolean flag */ static inline void set_static_key(struct static_key *key, bool enable) { if (enable && !static_key_enabled(key)) static_key_slow_inc(key); else if (!enable && static_key_enabled(key)) static_key_slow_dec(key); } void spec_ctrl_save_msr(void) { int cpu; unsigned int hival, loval; static int savecnt; spec_ctrl_msr_write = false; /* Allow STIBP in MSR_SPEC_CTRL if supported */ if (boot_cpu_has(X86_FEATURE_STIBP)) x86_spec_ctrl_mask |= SPEC_CTRL_STIBP; /* * Read the SPEC_CTRL MSR to account for reserved bits which may have * unknown values. AMD64_LS_CFG MSR is cached in the early AMD * init code as it is not enumerated and depends on the family. */ if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL) && !savecnt) { /* * This part is run only the first time it is called. */ rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); if (x86_spec_ctrl_base & x86_spec_ctrl_mask) { x86_spec_ctrl_base &= ~x86_spec_ctrl_mask; spec_ctrl_msr_write = true; native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); } } /* * RHEL only: update the PER_CPU spec_ctrl_pcp cached values */ loval = x86_spec_ctrl_base & 0xffffffff; hival = (x86_spec_ctrl_base >> 32) & 0xffffffff; for_each_possible_cpu(cpu) { WRITE_ONCE(per_cpu(spec_ctrl_pcp.hi32, cpu), hival); WRITE_ONCE(per_cpu(spec_ctrl_pcp.entry, cpu), loval); WRITE_ONCE(per_cpu(spec_ctrl_pcp.exit, cpu), loval); } savecnt++; } /* * This is called for setting the entry or exit values in the spec_ctrl_pcp * structure when the SSDB is user settable. The state of the SSBD bit * is maintained. */ static void set_spec_ctrl_value(unsigned int *ptr, unsigned int value) { unsigned int old, new, val; old = READ_ONCE(*ptr); for (;;) { new = value | (old & SPEC_CTRL_SSBD); val = cmpxchg(ptr, old, new); if (val == old) break; old = val; } } static void set_spec_ctrl_pcp(bool entry, bool exit) { unsigned int enabled = this_cpu_read(spec_ctrl_pcp.enabled); unsigned int entry_val = this_cpu_read(spec_ctrl_pcp.entry); unsigned int exit_val = this_cpu_read(spec_ctrl_pcp.exit); int cpu, redo_cnt; /* * Set if the SSBD bit of the SPEC_CTRL MSR is user settable. */ bool ssb_user_settable = boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && ssb_is_user_settable(READ_ONCE(ssb_mode)); /* * Mask off the SSBD bit first if it is user settable. * Otherwise, make sure that the SSBD bit of the entry and exit * values match that of the x86_spec_ctrl_base. */ if (ssb_user_settable) { entry_val &= ~SPEC_CTRL_SSBD; exit_val &= ~SPEC_CTRL_SSBD; } else { entry_val = (entry_val & ~SPEC_CTRL_SSBD) | (x86_spec_ctrl_base & SPEC_CTRL_SSBD); exit_val = (exit_val & ~SPEC_CTRL_SSBD) | (x86_spec_ctrl_base & SPEC_CTRL_SSBD); } /* * For ibrs_always, we only need to write the MSR at kernel entry * to fulfill the barrier semantics for some CPUs. For enhanced * IBRS, we don't need to write the MSR at kernel entry/exit anymore. */ if (entry && exit) enabled = (ibrs_mode == IBRS_ENHANCED) ? 0 : SPEC_CTRL_PCP_IBRS_ENTRY; else if (entry != exit) enabled = SPEC_CTRL_PCP_IBRS_ENTRY|SPEC_CTRL_PCP_IBRS_EXIT; else enabled = 0; /* * Set ibrs_entry_key and ibrs_exit_key to match the enabled flag. */ set_static_key(&ibrs_entry_key, enabled & SPEC_CTRL_PCP_IBRS_ENTRY); set_static_key(&ibrs_exit_key , enabled & SPEC_CTRL_PCP_IBRS_EXIT); if (entry) entry_val |= SPEC_CTRL_IBRS; else entry_val &= ~SPEC_CTRL_IBRS; if (exit) exit_val |= SPEC_CTRL_IBRS; else exit_val &= ~SPEC_CTRL_IBRS; for_each_possible_cpu(cpu) { unsigned int *pentry = &per_cpu(spec_ctrl_pcp.entry, cpu); unsigned int *pexit = &per_cpu(spec_ctrl_pcp.exit, cpu); WRITE_ONCE(per_cpu(spec_ctrl_pcp.enabled, cpu), enabled); if (!ssb_user_settable) { WRITE_ONCE(*pentry, entry_val); WRITE_ONCE(*pexit, exit_val); } else { /* * Since the entry and exit fields can be modified * concurrently by spec_ctrl_set_ssbd() to set or * clear the SSBD bit, We need to maintain the * SSBD bit and use atomic instruction to do the * modification here. */ set_spec_ctrl_value(pentry, entry_val); set_spec_ctrl_value(pexit, exit_val); } } if (!ssb_user_settable) return; /* * Because of the non-atomic read-modify-write nature of * spec_ctrl_set_ssbd() function, the atomic entry/exit value changes * above may be lost. So we need to recheck it again and reapply the * change, if necessary. */ recheck: redo_cnt = 0; smp_mb(); for_each_possible_cpu(cpu) { unsigned int *pentry = &per_cpu(spec_ctrl_pcp.entry, cpu); unsigned int *pexit = &per_cpu(spec_ctrl_pcp.exit, cpu); if ((READ_ONCE(*pentry) & ~SPEC_CTRL_SSBD) != entry_val) { set_spec_ctrl_value(pentry, entry_val); redo_cnt++; } if ((READ_ONCE(*pexit) & ~SPEC_CTRL_SSBD) != exit_val) { set_spec_ctrl_value(pexit, exit_val); redo_cnt++; } } if (redo_cnt) goto recheck; } static void set_spec_ctrl_ibpb(bool enable) { if (boot_cpu_has(X86_FEATURE_IBPB)) set_static_key(&ibpb_enabled_key, enable); } /* * The following values are written to IBRS on kernel entry/exit: * * entry exit * ibrs 1 0 * ibrs_always 1 x (not written on exit) * ibrs_user 0 1 */ static void set_spec_ctrl_pcp_ibrs(void) { set_spec_ctrl_pcp(true, false); set_spec_ctrl_ibpb(true); ibrs_mode = IBRS_ENABLED; } static void set_spec_ctrl_pcp_ibrs_always(void) { set_spec_ctrl_pcp(true, true); set_spec_ctrl_ibpb(true); ibrs_mode = IBRS_ENABLED_ALWAYS; } static void set_spec_ctrl_pcp_ibrs_user(void) { set_spec_ctrl_pcp(false, true); set_spec_ctrl_ibpb(true); ibrs_mode = IBRS_ENABLED_USER; } void clear_spec_ctrl_pcp(void) { ibrs_mode = IBRS_DISABLED; set_spec_ctrl_pcp(false, false); if (!static_key_enabled(&retp_enabled_key)) set_spec_ctrl_ibpb(false); } static void sync_all_cpus_spec_ctrl(void) { int cpu; get_online_cpus(); for_each_online_cpu(cpu) wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, SPEC_CTRL_MSR_REFRESH); put_online_cpus(); } static void __sync_this_cpu_ibp(void *data) { bool enable = *(bool *)data; u64 val; /* disable IBP on old CPU families */ rdmsrl(MSR_F15H_IC_CFG, val); if (!enable) val |= MSR_F15H_IC_CFG_DIS_IND; else val &= ~MSR_F15H_IC_CFG_DIS_IND; wrmsrl(MSR_F15H_IC_CFG, val); } /* enable means IBP should be enabled in the CPU (i.e. fast) */ static void sync_all_cpus_ibp(bool enable) { get_online_cpus(); __sync_this_cpu_ibp(&enable); smp_call_function_many(cpu_online_mask, __sync_this_cpu_ibp, &enable, 1); put_online_cpus(); } static void set_spec_ctrl_retp(bool enable) { /* * Make sure that IBPB is enabled if either IBRS or retpoline * is enabled. ibrs_mode should be properly set before calling * set_spec_ctrl_retp(). */ set_static_key(&retp_enabled_key, enable); if (enable || (ibrs_mode == IBRS_DISABLED)) set_spec_ctrl_ibpb(enable); } static void spec_ctrl_disable_all(void) { int cpu; for_each_possible_cpu(cpu) WRITE_ONCE(per_cpu(spec_ctrl_pcp.enabled, cpu), 0); set_spec_ctrl_retp(false); set_spec_ctrl_ibpb(false); } static int __init noibrs(char *str) { noibrs_cmdline = true; return 0; } early_param("noibrs", noibrs); static int __init noibpb(char *str) { /* deprecated */ return 0; } early_param("noibpb", noibpb); static bool is_skylake_era(void) { if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && boot_cpu_data.x86 == 6) { switch (boot_cpu_data.x86_model) { case INTEL_FAM6_SKYLAKE_MOBILE: case INTEL_FAM6_SKYLAKE_DESKTOP: case INTEL_FAM6_SKYLAKE_X: case INTEL_FAM6_KABYLAKE_MOBILE: case INTEL_FAM6_KABYLAKE_DESKTOP: return true; } } return false; } /* * The caller should have checked X86_FEATURE_IBRS_ENHANCED before calling * it, Which also implies X86_FEATURE_IBRS is present. */ void spec_ctrl_enable_ibrs_enhanced(void) { ibrs_mode = IBRS_ENHANCED; set_spec_ctrl_pcp(true, true); set_spec_ctrl_ibpb(true); } bool spec_ctrl_force_enable_ibrs(void) { if (cpu_has_spec_ctrl()) { if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) spec_ctrl_enable_ibrs_enhanced(); else set_spec_ctrl_pcp_ibrs(); return true; } return false; } bool spec_ctrl_cond_enable_ibrs(bool full_retp) { if (cpu_has_spec_ctrl() && (is_skylake_era() || !full_retp) && !noibrs_cmdline) { if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { spec_ctrl_enable_ibrs_enhanced(); } else { set_spec_ctrl_pcp_ibrs(); /* * Print a warning message about performance * impact of enabling IBRS vs. retpoline. */ pr_warn_once("Using IBRS as the default Spectre v2 mitigation for a Skylake-\n"); pr_warn_once("generation CPU. This may have a negative performance impact.\n"); } return true; } return false; } bool spec_ctrl_enable_ibrs_always(void) { if (cpu_has_spec_ctrl()) { if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) spec_ctrl_enable_ibrs_enhanced(); else set_spec_ctrl_pcp_ibrs_always(); return true; } return false; } bool spec_ctrl_force_enable_ibp_disabled(void) { /* * Some AMD CPUs don't need IBPB or IBRS CPUID bits, because * they can just disable indirect branch predictor * support (MSR 0xc0011021[14]). */ if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) { ibp_disabled = true; return true; } ibp_disabled = false; return false; } bool spec_ctrl_cond_enable_ibp_disabled(void) { if (boot_cpu_has(X86_FEATURE_IBP_DISABLE) && !noibrs_cmdline) { ibp_disabled = true; return true; } ibp_disabled = false; return false; } void spec_ctrl_enable_retpoline(void) { set_spec_ctrl_retp(true); } bool spec_ctrl_enable_retpoline_ibrs_user(void) { if (!cpu_has_spec_ctrl()) return false; if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { spec_ctrl_enable_ibrs_enhanced(); return true; } set_spec_ctrl_retp(true); set_spec_ctrl_pcp_ibrs_user(); return true; } void spec_ctrl_report_unsafe_module(struct module *mod) { #ifdef CONFIG_MODULE if (retp_compiler() && !is_skylake_era()) pr_warn_once("WARNING: module '%s' built without retpoline-enabled compiler, may affect Spectre v2 mitigation\n", mod->name); #endif unsafe_module = true; } enum spectre_v2_mitigation spec_ctrl_get_mitigation(void) { enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; if (ibp_disabled) mode = SPECTRE_V2_IBP_DISABLED; else if (ibrs_mode == IBRS_ENABLED_ALWAYS) mode = SPECTRE_V2_IBRS_ALWAYS; else if (ibrs_mode == IBRS_ENABLED) mode = SPECTRE_V2_IBRS; else if (ibrs_mode == IBRS_ENHANCED) mode = SPECTRE_V2_IBRS_ENHANCED; else if (retp_enabled()) { if (!retp_enabled_full()) mode = SPECTRE_V2_RETPOLINE_MINIMAL; else if (!boot_cpu_has(X86_FEATURE_IBPB)) mode = SPECTRE_V2_RETPOLINE_NO_IBPB; else if (unsafe_module) mode = SPECTRE_V2_RETPOLINE_UNSAFE_MODULE; else if (ibrs_mode == IBRS_ENABLED_USER) mode = SPECTRE_V2_RETPOLINE_IBRS_USER; else mode = SPECTRE_V2_RETPOLINE; } spectre_v2_enabled = mode; /* Update spectre_v2_enabled */ return spectre_v2_enabled; } static void spec_ctrl_print_features(void) { if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) { printk(KERN_INFO "FEATURE SPEC_CTRL Present (Implicit)\n"); printk(KERN_INFO "FEATURE IBPB_SUPPORT Present (Implicit)\n"); return; } if (boot_cpu_has(X86_FEATURE_IBRS)) printk(KERN_INFO "FEATURE SPEC_CTRL Present\n"); else printk(KERN_INFO "FEATURE SPEC_CTRL Not Present\n"); if (boot_cpu_has(X86_FEATURE_IBPB)) printk(KERN_INFO "FEATURE IBPB_SUPPORT Present\n"); else printk(KERN_INFO "FEATURE IBPB_SUPPORT Not Present\n"); } void spec_ctrl_cpu_init(void) { if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) { bool enabled = !ibp_disabled; __sync_this_cpu_ibp(&enabled); return; } if ((ibrs_mode == IBRS_ENABLED_ALWAYS) || (ibrs_mode == IBRS_ENHANCED) || (spec_ctrl_msr_write && (system_state == SYSTEM_BOOTING))) native_wrmsr(MSR_IA32_SPEC_CTRL, this_cpu_read(spec_ctrl_pcp.entry), this_cpu_read(spec_ctrl_pcp.hi32)); } static void spec_ctrl_reinit_all_cpus(void) { if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) { sync_all_cpus_ibp(!ibrs_mode); return; } if ((ibrs_mode == IBRS_ENABLED_ALWAYS) || (ibrs_mode == IBRS_DISABLED) || spec_ctrl_msr_write) { sync_all_cpus_spec_ctrl(); spec_ctrl_msr_write = false; } } void spec_ctrl_init(void) { set_static_key(&ibrs_present_key, boot_cpu_has(X86_FEATURE_IBRS)); spec_ctrl_print_features(); } void spec_ctrl_rescan_cpuid(void) { enum spectre_v2_mitigation old_mode; bool old_ibrs, old_ibpb, old_ssbd, old_mds; bool ssbd_changed; int cpu; if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) return; mutex_lock(&spec_ctrl_mutex); if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { old_ibrs = boot_cpu_has(X86_FEATURE_IBRS); old_ibpb = boot_cpu_has(X86_FEATURE_IBPB); old_ssbd = boot_cpu_has(X86_FEATURE_SSBD); old_mds = boot_cpu_has(X86_FEATURE_MD_CLEAR); old_mode = spec_ctrl_get_mitigation(); /* detect spec ctrl related cpuid additions */ get_cpu_cap(&boot_cpu_data); /* * If there were no spec ctrl or MDS related changes, * we're done */ ssbd_changed = (old_ssbd != boot_cpu_has(X86_FEATURE_SSBD)); if (old_ibrs == boot_cpu_has(X86_FEATURE_IBRS) && old_ibpb == boot_cpu_has(X86_FEATURE_IBPB) && old_mds == boot_cpu_has(X86_FEATURE_MD_CLEAR) && !ssbd_changed) goto done; /* * The IBRS, IBPB, SSBD & MDS cpuid bits may have * just been set in the boot_cpu_data, transfer them * to the per-cpu data too. */ if (boot_cpu_has(X86_FEATURE_IBRS)) for_each_online_cpu(cpu) set_cpu_cap(&cpu_data(cpu), X86_FEATURE_IBRS); if (boot_cpu_has(X86_FEATURE_IBPB)) for_each_online_cpu(cpu) set_cpu_cap(&cpu_data(cpu), X86_FEATURE_IBPB); if (boot_cpu_has(X86_FEATURE_SSBD)) for_each_online_cpu(cpu) set_cpu_cap(&cpu_data(cpu), X86_FEATURE_SSBD); if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) for_each_online_cpu(cpu) set_cpu_cap(&cpu_data(cpu), X86_FEATURE_MD_CLEAR); /* update static key, print the changed IBRS/IBPB features */ spec_ctrl_init(); if (ssbd_changed) { u64 old_spec_ctrl = x86_spec_ctrl_base; /* * Redo speculative store bypass setup. */ ssb_select_mitigation(); if (x86_spec_ctrl_base != old_spec_ctrl) { /* * Need to propagate the new baseline to all * the percpu spec_ctrl structures. The * spectre v2 re-initialization below will * reset to the right percpu values. */ spec_ctrl_save_msr(); spec_ctrl_msr_write = true; } } /* * Re-execute the v2 mitigation logic based on any new CPU * features. Note that any debugfs-based changes the user may * have made will be overwritten, because new features are now * available, so any previous changes may no longer be * relevant. Go back to the defaults unless they're overridden * by the cmdline. */ spec_ctrl_disable_all(); __spectre_v2_select_mitigation(); spec_ctrl_reinit_all_cpus(); /* print any mitigation changes */ if (old_mode != spec_ctrl_get_mitigation()) spectre_v2_print_mitigation(); /* * Look for X86_FEATURE_MD_CLEAR change for CPUs that are * vulnerable to MDS & reflect that in the mds vulnerabilities * file. */ if (boot_cpu_has_bug(X86_BUG_MDS) && (mds_mitigation != MDS_MITIGATION_OFF)) { enum mds_mitigations new; new = boot_cpu_has(X86_FEATURE_MD_CLEAR) ? MDS_MITIGATION_FULL : MDS_MITIGATION_VMWERV; if (new != mds_mitigation) { mds_mitigation = new; mds_print_mitigation(); } } } done: mutex_unlock(&spec_ctrl_mutex); } /* * Change the SSBD bit of the spec_ctrl structure of the current CPU. * The caller has to make sure that preemption is disabled so that * no CPU change is possible during the call. * * Since spec_ctrl_set_ssbd() is in the fast path, we are not doing * any atomic update to the entry and exit values. The percpu logical * operation used here is a single non-atomic read-modify-write instruction. * As a result, we need to do more checking at the slowpath set_spec_ctrl_pcp() * function to make sure that any changes in the ibrs_enabled value get * reflected correctly in all the spec_ctrl_pcp structures. */ void spec_ctrl_set_ssbd(bool ssbd_on) { if (ssbd_on) { this_cpu_or(spec_ctrl_pcp.entry, SPEC_CTRL_SSBD); this_cpu_or(spec_ctrl_pcp.exit, SPEC_CTRL_SSBD); } else { this_cpu_and(spec_ctrl_pcp.entry, ~(int)SPEC_CTRL_SSBD); this_cpu_and(spec_ctrl_pcp.exit, ~(int)SPEC_CTRL_SSBD); } } static ssize_t __enabled_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos, unsigned int *field) { char buf[32]; unsigned int len; len = sprintf(buf, "%d\n", READ_ONCE(*field)); return simple_read_from_buffer(user_buf, count, ppos, buf, len); } static ssize_t ibrs_enabled_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { unsigned int enabled = ibrs_mode; if (ibp_disabled) enabled = IBRS_ENABLED_ALWAYS; return __enabled_read(file, user_buf, count, ppos, &enabled); } static ssize_t ibrs_enabled_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { char buf[32]; ssize_t len; unsigned int enable; len = min(count, sizeof(buf) - 1); if (copy_from_user(buf, user_buf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtouint(buf, 0, &enable)) return -EINVAL; if (enable > IBRS_MAX) return -EINVAL; mutex_lock(&spec_ctrl_mutex); if ((!ibp_disabled && enable == ibrs_mode) || (ibp_disabled && enable == IBRS_ENABLED_ALWAYS)) goto out_unlock; if (boot_cpu_has(X86_FEATURE_IBP_DISABLE)) { if (enable == IBRS_ENABLED || enable == IBRS_ENABLED_USER) { count = -EINVAL; goto out_unlock; } if (enable == IBRS_DISABLED) { sync_all_cpus_ibp(true); ibp_disabled = false; } else { WARN_ON(enable != IBRS_ENABLED_ALWAYS); sync_all_cpus_ibp(false); ibp_disabled = true; set_spec_ctrl_retp(false); } goto out_unlock; } if (!cpu_has_spec_ctrl()) { count = -ENODEV; goto out_unlock; } /* * With enhanced IBRS, it is either IBRS_ENHANCED or IBRS_DISABLED. * Other options are not supported. */ if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) && (enable != IBRS_DISABLED) && (enable != IBRS_ENHANCED)) { count = -EINVAL; goto out_unlock; } if (!boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) && (enable == IBRS_ENHANCED)) { count = -EINVAL; goto out_unlock; } if (enable == IBRS_DISABLED) { clear_spec_ctrl_pcp(); sync_all_cpus_spec_ctrl(); } else if (enable == IBRS_ENABLED) { set_spec_ctrl_pcp_ibrs(); set_spec_ctrl_retp(false); } else if (enable == IBRS_ENABLED_ALWAYS) { set_spec_ctrl_pcp_ibrs_always(); set_spec_ctrl_retp(false); sync_all_cpus_spec_ctrl(); } else if (enable == IBRS_ENHANCED) { spec_ctrl_enable_ibrs_enhanced(); set_spec_ctrl_retp(false); sync_all_cpus_spec_ctrl(); } else { WARN_ON(enable != IBRS_ENABLED_USER); set_spec_ctrl_pcp_ibrs_user(); set_spec_ctrl_retp(true); } spec_ctrl_get_mitigation(); out_unlock: mutex_unlock(&spec_ctrl_mutex); return count; } static const struct file_operations fops_ibrs_enabled = { .read = ibrs_enabled_read, .write = ibrs_enabled_write, .llseek = default_llseek, }; static ssize_t ibpb_enabled_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { unsigned int enabled = ibpb_enabled(); if (ibp_disabled) enabled = 1; return __enabled_read(file, user_buf, count, ppos, &enabled); } static const struct file_operations fops_ibpb_enabled = { .read = ibpb_enabled_read, .llseek = default_llseek, }; static ssize_t retp_enabled_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { unsigned int enabled = retp_enabled(); return __enabled_read(file, user_buf, count, ppos, &enabled); } static ssize_t retp_enabled_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { char buf[32]; ssize_t len; unsigned int enable; len = min(count, sizeof(buf) - 1); if (copy_from_user(buf, user_buf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtouint(buf, 0, &enable)) return -EINVAL; if (enable > 1) return -EINVAL; mutex_lock(&spec_ctrl_mutex); if (enable == retp_enabled()) goto out_unlock; set_spec_ctrl_retp(enable); if (enable) { /* enforce sane combinations */ if (ibp_disabled) { sync_all_cpus_ibp(true); ibp_disabled = false; } else if ((ibrs_mode == IBRS_ENABLED) || (ibrs_mode == IBRS_ENHANCED)) { clear_spec_ctrl_pcp(); sync_all_cpus_spec_ctrl(); } else if (ibrs_mode == IBRS_ENABLED_ALWAYS) { set_spec_ctrl_pcp_ibrs_user(); } } spec_ctrl_get_mitigation(); out_unlock: mutex_unlock(&spec_ctrl_mutex); return count; } static const struct file_operations fops_retp_enabled = { .read = retp_enabled_read, .write = retp_enabled_write, .llseek = default_llseek, }; /* * The ssb_mode variable controls the state of the Speculative Store Bypass * Disable (SSBD) mitigation. * 0 - SSBD is disabled (speculative store bypass is enabled). * 1 - SSBD is enabled (speculative store bypass is disabled). * 2 - SSBD is controlled by prctl only. * 3 - SSBD is controlled by both prctl and seccomp. */ static ssize_t ssbd_enabled_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { unsigned int enabled = ssb_mode; return __enabled_read(file, user_buf, count, ppos, &enabled); } static void ssbd_spec_ctrl_write(unsigned int mode) { /* * We have to update the x86_spec_ctrl_base first and then all the * SPEC_CTRL MSRs. We also need to update the ssb_mode prior to * that if the new mode isn't user settable to make sure that * the existing SSBD bit in the spec_ctrl_pcp won't carry over. */ if (!ssb_is_user_settable(mode)) smp_store_mb(ssb_mode, mode); switch (ibrs_mode) { case IBRS_DISABLED: clear_spec_ctrl_pcp(); break; case IBRS_ENABLED: set_spec_ctrl_pcp_ibrs(); break; case IBRS_ENABLED_ALWAYS: set_spec_ctrl_pcp_ibrs_always(); break; case IBRS_ENABLED_USER: set_spec_ctrl_pcp_ibrs_user(); break; case IBRS_ENHANCED: spec_ctrl_enable_ibrs_enhanced(); break; } sync_all_cpus_spec_ctrl(); } static void ssbd_amd_write(unsigned int mode) { u64 msrval; int msr, cpu; if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) { msr = MSR_AMD64_VIRT_SPEC_CTRL; msrval = (mode == SPEC_STORE_BYPASS_DISABLE) ? SPEC_CTRL_SSBD : 0; } else { msr = MSR_AMD64_LS_CFG; msrval = x86_amd_ls_cfg_base; if (mode == SPEC_STORE_BYPASS_DISABLE) msrval |= x86_amd_ls_cfg_ssbd_mask; } /* * If the new mode isn't settable, we have to update the * ssb_mode first. */ if (!ssb_is_user_settable(mode)) smp_store_mb(ssb_mode, mode); /* * If the old mode isn't user settable, it is assumed that no * existing task will have the TIF_SSBD bit set. So we can safely * overwrite the MSRs. */ get_online_cpus(); for_each_online_cpu(cpu) wrmsrl_on_cpu(cpu, msr, msrval); put_online_cpus(); } static ssize_t ssbd_enabled_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { char buf[32]; ssize_t len; unsigned int mode; const unsigned int mode_max = IS_ENABLED(CONFIG_SECCOMP) ? SPEC_STORE_BYPASS_SECCOMP : SPEC_STORE_BYPASS_PRCTL; if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) || !boot_cpu_has(X86_FEATURE_SSBD)) return -ENODEV; len = min(count, sizeof(buf) - 1); if (copy_from_user(buf, user_buf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtouint(buf, 0, &mode)) return -EINVAL; if (mode > mode_max) return -EINVAL; mutex_lock(&spec_ctrl_mutex); if (mode == ssb_mode) goto out_unlock; WARN_ON_ONCE(ssb_is_user_settable(ssb_mode) != static_key_enabled(&ssbd_userset_key)); /* * User settable => !settable: clear ssbd_userset_key early * User !settable => settable : set ssbd_userset_key late */ if (static_key_enabled(&ssbd_userset_key) && !ssb_is_user_settable(mode)) static_key_slow_dec(&ssbd_userset_key); /* Set/clear the SSBD bit in x86_spec_ctrl_base accordingly */ if (mode == SPEC_STORE_BYPASS_DISABLE) x86_spec_ctrl_base |= SPEC_CTRL_SSBD; else x86_spec_ctrl_base &= ~SPEC_CTRL_SSBD; /* * If both the old and new SSB modes are user settable or it is * transitioning from SPEC_STORE_BYPASS_NONE to a user settable * mode, we don't need to touch the spec_ctrl_pcp structure or the * AMD LS_CFG MSRs at all and so the change can be made directly. */ if (ssb_is_user_settable(mode) && (ssb_is_user_settable(ssb_mode) || (ssb_mode == SPEC_STORE_BYPASS_NONE))) goto out; if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD)) ssbd_spec_ctrl_write(mode); else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || boot_cpu_has(X86_FEATURE_VIRT_SSBD)) ssbd_amd_write(mode); out: WRITE_ONCE(ssb_mode, mode); if (!static_key_enabled(&ssbd_userset_key) && ssb_is_user_settable(mode)) static_key_slow_inc(&ssbd_userset_key); ssb_print_mitigation(); out_unlock: mutex_unlock(&spec_ctrl_mutex); return count; } static const struct file_operations fops_ssbd_enabled = { .read = ssbd_enabled_read, .write = ssbd_enabled_write, .llseek = default_llseek, }; static ssize_t smt_present_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { unsigned int present = atomic_read(&sched_smt_present.enabled); return __enabled_read(file, user_buf, count, ppos, &present); } static const struct file_operations fops_smt_present = { .read = smt_present_read, .llseek = default_llseek, }; static int __init debugfs_spec_ctrl(void) { debugfs_create_file("ibrs_enabled", S_IRUSR | S_IWUSR, arch_debugfs_dir, NULL, &fops_ibrs_enabled); debugfs_create_file("ibpb_enabled", S_IRUSR, arch_debugfs_dir, NULL, &fops_ibpb_enabled); debugfs_create_file("retp_enabled", S_IRUSR | S_IWUSR, arch_debugfs_dir, NULL, &fops_retp_enabled); debugfs_create_file("ssbd_enabled", S_IRUSR, arch_debugfs_dir, NULL, &fops_ssbd_enabled); debugfs_create_file("smt_present", S_IRUSR, arch_debugfs_dir, NULL, &fops_smt_present); return 0; } late_initcall(debugfs_spec_ctrl); #if defined(RETPOLINE) /* * RETPOLINE does not protect against indirect speculation * in firmware code. Enable IBRS to protect firmware execution. */ bool unprotected_firmware_begin(void) { return spec_ctrl_ibrs_on_firmware(); } EXPORT_SYMBOL_GPL(unprotected_firmware_begin); void unprotected_firmware_end(bool ibrs_on) { spec_ctrl_ibrs_off_firmware(ibrs_on); } EXPORT_SYMBOL_GPL(unprotected_firmware_end); #else bool unprotected_firmware_begin(void) { return false; } EXPORT_SYMBOL_GPL(unprotected_firmware_begin); void unprotected_firmware_end(bool ibrs_on) { } EXPORT_SYMBOL_GPL(unprotected_firmware_end); #endif