diff options
Diffstat (limited to 'drivers/firmware')
| -rw-r--r-- | drivers/firmware/Kconfig | 12 | ||||
| -rw-r--r-- | drivers/firmware/Makefile | 2 | ||||
| -rw-r--r-- | drivers/firmware/efi/libstub/Makefile | 5 | ||||
| -rw-r--r-- | drivers/firmware/psci.c | 2 | ||||
| -rw-r--r-- | drivers/firmware/psci_checker.c | 490 | ||||
| -rw-r--r-- | drivers/firmware/qcom_scm.c | 4 | ||||
| -rw-r--r-- | drivers/firmware/tegra/Kconfig | 25 | ||||
| -rw-r--r-- | drivers/firmware/tegra/Makefile | 2 | ||||
| -rw-r--r-- | drivers/firmware/tegra/bpmp.c | 868 | ||||
| -rw-r--r-- | drivers/firmware/tegra/ivc.c | 695 |
10 files changed, 2101 insertions, 4 deletions
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index bca172d42c74..6a42a098fe59 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -8,6 +8,17 @@ menu "Firmware Drivers" config ARM_PSCI_FW bool +config ARM_PSCI_CHECKER + bool "ARM PSCI checker" + depends on ARM_PSCI_FW && HOTPLUG_CPU && !TORTURE_TEST + help + Run the PSCI checker during startup. This checks that hotplug and + suspend operations work correctly when using PSCI. + + The torture tests may interfere with the PSCI checker by turning CPUs + on and off through hotplug, so for now torture tests and PSCI checker + are mutually exclusive. + config ARM_SCPI_PROTOCOL tristate "ARM System Control and Power Interface (SCPI) Message Protocol" depends on MAILBOX @@ -210,5 +221,6 @@ source "drivers/firmware/broadcom/Kconfig" source "drivers/firmware/google/Kconfig" source "drivers/firmware/efi/Kconfig" source "drivers/firmware/meson/Kconfig" +source "drivers/firmware/tegra/Kconfig" endmenu diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 898ac41fa8b3..e9e178b38f7e 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -2,6 +2,7 @@ # Makefile for the linux kernel. # obj-$(CONFIG_ARM_PSCI_FW) += psci.o +obj-$(CONFIG_ARM_PSCI_CHECKER) += psci_checker.o obj-$(CONFIG_ARM_SCPI_PROTOCOL) += arm_scpi.o obj-$(CONFIG_ARM_SCPI_POWER_DOMAIN) += scpi_pm_domain.o obj-$(CONFIG_DMI) += dmi_scan.o @@ -26,3 +27,4 @@ obj-y += meson/ obj-$(CONFIG_GOOGLE_FIRMWARE) += google/ obj-$(CONFIG_EFI) += efi/ obj-$(CONFIG_UEFI_CPER) += efi/ +obj-y += tegra/ diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index c06945160a41..5e23e2d305e7 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -11,7 +11,7 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 \ -mno-mmx -mno-sse cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \ +cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) -g0 \ -fno-builtin -fpic -mno-single-pic-base cflags-$(CONFIG_EFI_ARMSTUB) += -I$(srctree)/scripts/dtc/libfdt @@ -79,5 +79,6 @@ quiet_cmd_stubcopy = STUBCPY $@ # decompressor. So move our .data to .data.efistub, which is preserved # explicitly by the decompressor linker script. # -STUBCOPY_FLAGS-$(CONFIG_ARM) += --rename-section .data=.data.efistub +STUBCOPY_FLAGS-$(CONFIG_ARM) += --rename-section .data=.data.efistub \ + -R ___ksymtab+sort -R ___kcrctab+sort STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c index 8263429e21b8..6c60a5087caf 100644 --- a/drivers/firmware/psci.c +++ b/drivers/firmware/psci.c @@ -630,7 +630,7 @@ int __init psci_dt_init(void) np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); - if (!np) + if (!np || !of_device_is_available(np)) return -ENODEV; init_fn = (psci_initcall_t)matched_np->data; diff --git a/drivers/firmware/psci_checker.c b/drivers/firmware/psci_checker.c new file mode 100644 index 000000000000..44bdb78f837b --- /dev/null +++ b/drivers/firmware/psci_checker.c @@ -0,0 +1,490 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Copyright (C) 2016 ARM Limited + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/atomic.h> +#include <linux/completion.h> +#include <linux/cpu.h> +#include <linux/cpuidle.h> +#include <linux/cpu_pm.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/module.h> +#include <linux/preempt.h> +#include <linux/psci.h> +#include <linux/slab.h> +#include <linux/tick.h> +#include <linux/topology.h> + +#include <asm/cpuidle.h> + +#include <uapi/linux/psci.h> + +#define NUM_SUSPEND_CYCLE (10) + +static unsigned int nb_available_cpus; +static int tos_resident_cpu = -1; + +static atomic_t nb_active_threads; +static struct completion suspend_threads_started = + COMPLETION_INITIALIZER(suspend_threads_started); +static struct completion suspend_threads_done = + COMPLETION_INITIALIZER(suspend_threads_done); + +/* + * We assume that PSCI operations are used if they are available. This is not + * necessarily true on arm64, since the decision is based on the + * "enable-method" property of each CPU in the DT, but given that there is no + * arch-specific way to check this, we assume that the DT is sensible. + */ +static int psci_ops_check(void) +{ + int migrate_type = -1; + int cpu; + + if (!(psci_ops.cpu_off && psci_ops.cpu_on && psci_ops.cpu_suspend)) { + pr_warn("Missing PSCI operations, aborting tests\n"); + return -EOPNOTSUPP; + } + + if (psci_ops.migrate_info_type) + migrate_type = psci_ops.migrate_info_type(); + + if (migrate_type == PSCI_0_2_TOS_UP_MIGRATE || + migrate_type == PSCI_0_2_TOS_UP_NO_MIGRATE) { + /* There is a UP Trusted OS, find on which core it resides. */ + for_each_online_cpu(cpu) + if (psci_tos_resident_on(cpu)) { + tos_resident_cpu = cpu; + break; + } + if (tos_resident_cpu == -1) + pr_warn("UP Trusted OS resides on no online CPU\n"); + } + + return 0; +} + +static int find_clusters(const struct cpumask *cpus, + const struct cpumask **clusters) +{ + unsigned int nb = 0; + cpumask_var_t tmp; + + if (!alloc_cpumask_var(&tmp, GFP_KERNEL)) + return -ENOMEM; + cpumask_copy(tmp, cpus); + + while (!cpumask_empty(tmp)) { + const struct cpumask *cluster = + topology_core_cpumask(cpumask_any(tmp)); + + clusters[nb++] = cluster; + cpumask_andnot(tmp, tmp, cluster); + } + + free_cpumask_var(tmp); + return nb; +} + +/* + * offlined_cpus is a temporary array but passing it as an argument avoids + * multiple allocations. + */ +static unsigned int down_and_up_cpus(const struct cpumask *cpus, + struct cpumask *offlined_cpus) +{ + int cpu; + int err = 0; + + cpumask_clear(offlined_cpus); + + /* Try to power down all CPUs in the mask. */ + for_each_cpu(cpu, cpus) { + int ret = cpu_down(cpu); + + /* + * cpu_down() checks the number of online CPUs before the TOS + * resident CPU. + */ + if (cpumask_weight(offlined_cpus) + 1 == nb_available_cpus) { + if (ret != -EBUSY) { + pr_err("Unexpected return code %d while trying " + "to power down last online CPU %d\n", + ret, cpu); + ++err; + } + } else if (cpu == tos_resident_cpu) { + if (ret != -EPERM) { + pr_err("Unexpected return code %d while trying " + "to power down TOS resident CPU %d\n", + ret, cpu); + ++err; + } + } else if (ret != 0) { + pr_err("Error occurred (%d) while trying " + "to power down CPU %d\n", ret, cpu); + ++err; + } + + if (ret == 0) + cpumask_set_cpu(cpu, offlined_cpus); + } + + /* Try to power up all the CPUs that have been offlined. */ + for_each_cpu(cpu, offlined_cpus) { + int ret = cpu_up(cpu); + + if (ret != 0) { + pr_err("Error occurred (%d) while trying " + "to power up CPU %d\n", ret, cpu); + ++err; + } else { + cpumask_clear_cpu(cpu, offlined_cpus); + } + } + + /* + * Something went bad at some point and some CPUs could not be turned + * back on. + */ + WARN_ON(!cpumask_empty(offlined_cpus) || + num_online_cpus() != nb_available_cpus); + + return err; +} + +static int hotplug_tests(void) +{ + int err; + cpumask_var_t offlined_cpus; + int i, nb_cluster; + const struct cpumask **clusters; + char *page_buf; + + err = -ENOMEM; + if (!alloc_cpumask_var(&offlined_cpus, GFP_KERNEL)) + return err; + /* We may have up to nb_available_cpus clusters. */ + clusters = kmalloc_array(nb_available_cpus, sizeof(*clusters), + GFP_KERNEL); + if (!clusters) + goto out_free_cpus; + page_buf = (char *)__get_free_page(GFP_KERNEL); + if (!page_buf) + goto out_free_clusters; + + err = 0; + nb_cluster = find_clusters(cpu_online_mask, clusters); + + /* + * Of course the last CPU cannot be powered down and cpu_down() should + * refuse doing that. + */ + pr_info("Trying to turn off and on again all CPUs\n"); + err += down_and_up_cpus(cpu_online_mask, offlined_cpus); + + /* + * Take down CPUs by cluster this time. When the last CPU is turned + * off, the cluster itself should shut down. + */ + for (i = 0; i < nb_cluster; ++i) { + int cluster_id = + topology_physical_package_id(cpumask_any(clusters[i])); + ssize_t len = cpumap_print_to_pagebuf(true, page_buf, + clusters[i]); + /* Remove trailing newline. */ + page_buf[len - 1] = '\0'; + pr_info("Trying to turn off and on again cluster %d " + "(CPUs %s)\n", cluster_id, page_buf); + err += down_and_up_cpus(clusters[i], offlined_cpus); + } + + free_page((unsigned long)page_buf); +out_free_clusters: + kfree(clusters); +out_free_cpus: + free_cpumask_var(offlined_cpus); + return err; +} + +static void dummy_callback(unsigned long ignored) {} + +static int suspend_cpu(int index, bool broadcast) +{ + int ret; + + arch_cpu_idle_enter(); + + if (broadcast) { + /* + * The local timer will be shut down, we need to enter tick + * broadcast. + */ + ret = tick_broadcast_enter(); + if (ret) { + /* + * In the absence of hardware broadcast mechanism, + * this CPU might be used to broadcast wakeups, which + * may be why entering tick broadcast has failed. + * There is little the kernel can do to work around + * that, so enter WFI instead (idle state 0). + */ + cpu_do_idle(); + ret = 0; + goto out_arch_exit; + } + } + + /* + * Replicate the common ARM cpuidle enter function + * (arm_enter_idle_state). + */ + ret = CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, index); + + if (broadcast) + tick_broadcast_exit(); + +out_arch_exit: + arch_cpu_idle_exit(); + + return ret; +} + +static int suspend_test_thread(void *arg) +{ + int cpu = (long)arg; + int i, nb_suspend = 0, nb_shallow_sleep = 0, nb_err = 0; + struct sched_param sched_priority = { .sched_priority = MAX_RT_PRIO-1 }; + struct cpuidle_device *dev; + struct cpuidle_driver *drv; + /* No need for an actual callback, we just want to wake up the CPU. */ + struct timer_list wakeup_timer = + TIMER_INITIALIZER(dummy_callback, 0, 0); + + /* Wait for the main thread to give the start signal. */ + wait_for_completion(&suspend_threads_started); + + /* Set maximum priority to preempt all other threads on this CPU. */ + if (sched_setscheduler_nocheck(current, SCHED_FIFO, &sched_priority)) + pr_warn("Failed to set suspend thread scheduler on CPU %d\n", + cpu); + + dev = this_cpu_read(cpuidle_devices); + drv = cpuidle_get_cpu_driver(dev); + + pr_info("CPU %d entering suspend cycles, states 1 through %d\n", + cpu, drv->state_count - 1); + + for (i = 0; i < NUM_SUSPEND_CYCLE; ++i) { + int index; + /* + * Test all possible states, except 0 (which is usually WFI and + * doesn't use PSCI). + */ + for (index = 1; index < drv->state_count; ++index) { + struct cpuidle_state *state = &drv->states[index]; + bool broadcast = state->flags & CPUIDLE_FLAG_TIMER_STOP; + int ret; + + /* + * Set the timer to wake this CPU up in some time (which + * should be largely sufficient for entering suspend). + * If the local tick is disabled when entering suspend, + * suspend_cpu() takes care of switching to a broadcast + * tick, so the timer will still wake us up. + */ + mod_timer(&wakeup_timer, jiffies + + usecs_to_jiffies(state->target_residency)); + + /* IRQs must be disabled during suspend operations. */ + local_irq_disable(); + + ret = suspend_cpu(index, broadcast); + + /* + * We have woken up. Re-enable IRQs to handle any + * pending interrupt, do not wait until the end of the + * loop. + */ + local_irq_enable(); + + if (ret == index) { + ++nb_suspend; + } else if (ret >= 0) { + /* We did not enter the expected state. */ + ++nb_shallow_sleep; + } else { + pr_err("Failed to suspend CPU %d: error %d " + "(requested state %d, cycle %d)\n", + cpu, ret, index, i); + ++nb_err; + } + } + } + + /* + * Disable the timer to make sure that the timer will not trigger + * later. + */ + del_timer(&wakeup_timer); + + if (atomic_dec_return_relaxed(&nb_active_threads) == 0) + complete(&suspend_threads_done); + + /* Give up on RT scheduling and wait for termination. */ + sched_priority.sched_priority = 0; + if (sched_setscheduler_nocheck(current, SCHED_NORMAL, &sched_priority)) + pr_warn("Failed to set suspend thread scheduler on CPU %d\n", + cpu); + for (;;) { + /* Needs to be set first to avoid missing a wakeup. */ + set_current_state(TASK_INTERRUPTIBLE); + if (kthread_should_stop()) { + __set_current_state(TASK_RUNNING); + break; + } + schedule(); + } + + pr_info("CPU %d suspend test results: success %d, shallow states %d, errors %d\n", + cpu, nb_suspend, nb_shallow_sleep, nb_err); + + return nb_err; +} + +static int suspend_tests(void) +{ + int i, cpu, err = 0; + struct task_struct **threads; + int nb_threads = 0; + + threads = kmalloc_array(nb_available_cpus, sizeof(*threads), + GFP_KERNEL); + if (!threads) + return -ENOMEM; + + /* + * Stop cpuidle to prevent the idle tasks from entering a deep sleep + * mode, as it might interfere with the suspend threads on other CPUs. + * This does not prevent the suspend threads from using cpuidle (only + * the idle tasks check this status). Take the idle lock so that + * the cpuidle driver and device look-up can be carried out safely. + */ + cpuidle_pause_and_lock(); + + for_each_online_cpu(cpu) { + struct task_struct *thread; + /* Check that cpuidle is available on that CPU. */ + struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); + struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); + + if (!dev || !drv) { + pr_warn("cpuidle not available on CPU %d, ignoring\n", + cpu); + continue; + } + + thread = kthread_create_on_cpu(suspend_test_thread, + (void *)(long)cpu, cpu, + "psci_suspend_test"); + if (IS_ERR(thread)) + pr_err("Failed to create kthread on CPU %d\n", cpu); + else + threads[nb_threads++] = thread; + } + + if (nb_threads < 1) { + err = -ENODEV; + goto out; + } + + atomic_set(&nb_active_threads, nb_threads); + + /* + * Wake up the suspend threads. To avoid the main thread being preempted + * before all the threads have been unparked, the suspend threads will + * wait for the completion of suspend_threads_started. + */ + for (i = 0; i < nb_threads; ++i) + wake_up_process(threads[i]); + complete_all(&suspend_threads_started); + + wait_for_completion(&suspend_threads_done); + + + /* Stop and destroy all threads, get return status. */ + for (i = 0; i < nb_threads; ++i) + err += kthread_stop(threads[i]); + out: + cpuidle_resume_and_unlock(); + kfree(threads); + return err; +} + +static int __init psci_checker(void) +{ + int ret; + + /* + * Since we're in an initcall, we assume that all the CPUs that all + * CPUs that can be onlined have been onlined. + * + * The tests assume that hotplug is enabled but nobody else is using it, + * otherwise the results will be unpredictable. However, since there + * is no userspace yet in initcalls, that should be fine, as long as + * no torture test is running at the same time (see Kconfig). + */ + nb_available_cpus = num_online_cpus(); + + /* Check PSCI operations are set up and working. */ + ret = psci_ops_check(); + if (ret) + return ret; + + pr_info("PSCI checker started using %u CPUs\n", nb_available_cpus); + + pr_info("Starting hotplug tests\n"); + ret = hotplug_tests(); + if (ret == 0) + pr_info("Hotplug tests passed OK\n"); + else if (ret > 0) + pr_err("%d error(s) encountered in hotplug tests\n", ret); + else { + pr_err("Out of memory\n"); + return ret; + } + + pr_info("Starting suspend tests (%d cycles per state)\n", + NUM_SUSPEND_CYCLE); + ret = suspend_tests(); + if (ret == 0) + pr_info("Suspend tests passed OK\n"); + else if (ret > 0) + pr_err("%d error(s) encountered in suspend tests\n", ret); + else { + switch (ret) { + case -ENOMEM: + pr_err("Out of memory\n"); + break; + case -ENODEV: + pr_warn("Could not start suspend tests on any CPU\n"); + break; + } + } + + pr_info("PSCI checker completed\n"); + return ret < 0 ? ret : 0; +} +late_initcall(psci_checker); diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c index d95c70227c05..9bf66aefdbd0 100644 --- a/drivers/firmware/qcom_scm.c +++ b/drivers/firmware/qcom_scm.c @@ -356,7 +356,9 @@ static int qcom_scm_probe(struct platform_device *pdev) scm->reset.ops = &qcom_scm_pas_reset_ops; scm->reset.nr_resets = 1; scm->reset.of_node = pdev->dev.of_node; - reset_controller_register(&scm->reset); + ret = devm_reset_controller_register(&pdev->dev, &scm->reset); + if (ret) + return ret; /* vote for max clk rate for highest performance */ ret = clk_set_rate(scm->core_clk, INT_MAX); diff --git a/drivers/firmware/tegra/Kconfig b/drivers/firmware/tegra/Kconfig new file mode 100644 index 000000000000..ff2730d5c468 --- /dev/null +++ b/drivers/firmware/tegra/Kconfig @@ -0,0 +1,25 @@ +menu "Tegra firmware driver" + +config TEGRA_IVC + bool "Tegra IVC protocol" + depends on ARCH_TEGRA + help + IVC (Inter-VM Communication) protocol is part of the IPC + (Inter Processor Communication) framework on Tegra. It maintains the + data and the different commuication channels in SysRAM or RAM and + keeps the content is synchronization between host CPU and remote + processors. + +config TEGRA_BPMP + bool "Tegra BPMP driver" + depends on ARCH_TEGRA && TEGRA_HSP_MBOX && TEGRA_IVC + help + BPMP (Boot and Power Management Processor) is designed to off-loading + the PM functions which include clock/DVFS/thermal/power from the CPU. + It needs HSP as the HW synchronization and notification module and + IVC module as the message communication protocol. + + This driver manages the IPC interface between host CPU and the + firmware running on BPMP. + +endmenu diff --git a/drivers/firmware/tegra/Makefile b/drivers/firmware/tegra/Makefile new file mode 100644 index 000000000000..e34a2f79e1ad --- /dev/null +++ b/drivers/firmware/tegra/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_TEGRA_BPMP) += bpmp.o +obj-$(CONFIG_TEGRA_IVC) += ivc.o diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c new file mode 100644 index 000000000000..4ff02d310868 --- /dev/null +++ b/drivers/firmware/tegra/bpmp.c @@ -0,0 +1,868 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include <linux/clk/tegra.h> +#include <linux/genalloc.h> +#include <linux/mailbox_client.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/semaphore.h> + +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> +#include <soc/tegra/ivc.h> + +#define MSG_ACK BIT(0) +#define MSG_RING BIT(1) + +static inline struct tegra_bpmp * +mbox_client_to_bpmp(struct mbox_client *client) +{ + return container_of(client, struct tegra_bpmp, mbox.client); +} + +struct tegra_bpmp *tegra_bpmp_get(struct device *dev) +{ + struct platform_device *pdev; + struct tegra_bpmp *bpmp; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0); + if (!np) + return ERR_PTR(-ENOENT); + + pdev = of_find_device_by_node(np); + if (!pdev) { + bpmp = ERR_PTR(-ENODEV); + goto put; + } + + bpmp = platform_get_drvdata(pdev); + if (!bpmp) { + bpmp = ERR_PTR(-EPROBE_DEFER); + put_device(&pdev->dev); + goto put; + } + +put: + of_node_put(np); + return bpmp; +} +EXPORT_SYMBOL_GPL(tegra_bpmp_get); + +void tegra_bpmp_put(struct tegra_bpmp *bpmp) +{ + if (bpmp) + put_device(bpmp->dev); +} +EXPORT_SYMBOL_GPL(tegra_bpmp_put); + +static int tegra_bpmp_channel_get_index(struct tegra_bpmp_channel *channel) +{ + return channel - channel->bpmp->channels; +} + +static int +tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel) +{ + struct tegra_bpmp *bpmp = channel->bpmp; + unsigned int offset, count; + int index; + + offset = bpmp->soc->channels.thread.offset; + count = bpmp->soc->channels.thread.count; + + index = tegra_bpmp_channel_get_index(channel); + if (index < 0) + return index; + + if (index < offset || index >= offset + count) + return -EINVAL; + + return index - offset; +} + +static struct tegra_bpmp_channel * +tegra_bpmp_channel_get_thread(struct tegra_bpmp *bpmp, unsigned int index) +{ + unsigned int offset = bpmp->soc->channels.thread.offset; + unsigned int count = bpmp->soc->channels.thread.count; + + if (index >= count) + return NULL; + + return &bpmp->channels[offset + index]; +} + +static struct tegra_bpmp_channel * +tegra_bpmp_channel_get_tx(struct tegra_bpmp *bpmp) +{ + unsigned int offset = bpmp->soc->channels.cpu_tx.offset; + + return &bpmp->channels[offset + smp_processor_id()]; +} + +static struct tegra_bpmp_channel * +tegra_bpmp_channel_get_rx(struct tegra_bpmp *bpmp) +{ + unsigned int offset = bpmp->soc->channels.cpu_rx.offset; + + return &bpmp->channels[offset]; +} + +static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg) +{ + return (msg->tx.size <= MSG_DATA_MIN_SZ) && + (msg->rx.size <= MSG_DATA_MIN_SZ) && + (msg->tx.size == 0 || msg->tx.data) && + (msg->rx.size == 0 || msg->rx.data); +} + +static bool tegra_bpmp_master_acked(struct tegra_bpmp_channel *channel) +{ + void *frame; + + frame = tegra_ivc_read_get_next_frame(channel->ivc); + if (IS_ERR(frame)) { + channel->ib = NULL; + return false; + } + + channel->ib = frame; + + return true; +} + +static int tegra_bpmp_wait_ack(struct tegra_bpmp_channel *channel) +{ + unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout; + ktime_t end; + + end = ktime_add_us(ktime_get(), timeout); + + do { + if (tegra_bpmp_master_acked(channel)) + return 0; + } while (ktime_before(ktime_get(), end)); + + return -ETIMEDOUT; +} + +static bool tegra_bpmp_master_free(struct tegra_bpmp_channel *channel) +{ + void *frame; + + frame = tegra_ivc_write_get_next_frame(channel->ivc); + if (IS_ERR(frame)) { + channel->ob = NULL; + return false; + } + + channel->ob = frame; + + return true; +} + +static int tegra_bpmp_wait_master_free(struct tegra_bpmp_channel *channel) +{ + unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout; + ktime_t start, now; + + start = ns_to_ktime(local_clock()); + + do { + if (tegra_bpmp_master_free(channel)) + return 0; + + now = ns_to_ktime(local_clock()); + } while (ktime_us_delta(now, start) < timeout); + + return -ETIMEDOUT; +} + +static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel, + void *data, size_t size) +{ + if (data && size > 0) + memcpy(data, channel->ib->data, size); + + return tegra_ivc_read_advance(channel->ivc); +} + +static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel, + void *data, size_t size) +{ + struct tegra_bpmp *bpmp = channel->bpmp; + unsigned long flags; + ssize_t err; + int index; + + index = tegra_bpmp_channel_get_thread_index(channel); + if (index < 0) + return index; + + spin_lock_irqsave(&bpmp->lock, flags); + err = __tegra_bpmp_channel_read(channel, data, size); + clear_bit(index, bpmp->threaded.allocated); + spin_unlock_irqrestore(&bpmp->lock, flags); + + up(&bpmp->threaded.lock); + + return err; +} + +static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel, + unsigned int mrq, unsigned long flags, + const void *data, size_t size) +{ + channel->ob->code = mrq; + channel->ob->flags = flags; + + if (data && size > 0) + memcpy(channel->ob->data, data, size); + + return tegra_ivc_write_advance(channel->ivc); +} + +static struct tegra_bpmp_channel * +tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq, + const void *data, size_t size) +{ + unsigned long timeout = bpmp->soc->channels.thread.timeout; + unsigned int count = bpmp->soc->channels.thread.count; + struct tegra_bpmp_channel *channel; + unsigned long flags; + unsigned int index; + int err; + + err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout)); + if (err < 0) + return ERR_PTR(err); + + spin_lock_irqsave(&bpmp->lock, flags); + + index = find_first_zero_bit(bpmp->threaded.allocated, count); + if (index == count) { + channel = ERR_PTR(-EBUSY); + goto unlock; + } + + channel = tegra_bpmp_channel_get_thread(bpmp, index); + if (!channel) { + channel = ERR_PTR(-EINVAL); + goto unlock; + } + + if (!tegra_bpmp_master_free(channel)) { + channel = ERR_PTR(-EBUSY); + goto unlock; + } + + set_bit(index, bpmp->threaded.allocated); + + err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING, + data, size); + if (err < 0) { + clear_bit(index, bpmp->threaded.allocated); + goto unlock; + } + + set_bit(index, bpmp->threaded.busy); + +unlock: + spin_unlock_irqrestore(&bpmp->lock, flags); + return channel; +} + +static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel, + unsigned int mrq, unsigned long flags, + const void *data, size_t size) +{ + int err; + + err = tegra_bpmp_wait_master_free(channel); + if (err < 0) + return err; + + return __tegra_bpmp_channel_write(channel, mrq, flags, data, size); +} + +int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp, + struct tegra_bpmp_message *msg) +{ + struct tegra_bpmp_channel *channel; + int err; + + if (WARN_ON(!irqs_disabled())) + return -EPERM; + + if (!tegra_bpmp_message_valid(msg)) + return -EINVAL; + + channel = tegra_bpmp_channel_get_tx(bpmp); + + err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK, + msg->tx.data, msg->tx.size); + if (err < 0) + return err; + + err = mbox_send_message(bpmp->mbox.channel, NULL); + if (err < 0) + return err; + + mbox_client_txdone(bpmp->mbox.channel, 0); + + err = tegra_bpmp_wait_ack(channel); + if (err < 0) + return err; + + return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size); +} +EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic); + +int tegra_bpmp_transfer(struct tegra_bpmp *bpmp, + struct tegra_bpmp_message *msg) +{ + struct tegra_bpmp_channel *channel; + unsigned long timeout; + int err; + + if (WARN_ON(irqs_disabled())) + return -EPERM; + + if (!tegra_bpmp_message_valid(msg)) + return -EINVAL; + + channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data, + msg->tx.size); + if (IS_ERR(channel)) + return PTR_ERR(channel); + + err = mbox_send_message(bpmp->mbox.channel, NULL); + if (err < 0) + return err; + + mbox_client_txdone(bpmp->mbox.channel, 0); + + timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout); + + err = wait_for_completion_timeout(&channel->completion, timeout); + if (err == 0) + return -ETIMEDOUT; + + return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size); +} +EXPORT_SYMBOL_GPL(tegra_bpmp_transfer); + +static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp, + unsigned int mrq) +{ + struct tegra_bpmp_mrq *entry; + + list_for_each_entry(entry, &bpmp->mrqs, list) + if (entry->mrq == mrq) + return entry; + + return NULL; +} + +static void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, + int code, const void *data, size_t size) +{ + unsigned long flags = channel->ib->flags; + struct tegra_bpmp *bpmp = channel->bpmp; + struct tegra_bpmp_mb_data *frame; + int err; + + if (WARN_ON(size > MSG_DATA_MIN_SZ)) + return; + + err = tegra_ivc_read_advance(channel->ivc); + if (WARN_ON(err < 0)) + return; + + if ((flags & MSG_ACK) == 0) + return; + + frame = tegra_ivc_write_get_next_frame(channel->ivc); + if (WARN_ON(IS_ERR(frame))) + return; + + frame->code = code; + + if (data && size > 0) + memcpy(frame->data, data, size); + + err = tegra_ivc_write_advance(channel->ivc); + if (WARN_ON(err < 0)) + return; + + if (flags & MSG_RING) { + err = mbox_send_message(bpmp->mbox.channel, NULL); + if (WARN_ON(err < 0)) + return; + + mbox_client_txdone(bpmp->mbox.channel, 0); + } +} + +static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp, + unsigned int mrq, + struct tegra_bpmp_channel *channel) +{ + struct tegra_bpmp_mrq *entry; + u32 zero = 0; + + spin_lock(&bpmp->lock); + + entry = tegra_bpmp_find_mrq(bpmp, mrq); + if (!entry) { + spin_unlock(&bpmp->lock); + tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero)); + return; + } + + entry->handler(mrq, channel, entry->data); + + spin_unlock(&bpmp->lock); +} + +int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, + tegra_bpmp_mrq_handler_t handler, void *data) +{ + struct tegra_bpmp_mrq *entry; + unsigned long flags; + + if (!handler) + return -EINVAL; + + entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + spin_lock_irqsave(&bpmp->lock, flags); + + entry->mrq = mrq; + entry->handler = handler; + entry->data = data; + list_add(&entry->list, &bpmp->mrqs); + + spin_unlock_irqrestore(&bpmp->lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq); + +void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data) +{ + struct tegra_bpmp_mrq *entry; + unsigned long flags; + + spin_lock_irqsave(&bpmp->lock, flags); + + entry = tegra_bpmp_find_mrq(bpmp, mrq); + if (!entry) + goto unlock; + + list_del(&entry->list); + devm_kfree(bpmp->dev, entry); + +unlock: + spin_unlock_irqrestore(&bpmp->lock, flags); +} +EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq); + +static void tegra_bpmp_mrq_handle_ping(unsigned int mrq, + struct tegra_bpmp_channel *channel, + void *data) +{ + struct mrq_ping_request *request; + struct mrq_ping_response response; + + request = (struct mrq_ping_request *)channel->ib->data; + + memset(&response, 0, sizeof(response)); + response.reply = request->challenge << 1; + + tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response)); +} + +static int tegra_bpmp_ping(struct tegra_bpmp *bpmp) +{ + struct mrq_ping_response response; + struct mrq_ping_request request; + struct tegra_bpmp_message msg; + unsigned long flags; + ktime_t start, end; + int err; + + memset(&request, 0, sizeof(request)); + request.challenge = 1; + + memset(&response, 0, sizeof(response)); + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_PING; + msg.tx.data = &request; + msg.tx.size = sizeof(request); + msg.rx.data = &response; + msg.rx.size = sizeof(response); + + local_irq_save(flags); + start = ktime_get(); + err = tegra_bpmp_transfer_atomic(bpmp, &msg); + end = ktime_get(); + local_irq_restore(flags); + + if (!err) + dev_dbg(bpmp->dev, + "ping ok: challenge: %u, response: %u, time: %lld\n", + request.challenge, response.reply, + ktime_to_us(ktime_sub(end, start))); + + return err; +} + +static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag, + size_t size) +{ + struct mrq_query_tag_request request; + struct tegra_bpmp_message msg; + unsigned long flags; + dma_addr_t phys; + void *virt; + int err; + + virt = dma_alloc_coherent(bpmp->dev, MSG_DATA_MIN_SZ, &phys, + GFP_KERNEL | GFP_DMA32); + if (!virt) + return -ENOMEM; + + memset(&request, 0, sizeof(request)); + request.addr = phys; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_QUERY_TAG; + msg.tx.data = &request; + msg.tx.size = sizeof(request); + + local_irq_save(flags); + err = tegra_bpmp_transfer_atomic(bpmp, &msg); + local_irq_restore(flags); + + if (err == 0) + strlcpy(tag, virt, size); + + dma_free_coherent(bpmp->dev, MSG_DATA_MIN_SZ, virt, phys); + + return err; +} + +static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel) +{ + unsigned long flags = channel->ob->flags; + + if ((flags & MSG_RING) == 0) + return; + + complete(&channel->completion); +} + +static void tegra_bpmp_handle_rx(struct mbox_client *client, void *data) +{ + struct tegra_bpmp *bpmp = mbox_client_to_bpmp(client); + struct tegra_bpmp_channel *channel; + unsigned int i, count; + unsigned long *busy; + + channel = tegra_bpmp_channel_get_rx(bpmp); + count = bpmp->soc->channels.thread.count; + busy = bpmp->threaded.busy; + + if (tegra_bpmp_master_acked(channel)) + tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel); + + spin_lock(&bpmp->lock); + + for_each_set_bit(i, busy, count) { + struct tegra_bpmp_channel *channel; + + channel = tegra_bpmp_channel_get_thread(bpmp, i); + if (!channel) + continue; + + if (tegra_bpmp_master_acked(channel)) { + tegra_bpmp_channel_signal(channel); + clear_bit(i, busy); + } + } + + spin_unlock(&bpmp->lock); +} + +static void tegra_bpmp_ivc_notify(struct tegra_ivc *ivc, void *data) +{ + struct tegra_bpmp *bpmp = data; + int err; + + if (WARN_ON(bpmp->mbox.channel == NULL)) + return; + + err = mbox_send_message(bpmp->mbox.channel, NULL); + if (err < 0) + return; + + mbox_client_txdone(bpmp->mbox.channel, 0); +} + +static int tegra_bpmp_channel_init(struct tegra_bpmp_channel *channel, + struct tegra_bpmp *bpmp, + unsigned int index) +{ + size_t message_size, queue_size; + unsigned int offset; + int err; + + channel->ivc = devm_kzalloc(bpmp->dev, sizeof(*channel->ivc), + GFP_KERNEL); + if (!channel->ivc) + return -ENOMEM; + + message_size = tegra_ivc_align(MSG_MIN_SZ); + queue_size = tegra_ivc_total_queue_size(message_size); + offset = queue_size * index; + + err = tegra_ivc_init(channel->ivc, NULL, + bpmp->rx.virt + offset, bpmp->rx.phys + offset, + bpmp->tx.virt + offset, bpmp->tx.phys + offset, + 1, message_size, tegra_bpmp_ivc_notify, + bpmp); + if (err < 0) { + dev_err(bpmp->dev, "failed to setup IVC for channel %u: %d\n", + index, err); + return err; + } + + init_completion(&channel->completion); + channel->bpmp = bpmp; + + return 0; +} + +static void tegra_bpmp_channel_reset(struct tegra_bpmp_channel *channel) +{ + /* reset the channel state */ + tegra_ivc_reset(channel->ivc); + + /* sync the channel state with BPMP */ + while (tegra_ivc_notified(channel->ivc)) + ; +} + +static void tegra_bpmp_channel_cleanup(struct tegra_bpmp_channel *channel) +{ + tegra_ivc_cleanup(channel->ivc); +} + +static int tegra_bpmp_probe(struct platform_device *pdev) +{ + struct tegra_bpmp_channel *channel; + struct tegra_bpmp *bpmp; + unsigned int i; + char tag[32]; + size_t size; + int err; + + bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL); + if (!bpmp) + return -ENOMEM; + + bpmp->soc = of_device_get_match_data(&pdev->dev); + bpmp->dev = &pdev->dev; + + bpmp->tx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 0); + if (!bpmp->tx.pool) { + dev_err(&pdev->dev, "TX shmem pool not found\n"); + return -ENOMEM; + } + + bpmp->tx.virt = gen_pool_dma_alloc(bpmp->tx.pool, 4096, &bpmp->tx.phys); + if (!bpmp->tx.virt) { + dev_err(&pdev->dev, "failed to allocate from TX pool\n"); + return -ENOMEM; + } + + bpmp->rx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 1); + if (!bpmp->rx.pool) { + dev_err(&pdev->dev, "RX shmem pool not found\n"); + err = -ENOMEM; + goto free_tx; + } + + bpmp->rx.virt = gen_pool_dma_alloc(bpmp->rx.pool, 4096, &bpmp->rx.phys); + if (!bpmp->rx.pool) { + dev_err(&pdev->dev, "failed to allocate from RX pool\n"); + err = -ENOMEM; + goto free_tx; + } + + INIT_LIST_HEAD(&bpmp->mrqs); + spin_lock_init(&bpmp->lock); + + bpmp->threaded.count = bpmp->soc->channels.thread.count; + sema_init(&bpmp->threaded.lock, bpmp->threaded.count); + + size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long); + + bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + if (!bpmp->threaded.allocated) { + err = -ENOMEM; + goto free_rx; + } + + bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + if (!bpmp->threaded.busy) { + err = -ENOMEM; + goto free_rx; + } + + bpmp->num_channels = bpmp->soc->channels.cpu_tx.count + + bpmp->soc->channels.thread.count + + bpmp->soc->channels.cpu_rx.count; + + bpmp->channels = devm_kcalloc(&pdev->dev, bpmp->num_channels, + sizeof(*channel), GFP_KERNEL); + if (!bpmp->channels) { + err = -ENOMEM; + goto free_rx; + } + + /* message channel initialization */ + for (i = 0; i < bpmp->num_channels; i++) { + struct tegra_bpmp_channel *channel = &bpmp->channels[i]; + + err = tegra_bpmp_channel_init(channel, bpmp, i); + if (err < 0) + goto cleanup_channels; + } + + /* mbox registration */ + bpmp->mbox.client.dev = &pdev->dev; + bpmp->mbox.client.rx_callback = tegra_bpmp_handle_rx; + bpmp->mbox.client.tx_block = false; + bpmp->mbox.client.knows_txdone = false; + + bpmp->mbox.channel = mbox_request_channel(&bpmp->mbox.client, 0); + if (IS_ERR(bpmp->mbox.channel)) { + err = PTR_ERR(bpmp->mbox.channel); + dev_err(&pdev->dev, "failed to get HSP mailbox: %d\n", err); + goto cleanup_channels; + } + + /* reset message channels */ + for (i = 0; i < bpmp->num_channels; i++) { + struct tegra_bpmp_channel *channel = &bpmp->channels[i]; + + tegra_bpmp_channel_reset(channel); + } + + err = tegra_bpmp_request_mrq(bpmp, MRQ_PING, + tegra_bpmp_mrq_handle_ping, bpmp); + if (err < 0) + goto free_mbox; + + err = tegra_bpmp_ping(bpmp); + if (err < 0) { + dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err); + goto free_mrq; + } + + err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag) - 1); + if (err < 0) { + dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err); + goto free_mrq; + } + + dev_info(&pdev->dev, "firmware: %s\n", tag); + + err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev); + if (err < 0) + goto free_mrq; + + err = tegra_bpmp_init_clocks(bpmp); + if (err < 0) + goto free_mrq; + + err = tegra_bpmp_init_resets(bpmp); + if (err < 0) + goto free_mrq; + + platform_set_drvdata(pdev, bpmp); + + return 0; + +free_mrq: + tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp); +free_mbox: + mbox_free_channel(bpmp->mbox.channel); +cleanup_channels: + while (i--) + tegra_bpmp_channel_cleanup(&bpmp->channels[i]); +free_rx: + gen_pool_free(bpmp->rx.pool, (unsigned long)bpmp->rx.virt, 4096); +free_tx: + gen_pool_free(bpmp->tx.pool, (unsigned long)bpmp->tx.virt, 4096); + return err; +} + +static const struct tegra_bpmp_soc tegra186_soc = { + .channels = { + .cpu_tx = { + .offset = 0, + .count = 6, + .timeout = 60 * USEC_PER_SEC, + }, + .thread = { + .offset = 6, + .count = 7, + .timeout = 600 * USEC_PER_SEC, + }, + .cpu_rx = { + .offset = 13, + .count = 1, + .timeout = 0, + }, + }, + .num_resets = 193, +}; + +static const struct of_device_id tegra_bpmp_match[] = { + { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc }, + { } +}; + +static struct platform_driver tegra_bpmp_driver = { + .driver = { + .name = "tegra-bpmp", + .of_match_table = tegra_bpmp_match, + }, + .probe = tegra_bpmp_probe, +}; + +static int __init tegra_bpmp_init(void) +{ + return platform_driver_register(&tegra_bpmp_driver); +} +core_initcall(tegra_bpmp_init); diff --git a/drivers/firmware/tegra/ivc.c b/drivers/firmware/tegra/ivc.c new file mode 100644 index 000000000000..29ecfd815320 --- /dev/null +++ b/drivers/firmware/tegra/ivc.c @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include <soc/tegra/ivc.h> + +#define TEGRA_IVC_ALIGN 64 + +/* + * IVC channel reset protocol. + * + * Each end uses its tx_channel.state to indicate its synchronization state. + */ +enum tegra_ivc_state { + /* + * This value is zero for backwards compatibility with services that + * assume channels to be initially zeroed. Such channels are in an + * initially valid state, but cannot be asynchronously reset, and must + * maintain a valid state at all times. + * + * The transmitting end can enter the established state from the sync or + * ack state when it observes the receiving endpoint in the ack or + * established state, indicating that has cleared the counters in our + * rx_channel. + */ + TEGRA_IVC_STATE_ESTABLISHED = 0, + + /* + * If an endpoint is observed in the sync state, the remote endpoint is + * allowed to clear the counters it owns asynchronously with respect to + * the current endpoint. Therefore, the current endpoint is no longer + * allowed to communicate. + */ + TEGRA_IVC_STATE_SYNC, + + /* + * When the transmitting end observes the receiving end in the sync + * state, it can clear the w_count and r_count and transition to the ack + * state. If the remote endpoint observes us in the ack state, it can + * return to the established state once it has cleared its counters. + */ + TEGRA_IVC_STATE_ACK +}; + +/* + * This structure is divided into two-cache aligned parts, the first is only + * written through the tx.channel pointer, while the second is only written + * through the rx.channel pointer. This delineates ownership of the cache + * lines, which is critical to performance and necessary in non-cache coherent + * implementations. + */ +struct tegra_ivc_header { + union { + struct { + /* fields owned by the transmitting end */ + u32 count; + u32 state; + }; + + u8 pad[TEGRA_IVC_ALIGN]; + } tx; + + union { + /* fields owned by the receiving end */ + u32 count; + u8 pad[TEGRA_IVC_ALIGN]; + } rx; +}; + +static inline void tegra_ivc_invalidate(struct tegra_ivc *ivc, dma_addr_t phys) +{ + if (!ivc->peer) + return; + + dma_sync_single_for_cpu(ivc->peer, phys, TEGRA_IVC_ALIGN, + DMA_FROM_DEVICE); +} + +static inline void tegra_ivc_flush(struct tegra_ivc *ivc, dma_addr_t phys) +{ + if (!ivc->peer) + return; + + dma_sync_single_for_device(ivc->peer, phys, TEGRA_IVC_ALIGN, + DMA_TO_DEVICE); +} + +static inline bool tegra_ivc_empty(struct tegra_ivc *ivc, + struct tegra_ivc_header *header) +{ + /* + * This function performs multiple checks on the same values with + * security implications, so create snapshots with ACCESS_ONCE() to + * ensure that these checks use the same values. + */ + u32 tx = ACCESS_ONCE(header->tx.count); + u32 rx = ACCESS_ONCE(header->rx.count); + + /* + * Perform an over-full check to prevent denial of service attacks + * where a server could be easily fooled into believing that there's + * an extremely large number of frames ready, since receivers are not + * expected to check for full or over-full conditions. + * + * Although the channel isn't empty, this is an invalid case caused by + * a potentially malicious peer, so returning empty is safer, because + * it gives the impression that the channel has gone silent. + */ + if (tx - rx > ivc->num_frames) + return true; + + return tx == rx; +} + +static inline bool tegra_ivc_full(struct tegra_ivc *ivc, + struct tegra_ivc_header *header) +{ + u32 tx = ACCESS_ONCE(header->tx.count); + u32 rx = ACCESS_ONCE(header->rx.count); + + /* + * Invalid cases where the counters indicate that the queue is over + * capacity also appear full. + */ + return tx - rx >= ivc->num_frames; +} + +static inline u32 tegra_ivc_available(struct tegra_ivc *ivc, + struct tegra_ivc_header *header) +{ + u32 tx = ACCESS_ONCE(header->tx.count); + u32 rx = ACCESS_ONCE(header->rx.count); + + /* + * This function isn't expected to be used in scenarios where an + * over-full situation can lead to denial of service attacks. See the + * comment in tegra_ivc_empty() for an explanation about special + * over-full considerations. + */ + return tx - rx; +} + +static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc) +{ + ACCESS_ONCE(ivc->tx.channel->tx.count) = + ACCESS_ONCE(ivc->tx.channel->tx.count) + 1; + + if (ivc->tx.position == ivc->num_frames - 1) + ivc->tx.position = 0; + else + ivc->tx.position++; +} + +static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc) +{ + ACCESS_ONCE(ivc->rx.channel->rx.count) = + ACCESS_ONCE(ivc->rx.channel->rx.count) + 1; + + if (ivc->rx.position == ivc->num_frames - 1) + ivc->rx.position = 0; + else + ivc->rx.position++; +} + +static inline int tegra_ivc_check_read(struct tegra_ivc *ivc) +{ + unsigned int offset = offsetof(struct tegra_ivc_header, tx.count); + + /* + * tx.channel->state is set locally, so it is not synchronized with + * state from the remote peer. The remote peer cannot reset its + * transmit counters until we've acknowledged its synchronization + * request, so no additional synchronization is required because an + * asynchronous transition of rx.channel->state to + * TEGRA_IVC_STATE_ACK is not allowed. + */ + if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED) + return -ECONNRESET; + + /* + * Avoid unnecessary invalidations when performing repeated accesses + * to an IVC channel by checking the old queue pointers first. + * + * Synchronization is only necessary when these pointers indicate + * empty or full. + */ + if (!tegra_ivc_empty(ivc, ivc->rx.channel)) + return 0; + + tegra_ivc_invalidate(ivc, ivc->rx.phys + offset); + + if (tegra_ivc_empty(ivc, ivc->rx.channel)) + return -ENOSPC; + + return 0; +} + +static inline int tegra_ivc_check_write(struct tegra_ivc *ivc) +{ + unsigned int offset = offsetof(struct tegra_ivc_header, rx.count); + + if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED) + return -ECONNRESET; + + if (!tegra_ivc_full(ivc, ivc->tx.channel)) + return 0; + + tegra_ivc_invalidate(ivc, ivc->tx.phys + offset); + + if (tegra_ivc_full(ivc, ivc->tx.channel)) + return -ENOSPC; + + return 0; +} + +static void *tegra_ivc_frame_virt(struct tegra_ivc *ivc, + struct tegra_ivc_header *header, + unsigned int frame) +{ + if (WARN_ON(frame >= ivc->num_frames)) + return ERR_PTR(-EINVAL); + + return (void *)(header + 1) + ivc->frame_size * frame; +} + +static inline dma_addr_t tegra_ivc_frame_phys(struct tegra_ivc *ivc, + dma_addr_t phys, + unsigned int frame) +{ + unsigned long offset; + + offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame; + + return phys + offset; +} + +static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc, + dma_addr_t phys, + unsigned int frame, + unsigned int offset, + size_t size) +{ + if (!ivc->peer || WARN_ON(frame >= ivc->num_frames)) + return; + + phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset; + + dma_sync_single_for_cpu(ivc->peer, phys, size, DMA_FROM_DEVICE); +} + +static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc, + dma_addr_t phys, + unsigned int frame, + unsigned int offset, + size_t size) +{ + if (!ivc->peer || WARN_ON(frame >= ivc->num_frames)) + return; + + phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset; + + dma_sync_single_for_device(ivc->peer, phys, size, DMA_TO_DEVICE); +} + +/* directly peek at the next frame rx'ed */ +void *tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc) +{ + int err; + + if (WARN_ON(ivc == NULL)) + return ERR_PTR(-EINVAL); + + err = tegra_ivc_check_read(ivc); + if (err < 0) + return ERR_PTR(err); + + /* + * Order observation of ivc->rx.position potentially indicating new + * data before data read. + */ + smp_rmb(); + + tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0, + ivc->frame_size); + + return tegra_ivc_frame_virt(ivc, ivc->rx.channel, ivc->rx.position); +} +EXPORT_SYMBOL(tegra_ivc_read_get_next_frame); + +int tegra_ivc_read_advance(struct tegra_ivc *ivc) +{ + unsigned int rx = offsetof(struct tegra_ivc_header, rx.count); + unsigned int tx = offsetof(struct tegra_ivc_header, tx.count); + int err; + + /* + * No read barriers or synchronization here: the caller is expected to + * have already observed the channel non-empty. This check is just to + * catch programming errors. + */ + err = tegra_ivc_check_read(ivc); + if (err < 0) + return err; + + tegra_ivc_advance_rx(ivc); + + tegra_ivc_flush(ivc, ivc->rx.phys + rx); + + /* + * Ensure our write to ivc->rx.position occurs before our read from + * ivc->tx.position. + */ + smp_mb(); + + /* + * Notify only upon transition from full to non-full. The available + * count can only asynchronously increase, so the worst possible + * side-effect will be a spurious notification. + */ + tegra_ivc_invalidate(ivc, ivc->rx.phys + tx); + + if (tegra_ivc_available(ivc, ivc->rx.channel) == ivc->num_frames - 1) + ivc->notify(ivc, ivc->notify_data); + + return 0; +} +EXPORT_SYMBOL(tegra_ivc_read_advance); + +/* directly poke at the next frame to be tx'ed */ +void *tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc) +{ + int err; + + err = tegra_ivc_check_write(ivc); + if (err < 0) + return ERR_PTR(err); + + return tegra_ivc_frame_virt(ivc, ivc->tx.channel, ivc->tx.position); +} +EXPORT_SYMBOL(tegra_ivc_write_get_next_frame); + +/* advance the tx buffer */ +int tegra_ivc_write_advance(struct tegra_ivc *ivc) +{ + unsigned int tx = offsetof(struct tegra_ivc_header, tx.count); + unsigned int rx = offsetof(struct tegra_ivc_header, rx.count); + int err; + + err = tegra_ivc_check_write(ivc); + if (err < 0) + return err; + + tegra_ivc_flush_frame(ivc, ivc->tx.phys, ivc->tx.position, 0, + ivc->frame_size); + + /* + * Order any possible stores to the frame before update of + * ivc->tx.position. + */ + smp_wmb(); + + tegra_ivc_advance_tx(ivc); + tegra_ivc_flush(ivc, ivc->tx.phys + tx); + + /* + * Ensure our write to ivc->tx.position occurs before our read from + * ivc->rx.position. + */ + smp_mb(); + + /* + * Notify only upon transition from empty to non-empty. The available + * count can only asynchronously decrease, so the worst possible + * side-effect will be a spurious notification. + */ + tegra_ivc_invalidate(ivc, ivc->tx.phys + rx); + + if (tegra_ivc_available(ivc, ivc->tx.channel) == 1) + ivc->notify(ivc, ivc->notify_data); + + return 0; +} +EXPORT_SYMBOL(tegra_ivc_write_advance); + +void tegra_ivc_reset(struct tegra_ivc *ivc) +{ + unsigned int offset = offsetof(struct tegra_ivc_header, tx.count); + + ivc->tx.channel->tx.state = TEGRA_IVC_STATE_SYNC; + tegra_ivc_flush(ivc, ivc->tx.phys + offset); + ivc->notify(ivc, ivc->notify_data); +} +EXPORT_SYMBOL(tegra_ivc_reset); + +/* + * ======================================================= + * IVC State Transition Table - see tegra_ivc_notified() + * ======================================================= + * + * local remote action + * ----- ------ ----------------------------------- + * SYNC EST <none> + * SYNC ACK reset counters; move to EST; notify + * SYNC SYNC reset counters; move to ACK; notify + * ACK EST move to EST; notify + * ACK ACK move to EST; notify + * ACK SYNC reset counters; move to ACK; notify + * EST EST <none> + * EST ACK <none> + * EST SYNC reset counters; move to ACK; notify + * + * =============================================================== + */ + +int tegra_ivc_notified(struct tegra_ivc *ivc) +{ + unsigned int offset = offsetof(struct tegra_ivc_header, tx.count); + enum tegra_ivc_state state; + + /* Copy the receiver's state out of shared memory. */ + tegra_ivc_invalidate(ivc, ivc->rx.phys + offset); + state = ACCESS_ONCE(ivc->rx.channel->tx.state); + + if (state == TEGRA_IVC_STATE_SYNC) { + offset = offsetof(struct tegra_ivc_header, tx.count); + + /* + * Order observation of TEGRA_IVC_STATE_SYNC before stores + * clearing tx.channel. + */ + smp_rmb(); + + /* + * Reset tx.channel counters. The remote end is in the SYNC + * state and won't make progress until we change our state, + * so the counters are not in use at this time. + */ + ivc->tx.channel->tx.count = 0; + ivc->rx.channel->rx.count = 0; + + ivc->tx.position = 0; + ivc->rx.position = 0; + + /* + * Ensure that counters appear cleared before new state can be + * observed. + */ + smp_wmb(); + + /* + * Move to ACK state. We have just cleared our counters, so it + * is now safe for the remote end to start using these values. + */ + ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ACK; + tegra_ivc_flush(ivc, ivc->tx.phys + offset); + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc, ivc->notify_data); + + } else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_SYNC && + state == TEGRA_IVC_STATE_ACK) { + offset = offsetof(struct tegra_ivc_header, tx.count); + + /* + * Order observation of ivc_state_sync before stores clearing + * tx_channel. + */ + smp_rmb(); + + /* + * Reset tx.channel counters. The remote end is in the ACK + * state and won't make progress until we change our state, + * so the counters are not in use at this time. + */ + ivc->tx.channel->tx.count = 0; + ivc->rx.channel->rx.count = 0; + + ivc->tx.position = 0; + ivc->rx.position = 0; + + /* + * Ensure that counters appear cleared before new state can be + * observed. + */ + smp_wmb(); + + /* + * Move to ESTABLISHED state. We know that the remote end has + * already cleared its counters, so it is safe to start + * writing/reading on this channel. + */ + ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED; + tegra_ivc_flush(ivc, ivc->tx.phys + offset); + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc, ivc->notify_data); + + } else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_ACK) { + offset = offsetof(struct tegra_ivc_header, tx.count); + + /* + * At this point, we have observed the peer to be in either + * the ACK or ESTABLISHED state. Next, order observation of + * peer state before storing to tx.channel. + */ + smp_rmb(); + + /* + * Move to ESTABLISHED state. We know that we have previously + * cleared our counters, and we know that the remote end has + * cleared its counters, so it is safe to start writing/reading + * on this channel. + */ + ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED; + tegra_ivc_flush(ivc, ivc->tx.phys + offset); + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc, ivc->notify_data); + + } else { + /* + * There is no need to handle any further action. Either the + * channel is already fully established, or we are waiting for + * the remote end to catch up with our current state. Refer + * to the diagram in "IVC State Transition Table" above. + */ + } + + if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED) + return -EAGAIN; + + return 0; +} +EXPORT_SYMBOL(tegra_ivc_notified); + +size_t tegra_ivc_align(size_t size) +{ + return ALIGN(size, TEGRA_IVC_ALIGN); +} +EXPORT_SYMBOL(tegra_ivc_align); + +unsigned tegra_ivc_total_queue_size(unsigned queue_size) +{ + if (!IS_ALIGNED(queue_size, TEGRA_IVC_ALIGN)) { + pr_err("%s: queue_size (%u) must be %u-byte aligned\n", + __func__, queue_size, TEGRA_IVC_ALIGN); + return 0; + } + + return queue_size + sizeof(struct tegra_ivc_header); +} +EXPORT_SYMBOL(tegra_ivc_total_queue_size); + +static int tegra_ivc_check_params(unsigned long rx, unsigned long tx, + unsigned int num_frames, size_t frame_size) +{ + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, tx.count), + TEGRA_IVC_ALIGN)); + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, rx.count), + TEGRA_IVC_ALIGN)); + BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct tegra_ivc_header), + TEGRA_IVC_ALIGN)); + + if ((uint64_t)num_frames * (uint64_t)frame_size >= 0x100000000UL) { + pr_err("num_frames * frame_size overflows\n"); + return -EINVAL; + } + + if (!IS_ALIGNED(frame_size, TEGRA_IVC_ALIGN)) { + pr_err("frame size not adequately aligned: %zu\n", frame_size); + return -EINVAL; + } + + /* + * The headers must at least be aligned enough for counters + * to be accessed atomically. + */ + if (!IS_ALIGNED(rx, TEGRA_IVC_ALIGN)) { + pr_err("IVC channel start not aligned: %#lx\n", rx); + return -EINVAL; + } + + if (!IS_ALIGNED(tx, TEGRA_IVC_ALIGN)) { + pr_err("IVC channel start not aligned: %#lx\n", tx); + return -EINVAL; + } + + if (rx < tx) { + if (rx + frame_size * num_frames > tx) { + pr_err("queue regions overlap: %#lx + %zx > %#lx\n", + rx, frame_size * num_frames, tx); + return -EINVAL; + } + } else { + if (tx + frame_size * num_frames > rx) { + pr_err("queue regions overlap: %#lx + %zx > %#lx\n", + tx, frame_size * num_frames, rx); + return -EINVAL; + } + } + + return 0; +} + +int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, void *rx, + dma_addr_t rx_phys, void *tx, dma_addr_t tx_phys, + unsigned int num_frames, size_t frame_size, + void (*notify)(struct tegra_ivc *ivc, void *data), + void *data) +{ + size_t queue_size; + int err; + + if (WARN_ON(!ivc || !notify)) + return -EINVAL; + + /* + * All sizes that can be returned by communication functions should + * fit in an int. + */ + if (frame_size > INT_MAX) + return -E2BIG; + + err = tegra_ivc_check_params((unsigned long)rx, (unsigned long)tx, + num_frames, frame_size); + if (err < 0) + return err; + + queue_size = tegra_ivc_total_queue_size(num_frames * frame_size); + + if (peer) { + ivc->rx.phys = dma_map_single(peer, rx, queue_size, + DMA_BIDIRECTIONAL); + if (ivc->rx.phys == DMA_ERROR_CODE) + return -ENOMEM; + + ivc->tx.phys = dma_map_single(peer, tx, queue_size, + DMA_BIDIRECTIONAL); + if (ivc->tx.phys == DMA_ERROR_CODE) { + dma_unmap_single(peer, ivc->rx.phys, queue_size, + DMA_BIDIRECTIONAL); + return -ENOMEM; + } + } else { + ivc->rx.phys = rx_phys; + ivc->tx.phys = tx_phys; + } + + ivc->rx.channel = rx; + ivc->tx.channel = tx; + ivc->peer = peer; + ivc->notify = notify; + ivc->notify_data = data; + ivc->frame_size = frame_size; + ivc->num_frames = num_frames; + + /* + * These values aren't necessarily correct until the channel has been + * reset. + */ + ivc->tx.position = 0; + ivc->rx.position = 0; + + return 0; +} +EXPORT_SYMBOL(tegra_ivc_init); + +void tegra_ivc_cleanup(struct tegra_ivc *ivc) +{ + if (ivc->peer) { + size_t size = tegra_ivc_total_queue_size(ivc->num_frames * + ivc->frame_size); + + dma_unmap_single(ivc->peer, ivc->rx.phys, size, + DMA_BIDIRECTIONAL); + dma_unmap_single(ivc->peer, ivc->tx.phys, size, + DMA_BIDIRECTIONAL); + } +} +EXPORT_SYMBOL(tegra_ivc_cleanup); |