diff options
Diffstat (limited to 'drivers/char')
27 files changed, 3258 insertions, 84 deletions
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 6e9f74a5c095..efefd12a0f7b 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -600,5 +600,7 @@ config TILE_SROM device appear much like a simple EEPROM, and knows how to partition a single ROM for multiple purposes. +source "drivers/char/xillybus/Kconfig" + endmenu diff --git a/drivers/char/Makefile b/drivers/char/Makefile index a324f9303e36..d06cde26031b 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -61,3 +61,4 @@ obj-$(CONFIG_JS_RTC) += js-rtc.o js-rtc-y = rtc.o obj-$(CONFIG_TILE_SROM) += tile-srom.o +obj-$(CONFIG_XILLYBUS) += xillybus/ diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 836b061ced35..91a04ae8003c 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -333,6 +333,19 @@ config HW_RANDOM_MSM If unsure, say Y. +config HW_RANDOM_XGENE + tristate "APM X-Gene True Random Number Generator (TRNG) support" + depends on HW_RANDOM && ARCH_XGENE + default HW_RANDOM + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on APM X-Gene SoC. + + To compile this driver as a module, choose M here: the + module will be called xgene_rng. + + If unsure, say Y. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 199ed283e149..0b4cd57f4e24 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -29,3 +29,4 @@ obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o +obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c index c6af038682f1..48f6a83cdd61 100644 --- a/drivers/char/hw_random/amd-rng.c +++ b/drivers/char/hw_random/amd-rng.c @@ -142,10 +142,10 @@ found: amd_rng.priv = (unsigned long)pmbase; amd_pdev = pdev; - printk(KERN_INFO "AMD768 RNG detected\n"); + pr_info("AMD768 RNG detected\n"); err = hwrng_register(&amd_rng); if (err) { - printk(KERN_ERR PFX "RNG registering failed (%d)\n", + pr_err(PFX "RNG registering failed (%d)\n", err); release_region(pmbase + 0xF0, 8); goto out; diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c index 4c4d4e140f98..0d0579fe465e 100644 --- a/drivers/char/hw_random/geode-rng.c +++ b/drivers/char/hw_random/geode-rng.c @@ -109,10 +109,10 @@ found: goto out; geode_rng.priv = (unsigned long)mem; - printk(KERN_INFO "AMD Geode RNG detected\n"); + pr_info("AMD Geode RNG detected\n"); err = hwrng_register(&geode_rng); if (err) { - printk(KERN_ERR PFX "RNG registering failed (%d)\n", + pr_err(PFX "RNG registering failed (%d)\n", err); goto err_unmap; } diff --git a/drivers/char/hw_random/intel-rng.c b/drivers/char/hw_random/intel-rng.c index 86fe45c19968..290c880266bf 100644 --- a/drivers/char/hw_random/intel-rng.c +++ b/drivers/char/hw_random/intel-rng.c @@ -199,7 +199,7 @@ static int intel_rng_init(struct hwrng *rng) if ((hw_status & INTEL_RNG_ENABLED) == 0) hw_status = hwstatus_set(mem, hw_status | INTEL_RNG_ENABLED); if ((hw_status & INTEL_RNG_ENABLED) == 0) { - printk(KERN_ERR PFX "cannot enable RNG, aborting\n"); + pr_err(PFX "cannot enable RNG, aborting\n"); goto out; } err = 0; @@ -216,7 +216,7 @@ static void intel_rng_cleanup(struct hwrng *rng) if (hw_status & INTEL_RNG_ENABLED) hwstatus_set(mem, hw_status & ~INTEL_RNG_ENABLED); else - printk(KERN_WARNING PFX "unusual: RNG already disabled\n"); + pr_warn(PFX "unusual: RNG already disabled\n"); } @@ -274,7 +274,7 @@ static int __init intel_rng_hw_init(void *_intel_rng_hw) if (mfc != INTEL_FWH_MANUFACTURER_CODE || (dvc != INTEL_FWH_DEVICE_CODE_8M && dvc != INTEL_FWH_DEVICE_CODE_4M)) { - printk(KERN_NOTICE PFX "FWH not detected\n"); + pr_notice(PFX "FWH not detected\n"); return -ENODEV; } @@ -306,7 +306,6 @@ static int __init intel_init_hw_struct(struct intel_rng_hw *intel_rng_hw, (BIOS_CNTL_LOCK_ENABLE_MASK|BIOS_CNTL_WRITE_ENABLE_MASK)) == BIOS_CNTL_LOCK_ENABLE_MASK) { static __initdata /*const*/ char warning[] = - KERN_WARNING PFX "Firmware space is locked read-only. If you can't or\n" PFX "don't want to disable this in firmware setup, and if\n" PFX "you are certain that your system has a functional\n" @@ -314,7 +313,7 @@ PFX "RNG, try using the 'no_fwh_detect' option.\n"; if (no_fwh_detect) return -ENODEV; - printk(warning); + pr_warn("%s", warning); return -EBUSY; } @@ -392,10 +391,10 @@ fwh_done: goto out; } - printk(KERN_INFO "Intel 82802 RNG detected\n"); + pr_info("Intel 82802 RNG detected\n"); err = hwrng_register(&intel_rng); if (err) { - printk(KERN_ERR PFX "RNG registering failed (%d)\n", + pr_err(PFX "RNG registering failed (%d)\n", err); iounmap(mem); } diff --git a/drivers/char/hw_random/pasemi-rng.c b/drivers/char/hw_random/pasemi-rng.c index c66279bb6ef3..c0347d1dded0 100644 --- a/drivers/char/hw_random/pasemi-rng.c +++ b/drivers/char/hw_random/pasemi-rng.c @@ -113,7 +113,7 @@ static int rng_probe(struct platform_device *ofdev) pasemi_rng.priv = (unsigned long)rng_regs; - printk(KERN_INFO "Registering PA Semi RNG\n"); + pr_info("Registering PA Semi RNG\n"); err = hwrng_register(&pasemi_rng); diff --git a/drivers/char/hw_random/pseries-rng.c b/drivers/char/hw_random/pseries-rng.c index ab7ffdec0ec3..6226aa08c36a 100644 --- a/drivers/char/hw_random/pseries-rng.c +++ b/drivers/char/hw_random/pseries-rng.c @@ -86,7 +86,7 @@ static struct vio_driver pseries_rng_driver = { static int __init rng_init(void) { - printk(KERN_INFO "Registering IBM pSeries RNG driver\n"); + pr_info("Registering IBM pSeries RNG driver\n"); return vio_register_driver(&pseries_rng_driver); } diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c index de5a6dcfb3e2..a3bebef255ad 100644 --- a/drivers/char/hw_random/via-rng.c +++ b/drivers/char/hw_random/via-rng.c @@ -141,7 +141,7 @@ static int via_rng_init(struct hwrng *rng) * register */ if ((c->x86 == 6) && (c->x86_model >= 0x0f)) { if (!cpu_has_xstore_enabled) { - printk(KERN_ERR PFX "can't enable hardware RNG " + pr_err(PFX "can't enable hardware RNG " "if XSTORE is not enabled\n"); return -ENODEV; } @@ -180,7 +180,7 @@ static int via_rng_init(struct hwrng *rng) unneeded */ rdmsr(MSR_VIA_RNG, lo, hi); if ((lo & VIA_RNG_ENABLE) == 0) { - printk(KERN_ERR PFX "cannot enable VIA C3 RNG, aborting\n"); + pr_err(PFX "cannot enable VIA C3 RNG, aborting\n"); return -ENODEV; } @@ -202,10 +202,10 @@ static int __init mod_init(void) if (!cpu_has_xstore) return -ENODEV; - printk(KERN_INFO "VIA RNG detected\n"); + pr_info("VIA RNG detected\n"); err = hwrng_register(&via_rng); if (err) { - printk(KERN_ERR PFX "RNG registering failed (%d)\n", + pr_err(PFX "RNG registering failed (%d)\n", err); goto out; } diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 2e3139eda93b..72295ea2fd1c 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -36,6 +36,7 @@ struct virtrng_info { int index; bool busy; bool hwrng_register_done; + bool hwrng_removed; }; @@ -68,6 +69,9 @@ static int virtio_read(struct hwrng *rng, void *buf, size_t size, bool wait) int ret; struct virtrng_info *vi = (struct virtrng_info *)rng->priv; + if (vi->hwrng_removed) + return -ENODEV; + if (!vi->busy) { vi->busy = true; init_completion(&vi->have_data); @@ -105,8 +109,8 @@ static int probe_common(struct virtio_device *vdev) vi->index = index = ida_simple_get(&rng_index_ida, 0, 0, GFP_KERNEL); if (index < 0) { - kfree(vi); - return index; + err = index; + goto err_ida; } sprintf(vi->name, "virtio_rng.%d", index); init_completion(&vi->have_data); @@ -124,19 +128,25 @@ static int probe_common(struct virtio_device *vdev) vi->vq = virtio_find_single_vq(vdev, random_recv_done, "input"); if (IS_ERR(vi->vq)) { err = PTR_ERR(vi->vq); - vi->vq = NULL; - kfree(vi); - ida_simple_remove(&rng_index_ida, index); - return err; + goto err_find; } return 0; + +err_find: + ida_simple_remove(&rng_index_ida, index); +err_ida: + kfree(vi); + return err; } static void remove_common(struct virtio_device *vdev) { struct virtrng_info *vi = vdev->priv; + vi->hwrng_removed = true; + vi->data_avail = 0; + complete(&vi->have_data); vdev->config->reset(vdev); vi->busy = false; if (vi->hwrng_register_done) diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c new file mode 100644 index 000000000000..23caa05380a8 --- /dev/null +++ b/drivers/char/hw_random/xgene-rng.c @@ -0,0 +1,423 @@ +/* + * APM X-Gene SoC RNG Driver + * + * Copyright (c) 2014, Applied Micro Circuits Corporation + * Author: Rameshwar Prasad Sahu <rsahu@apm.com> + * Shamal Winchurkar <swinchurkar@apm.com> + * Feng Kan <fkan@apm.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/hw_random.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/timer.h> + +#define RNG_MAX_DATUM 4 +#define MAX_TRY 100 +#define XGENE_RNG_RETRY_COUNT 20 +#define XGENE_RNG_RETRY_INTERVAL 10 + +/* RNG Registers */ +#define RNG_INOUT_0 0x00 +#define RNG_INTR_STS_ACK 0x10 +#define RNG_CONTROL 0x14 +#define RNG_CONFIG 0x18 +#define RNG_ALARMCNT 0x1c +#define RNG_FROENABLE 0x20 +#define RNG_FRODETUNE 0x24 +#define RNG_ALARMMASK 0x28 +#define RNG_ALARMSTOP 0x2c +#define RNG_OPTIONS 0x78 +#define RNG_EIP_REV 0x7c + +#define MONOBIT_FAIL_MASK BIT(7) +#define POKER_FAIL_MASK BIT(6) +#define LONG_RUN_FAIL_MASK BIT(5) +#define RUN_FAIL_MASK BIT(4) +#define NOISE_FAIL_MASK BIT(3) +#define STUCK_OUT_MASK BIT(2) +#define SHUTDOWN_OFLO_MASK BIT(1) +#define READY_MASK BIT(0) + +#define MAJOR_HW_REV_RD(src) (((src) & 0x0f000000) >> 24) +#define MINOR_HW_REV_RD(src) (((src) & 0x00f00000) >> 20) +#define HW_PATCH_LEVEL_RD(src) (((src) & 0x000f0000) >> 16) +#define MAX_REFILL_CYCLES_SET(dst, src) \ + ((dst & ~0xffff0000) | (((u32)src << 16) & 0xffff0000)) +#define MIN_REFILL_CYCLES_SET(dst, src) \ + ((dst & ~0x000000ff) | (((u32)src) & 0x000000ff)) +#define ALARM_THRESHOLD_SET(dst, src) \ + ((dst & ~0x000000ff) | (((u32)src) & 0x000000ff)) +#define ENABLE_RNG_SET(dst, src) \ + ((dst & ~BIT(10)) | (((u32)src << 10) & BIT(10))) +#define REGSPEC_TEST_MODE_SET(dst, src) \ + ((dst & ~BIT(8)) | (((u32)src << 8) & BIT(8))) +#define MONOBIT_FAIL_MASK_SET(dst, src) \ + ((dst & ~BIT(7)) | (((u32)src << 7) & BIT(7))) +#define POKER_FAIL_MASK_SET(dst, src) \ + ((dst & ~BIT(6)) | (((u32)src << 6) & BIT(6))) +#define LONG_RUN_FAIL_MASK_SET(dst, src) \ + ((dst & ~BIT(5)) | (((u32)src << 5) & BIT(5))) +#define RUN_FAIL_MASK_SET(dst, src) \ + ((dst & ~BIT(4)) | (((u32)src << 4) & BIT(4))) +#define NOISE_FAIL_MASK_SET(dst, src) \ + ((dst & ~BIT(3)) | (((u32)src << 3) & BIT(3))) +#define STUCK_OUT_MASK_SET(dst, src) \ + ((dst & ~BIT(2)) | (((u32)src << 2) & BIT(2))) +#define SHUTDOWN_OFLO_MASK_SET(dst, src) \ + ((dst & ~BIT(1)) | (((u32)src << 1) & BIT(1))) + +struct xgene_rng_dev { + u32 irq; + void __iomem *csr_base; + u32 revision; + u32 datum_size; + u32 failure_cnt; /* Failure count last minute */ + unsigned long failure_ts;/* First failure timestamp */ + struct timer_list failure_timer; + struct device *dev; + struct clk *clk; +}; + +static void xgene_rng_expired_timer(unsigned long arg) +{ + struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) arg; + + /* Clear failure counter as timer expired */ + disable_irq(ctx->irq); + ctx->failure_cnt = 0; + del_timer(&ctx->failure_timer); + enable_irq(ctx->irq); +} + +static void xgene_rng_start_timer(struct xgene_rng_dev *ctx) +{ + ctx->failure_timer.data = (unsigned long) ctx; + ctx->failure_timer.function = xgene_rng_expired_timer; + ctx->failure_timer.expires = jiffies + 120 * HZ; + add_timer(&ctx->failure_timer); +} + +/* + * Initialize or reinit free running oscillators (FROs) + */ +static void xgene_rng_init_fro(struct xgene_rng_dev *ctx, u32 fro_val) +{ + writel(fro_val, ctx->csr_base + RNG_FRODETUNE); + writel(0x00000000, ctx->csr_base + RNG_ALARMMASK); + writel(0x00000000, ctx->csr_base + RNG_ALARMSTOP); + writel(0xFFFFFFFF, ctx->csr_base + RNG_FROENABLE); +} + +static void xgene_rng_chk_overflow(struct xgene_rng_dev *ctx) +{ + u32 val; + + val = readl(ctx->csr_base + RNG_INTR_STS_ACK); + if (val & MONOBIT_FAIL_MASK) + /* + * LFSR detected an out-of-bounds number of 1s after + * checking 20,000 bits (test T1 as specified in the + * AIS-31 standard) + */ + dev_err(ctx->dev, "test monobit failure error 0x%08X\n", val); + if (val & POKER_FAIL_MASK) + /* + * LFSR detected an out-of-bounds value in at least one + * of the 16 poker_count_X counters or an out of bounds sum + * of squares value after checking 20,000 bits (test T2 as + * specified in the AIS-31 standard) + */ + dev_err(ctx->dev, "test poker failure error 0x%08X\n", val); + if (val & LONG_RUN_FAIL_MASK) + /* + * LFSR detected a sequence of 34 identical bits + * (test T4 as specified in the AIS-31 standard) + */ + dev_err(ctx->dev, "test long run failure error 0x%08X\n", val); + if (val & RUN_FAIL_MASK) + /* + * LFSR detected an outof-bounds value for at least one + * of the running counters after checking 20,000 bits + * (test T3 as specified in the AIS-31 standard) + */ + dev_err(ctx->dev, "test run failure error 0x%08X\n", val); + if (val & NOISE_FAIL_MASK) + /* LFSR detected a sequence of 48 identical bits */ + dev_err(ctx->dev, "noise failure error 0x%08X\n", val); + if (val & STUCK_OUT_MASK) + /* + * Detected output data registers generated same value twice + * in a row + */ + dev_err(ctx->dev, "stuck out failure error 0x%08X\n", val); + + if (val & SHUTDOWN_OFLO_MASK) { + u32 frostopped; + + /* FROs shut down after a second error event. Try recover. */ + if (++ctx->failure_cnt == 1) { + /* 1st time, just recover */ + ctx->failure_ts = jiffies; + frostopped = readl(ctx->csr_base + RNG_ALARMSTOP); + xgene_rng_init_fro(ctx, frostopped); + + /* + * We must start a timer to clear out this error + * in case the system timer wrap around + */ + xgene_rng_start_timer(ctx); + } else { + /* 2nd time failure in lesser than 1 minute? */ + if (time_after(ctx->failure_ts + 60 * HZ, jiffies)) { + dev_err(ctx->dev, + "FRO shutdown failure error 0x%08X\n", + val); + } else { + /* 2nd time failure after 1 minutes, recover */ + ctx->failure_ts = jiffies; + ctx->failure_cnt = 1; + /* + * We must start a timer to clear out this + * error in case the system timer wrap + * around + */ + xgene_rng_start_timer(ctx); + } + frostopped = readl(ctx->csr_base + RNG_ALARMSTOP); + xgene_rng_init_fro(ctx, frostopped); + } + } + /* Clear them all */ + writel(val, ctx->csr_base + RNG_INTR_STS_ACK); +} + +static irqreturn_t xgene_rng_irq_handler(int irq, void *id) +{ + struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) id; + + /* RNG Alarm Counter overflow */ + xgene_rng_chk_overflow(ctx); + + return IRQ_HANDLED; +} + +static int xgene_rng_data_present(struct hwrng *rng, int wait) +{ + struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv; + u32 i, val = 0; + + for (i = 0; i < XGENE_RNG_RETRY_COUNT; i++) { + val = readl(ctx->csr_base + RNG_INTR_STS_ACK); + if ((val & READY_MASK) || !wait) + break; + udelay(XGENE_RNG_RETRY_INTERVAL); + } + + return (val & READY_MASK); +} + +static int xgene_rng_data_read(struct hwrng *rng, u32 *data) +{ + struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv; + int i; + + for (i = 0; i < ctx->datum_size; i++) + data[i] = readl(ctx->csr_base + RNG_INOUT_0 + i * 4); + + /* Clear ready bit to start next transaction */ + writel(READY_MASK, ctx->csr_base + RNG_INTR_STS_ACK); + + return ctx->datum_size << 2; +} + +static void xgene_rng_init_internal(struct xgene_rng_dev *ctx) +{ + u32 val; + + writel(0x00000000, ctx->csr_base + RNG_CONTROL); + + val = MAX_REFILL_CYCLES_SET(0, 10); + val = MIN_REFILL_CYCLES_SET(val, 10); + writel(val, ctx->csr_base + RNG_CONFIG); + + val = ALARM_THRESHOLD_SET(0, 0xFF); + writel(val, ctx->csr_base + RNG_ALARMCNT); + + xgene_rng_init_fro(ctx, 0); + + writel(MONOBIT_FAIL_MASK | + POKER_FAIL_MASK | + LONG_RUN_FAIL_MASK | + RUN_FAIL_MASK | + NOISE_FAIL_MASK | + STUCK_OUT_MASK | + SHUTDOWN_OFLO_MASK | + READY_MASK, ctx->csr_base + RNG_INTR_STS_ACK); + + val = ENABLE_RNG_SET(0, 1); + val = MONOBIT_FAIL_MASK_SET(val, 1); + val = POKER_FAIL_MASK_SET(val, 1); + val = LONG_RUN_FAIL_MASK_SET(val, 1); + val = RUN_FAIL_MASK_SET(val, 1); + val = NOISE_FAIL_MASK_SET(val, 1); + val = STUCK_OUT_MASK_SET(val, 1); + val = SHUTDOWN_OFLO_MASK_SET(val, 1); + writel(val, ctx->csr_base + RNG_CONTROL); +} + +static int xgene_rng_init(struct hwrng *rng) +{ + struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv; + + ctx->failure_cnt = 0; + init_timer(&ctx->failure_timer); + + ctx->revision = readl(ctx->csr_base + RNG_EIP_REV); + + dev_dbg(ctx->dev, "Rev %d.%d.%d\n", + MAJOR_HW_REV_RD(ctx->revision), + MINOR_HW_REV_RD(ctx->revision), + HW_PATCH_LEVEL_RD(ctx->revision)); + + dev_dbg(ctx->dev, "Options 0x%08X", + readl(ctx->csr_base + RNG_OPTIONS)); + + xgene_rng_init_internal(ctx); + + ctx->datum_size = RNG_MAX_DATUM; + + return 0; +} + +static struct hwrng xgene_rng_func = { + .name = "xgene-rng", + .init = xgene_rng_init, + .data_present = xgene_rng_data_present, + .data_read = xgene_rng_data_read, +}; + +static int xgene_rng_probe(struct platform_device *pdev) +{ + struct resource *res; + struct xgene_rng_dev *ctx; + int rc = 0; + + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->dev = &pdev->dev; + platform_set_drvdata(pdev, ctx); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ctx->csr_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ctx->csr_base)) + return PTR_ERR(ctx->csr_base); + + ctx->irq = platform_get_irq(pdev, 0); + if (ctx->irq < 0) { + dev_err(&pdev->dev, "No IRQ resource\n"); + return ctx->irq; + } + + dev_dbg(&pdev->dev, "APM X-Gene RNG BASE %p ALARM IRQ %d", + ctx->csr_base, ctx->irq); + + rc = devm_request_irq(&pdev->dev, ctx->irq, xgene_rng_irq_handler, 0, + dev_name(&pdev->dev), ctx); + if (rc) { + dev_err(&pdev->dev, "Could not request RNG alarm IRQ\n"); + return rc; + } + + /* Enable IP clock */ + ctx->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(ctx->clk)) { + dev_warn(&pdev->dev, "Couldn't get the clock for RNG\n"); + } else { + rc = clk_prepare_enable(ctx->clk); + if (rc) { + dev_warn(&pdev->dev, + "clock prepare enable failed for RNG"); + return rc; + } + } + + xgene_rng_func.priv = (unsigned long) ctx; + + rc = hwrng_register(&xgene_rng_func); + if (rc) { + dev_err(&pdev->dev, "RNG registering failed error %d\n", rc); + if (!IS_ERR(ctx->clk)) + clk_disable_unprepare(ctx->clk); + return rc; + } + + rc = device_init_wakeup(&pdev->dev, 1); + if (rc) { + dev_err(&pdev->dev, "RNG device_init_wakeup failed error %d\n", + rc); + if (!IS_ERR(ctx->clk)) + clk_disable_unprepare(ctx->clk); + hwrng_unregister(&xgene_rng_func); + return rc; + } + + return 0; +} + +static int xgene_rng_remove(struct platform_device *pdev) +{ + struct xgene_rng_dev *ctx = platform_get_drvdata(pdev); + int rc; + + rc = device_init_wakeup(&pdev->dev, 0); + if (rc) + dev_err(&pdev->dev, "RNG init wakeup failed error %d\n", rc); + if (!IS_ERR(ctx->clk)) + clk_disable_unprepare(ctx->clk); + hwrng_unregister(&xgene_rng_func); + + return rc; +} + +static const struct of_device_id xgene_rng_of_match[] = { + { .compatible = "apm,xgene-rng" }, + { } +}; + +MODULE_DEVICE_TABLE(of, xgene_rng_of_match); + +static struct platform_driver xgene_rng_driver = { + .probe = xgene_rng_probe, + .remove = xgene_rng_remove, + .driver = { + .name = "xgene-rng", + .of_match_table = xgene_rng_of_match, + }, +}; + +module_platform_driver(xgene_rng_driver); +MODULE_DESCRIPTION("APM X-Gene RNG driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/char/i8k.c b/drivers/char/i8k.c index 65525c7e903c..34174d01462e 100644 --- a/drivers/char/i8k.c +++ b/drivers/char/i8k.c @@ -651,6 +651,7 @@ struct i8k_config_data { enum i8k_configs { DELL_LATITUDE_D520, + DELL_LATITUDE_E6540, DELL_PRECISION_490, DELL_STUDIO, DELL_XPS_M140, @@ -661,6 +662,10 @@ static const struct i8k_config_data i8k_config_data[] = { .fan_mult = 1, .fan_max = I8K_FAN_TURBO, }, + [DELL_LATITUDE_E6540] = { + .fan_mult = 1, + .fan_max = I8K_FAN_HIGH, + }, [DELL_PRECISION_490] = { .fan_mult = 1, .fan_max = I8K_FAN_TURBO, @@ -706,6 +711,14 @@ static struct dmi_system_id i8k_dmi_table[] __initdata = { .driver_data = (void *)&i8k_config_data[DELL_LATITUDE_D520], }, { + .ident = "Dell Latitude E6540", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E6540"), + }, + .driver_data = (void *)&i8k_config_data[DELL_LATITUDE_E6540], + }, + { .ident = "Dell Latitude 2", .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index e6db9381b2c7..f816211f062f 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -2796,7 +2796,6 @@ channel_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg) = IPMI_CHANNEL_MEDIUM_IPMB; intf->channels[0].protocol = IPMI_CHANNEL_PROTOCOL_IPMB; - rv = -ENOSYS; intf->curr_channel = IPMI_MAX_CHANNELS; wake_up(&intf->waitq); @@ -2821,12 +2820,12 @@ channel_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg) if (rv) { /* Got an error somehow, just give up. */ + printk(KERN_WARNING PFX + "Error sending channel information for channel" + " %d: %d\n", intf->curr_channel, rv); + intf->curr_channel = IPMI_MAX_CHANNELS; wake_up(&intf->waitq); - - printk(KERN_WARNING PFX - "Error sending channel information: %d\n", - rv); } } out: @@ -2964,8 +2963,12 @@ int ipmi_register_smi(struct ipmi_smi_handlers *handlers, intf->null_user_handler = channel_handler; intf->curr_channel = 0; rv = send_channel_info_cmd(intf, 0); - if (rv) + if (rv) { + printk(KERN_WARNING PFX + "Error sending channel information for channel" + " 0, %d\n", rv); goto out; + } /* Wait for the channel info to be read. */ wait_event(intf->waitq, diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 5d665680ae33..5c4e1f625bbb 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -965,9 +965,9 @@ static inline int ipmi_si_is_busy(struct timespec *ts) return ts->tv_nsec != -1; } -static int ipmi_thread_busy_wait(enum si_sm_result smi_result, - const struct smi_info *smi_info, - struct timespec *busy_until) +static inline int ipmi_thread_busy_wait(enum si_sm_result smi_result, + const struct smi_info *smi_info, + struct timespec *busy_until) { unsigned int max_busy_us = 0; @@ -2658,6 +2658,9 @@ static int ipmi_probe(struct platform_device *dev) if (!match) return -EINVAL; + if (!of_device_is_available(np)) + return -EINVAL; + ret = of_address_to_resource(np, 0, &resource); if (ret) { dev_warn(&dev->dev, PFX "invalid address from OF\n"); @@ -3655,6 +3658,9 @@ static void cleanup_one_si(struct smi_info *to_clean) if (!to_clean) return; + if (to_clean->dev) + dev_set_drvdata(to_clean->dev, NULL); + list_del(&to_clean->link); /* Tell the driver that we are shutting down. */ diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 917403fe10da..524b707894ef 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -622,53 +622,23 @@ static ssize_t splice_write_null(struct pipe_inode_info *pipe, struct file *out, return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null); } -static ssize_t read_zero(struct file *file, char __user *buf, - size_t count, loff_t *ppos) +static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter) { - size_t written; - - if (!count) - return 0; - - if (!access_ok(VERIFY_WRITE, buf, count)) - return -EFAULT; - - written = 0; - while (count) { - unsigned long unwritten; - size_t chunk = count; + size_t written = 0; + while (iov_iter_count(iter)) { + size_t chunk = iov_iter_count(iter), n; if (chunk > PAGE_SIZE) chunk = PAGE_SIZE; /* Just for latency reasons */ - unwritten = __clear_user(buf, chunk); - written += chunk - unwritten; - if (unwritten) - break; + n = iov_iter_zero(chunk, iter); + if (!n && iov_iter_count(iter)) + return written ? written : -EFAULT; + written += n; if (signal_pending(current)) return written ? written : -ERESTARTSYS; - buf += chunk; - count -= chunk; cond_resched(); } - return written ? written : -EFAULT; -} - -static ssize_t aio_read_zero(struct kiocb *iocb, const struct iovec *iov, - unsigned long nr_segs, loff_t pos) -{ - size_t written = 0; - unsigned long i; - ssize_t ret; - - for (i = 0; i < nr_segs; i++) { - ret = read_zero(iocb->ki_filp, iov[i].iov_base, iov[i].iov_len, - &pos); - if (ret < 0) - break; - written += ret; - } - - return written ? written : -EFAULT; + return written; } static int mmap_zero(struct file *file, struct vm_area_struct *vma) @@ -738,7 +708,6 @@ static int open_port(struct inode *inode, struct file *filp) #define zero_lseek null_lseek #define full_lseek null_lseek #define write_zero write_null -#define read_full read_zero #define aio_write_zero aio_write_null #define open_mem open_port #define open_kmem open_mem @@ -783,9 +752,9 @@ static const struct file_operations port_fops = { static const struct file_operations zero_fops = { .llseek = zero_lseek, - .read = read_zero, + .read = new_sync_read, .write = write_zero, - .aio_read = aio_read_zero, + .read_iter = read_iter_zero, .aio_write = aio_write_zero, .mmap = mmap_zero, }; @@ -802,7 +771,8 @@ static struct backing_dev_info zero_bdi = { static const struct file_operations full_fops = { .llseek = full_lseek, - .read = read_full, + .read = new_sync_read, + .read_iter = read_iter_zero, .write = write_full, }; diff --git a/drivers/char/random.c b/drivers/char/random.c index 8c86a95203a0..04645c09fe5e 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -874,7 +874,7 @@ static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs) void add_interrupt_randomness(int irq, int irq_flags) { struct entropy_store *r; - struct fast_pool *fast_pool = &__get_cpu_var(irq_randomness); + struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); struct pt_regs *regs = get_irq_regs(); unsigned long now = jiffies; cycles_t cycles = random_get_entropy(); diff --git a/drivers/char/tile-srom.c b/drivers/char/tile-srom.c index bd377472dcfb..02e76ac6d282 100644 --- a/drivers/char/tile-srom.c +++ b/drivers/char/tile-srom.c @@ -76,6 +76,7 @@ MODULE_LICENSE("GPL"); static int srom_devs; /* Number of SROM partitions */ static struct cdev srom_cdev; +static struct platform_device *srom_parent; static struct class *srom_class; static struct srom_dev *srom_devices; @@ -350,7 +351,7 @@ static int srom_setup_minor(struct srom_dev *srom, int index) SROM_PAGE_SIZE_OFF, sizeof(srom->page_size)) < 0) return -EIO; - dev = device_create(srom_class, &platform_bus, + dev = device_create(srom_class, &srom_parent->dev, MKDEV(srom_major, index), srom, "%d", index); return PTR_ERR_OR_ZERO(dev); } @@ -415,6 +416,13 @@ static int srom_init(void) if (result < 0) goto fail_chrdev; + /* Create a parent device */ + srom_parent = platform_device_register_simple("srom", -1, NULL, 0); + if (IS_ERR(srom_parent)) { + result = PTR_ERR(srom_parent); + goto fail_pdev; + } + /* Create a sysfs class. */ srom_class = class_create(THIS_MODULE, "srom"); if (IS_ERR(srom_class)) { @@ -438,6 +446,8 @@ fail_class: device_destroy(srom_class, MKDEV(srom_major, i)); class_destroy(srom_class); fail_cdev: + platform_device_unregister(srom_parent); +fail_pdev: cdev_del(&srom_cdev); fail_chrdev: unregister_chrdev_region(dev, srom_devs); @@ -454,6 +464,7 @@ static void srom_cleanup(void) device_destroy(srom_class, MKDEV(srom_major, i)); class_destroy(srom_class); cdev_del(&srom_cdev); + platform_device_unregister(srom_parent); unregister_chrdev_region(MKDEV(srom_major, 0), srom_devs); kfree(srom_devices); } diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c index 2064b4527040..441b44e54226 100644 --- a/drivers/char/tpm/xen-tpmfront.c +++ b/drivers/char/tpm/xen-tpmfront.c @@ -367,12 +367,13 @@ static const struct xenbus_device_id tpmfront_ids[] = { }; MODULE_ALIAS("xen:vtpm"); -static DEFINE_XENBUS_DRIVER(tpmfront, , - .probe = tpmfront_probe, - .remove = tpmfront_remove, - .resume = tpmfront_resume, - .otherend_changed = backend_changed, - ); +static struct xenbus_driver tpmfront_driver = { + .ids = tpmfront_ids, + .probe = tpmfront_probe, + .remove = tpmfront_remove, + .resume = tpmfront_resume, + .otherend_changed = backend_changed, +}; static int __init xen_tpmfront_init(void) { diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index b585b4789822..bfa640023e64 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1449,6 +1449,8 @@ static int add_port(struct ports_device *portdev, u32 id) spin_lock_init(&port->outvq_lock); init_waitqueue_head(&port->waitqueue); + virtio_device_ready(portdev->vdev); + /* Fill the in_vq with buffers so the host can send us data. */ nr_added_bufs = fill_queue(port->in_vq, &port->inbuf_lock); if (!nr_added_bufs) { @@ -2182,6 +2184,8 @@ static int virtcons_restore(struct virtio_device *vdev) if (ret) return ret; + virtio_device_ready(portdev->vdev); + if (use_multiport(portdev)) fill_queue(portdev->c_ivq, &portdev->c_ivq_lock); diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 9b1a5ac4881d..c07dfe5c4da3 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -843,7 +843,6 @@ static struct platform_driver hwicap_platform_driver = { .probe = hwicap_drv_probe, .remove = hwicap_drv_remove, .driver = { - .owner = THIS_MODULE, .name = DRIVER_NAME, .of_match_table = hwicap_of_match, }, diff --git a/drivers/char/xillybus/Kconfig b/drivers/char/xillybus/Kconfig new file mode 100644 index 000000000000..b53bdf12da0d --- /dev/null +++ b/drivers/char/xillybus/Kconfig @@ -0,0 +1,33 @@ +# +# Xillybus devices +# + +config XILLYBUS + tristate "Xillybus generic FPGA interface" + depends on PCI || (OF_ADDRESS && OF_IRQ) + select CRC32 + help + Xillybus is a generic interface for peripherals designed on + programmable logic (FPGA). The driver probes the hardware for + its capabilities, and creates device files accordingly. + + If unsure, say N. + +if XILLYBUS + +config XILLYBUS_PCIE + tristate "Xillybus over PCIe" + depends on PCI_MSI + help + Set to M if you want Xillybus to use PCI Express for communicating + with the FPGA. + +config XILLYBUS_OF + tristate "Xillybus over Device Tree" + depends on OF_ADDRESS && OF_IRQ + help + Set to M if you want Xillybus to find its resources from the + Open Firmware Flattened Device Tree. If the target is an embedded + system, say M. + +endif # if XILLYBUS diff --git a/drivers/char/xillybus/Makefile b/drivers/char/xillybus/Makefile new file mode 100644 index 000000000000..b68b7ebfd381 --- /dev/null +++ b/drivers/char/xillybus/Makefile @@ -0,0 +1,7 @@ +# +# Makefile for Xillybus driver +# + +obj-$(CONFIG_XILLYBUS) += xillybus_core.o +obj-$(CONFIG_XILLYBUS_PCIE) += xillybus_pcie.o +obj-$(CONFIG_XILLYBUS_OF) += xillybus_of.o diff --git a/drivers/char/xillybus/xillybus.h b/drivers/char/xillybus/xillybus.h new file mode 100644 index 000000000000..b9a9eb6d4f72 --- /dev/null +++ b/drivers/char/xillybus/xillybus.h @@ -0,0 +1,160 @@ +/* + * linux/drivers/misc/xillybus.h + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Header file for the Xillybus FPGA/host framework. + * + * This program is free software; you can redistribute it and/or modify + * it under the smems of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#ifndef __XILLYBUS_H +#define __XILLYBUS_H + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/cdev.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> + +struct xilly_endpoint_hardware; + +struct xilly_buffer { + void *addr; + dma_addr_t dma_addr; + int end_offset; /* Counting elements, not bytes */ +}; + +struct xilly_idt_handle { + unsigned char *chandesc; + unsigned char *idt; + int entries; +}; + +/* + * Read-write confusion: wr_* and rd_* notation sticks to FPGA view, so + * wr_* buffers are those consumed by read(), since the FPGA writes to them + * and vice versa. + */ + +struct xilly_channel { + struct xilly_endpoint *endpoint; + int chan_num; + int log2_element_size; + int seekable; + + struct xilly_buffer **wr_buffers; /* FPGA writes, driver reads! */ + int num_wr_buffers; + unsigned int wr_buf_size; /* In bytes */ + int wr_fpga_buf_idx; + int wr_host_buf_idx; + int wr_host_buf_pos; + int wr_empty; + int wr_ready; /* Significant only when wr_empty == 1 */ + int wr_sleepy; + int wr_eof; + int wr_hangup; + spinlock_t wr_spinlock; + struct mutex wr_mutex; + wait_queue_head_t wr_wait; + wait_queue_head_t wr_ready_wait; + int wr_ref_count; + int wr_synchronous; + int wr_allow_partial; + int wr_exclusive_open; + int wr_supports_nonempty; + + struct xilly_buffer **rd_buffers; /* FPGA reads, driver writes! */ + int num_rd_buffers; + unsigned int rd_buf_size; /* In bytes */ + int rd_fpga_buf_idx; + int rd_host_buf_pos; + int rd_host_buf_idx; + int rd_full; + spinlock_t rd_spinlock; + struct mutex rd_mutex; + wait_queue_head_t rd_wait; + int rd_ref_count; + int rd_allow_partial; + int rd_synchronous; + int rd_exclusive_open; + struct delayed_work rd_workitem; + unsigned char rd_leftovers[4]; +}; + +struct xilly_endpoint { + /* + * One of pdev and dev is always NULL, and the other is a valid + * pointer, depending on the type of device + */ + struct pci_dev *pdev; + struct device *dev; + struct xilly_endpoint_hardware *ephw; + + struct list_head ep_list; + int dma_using_dac; /* =1 if 64-bit DMA is used, =0 otherwise. */ + __iomem void *registers; + int fatal_error; + + struct mutex register_mutex; + wait_queue_head_t ep_wait; + + /* Channels and message handling */ + struct cdev cdev; + + int major; + int lowest_minor; /* Highest minor = lowest_minor + num_channels - 1 */ + + int num_channels; /* EXCLUDING message buffer */ + struct xilly_channel **channels; + int msg_counter; + int failed_messages; + int idtlen; + + u32 *msgbuf_addr; + dma_addr_t msgbuf_dma_addr; + unsigned int msg_buf_size; +}; + +struct xilly_endpoint_hardware { + struct module *owner; + void (*hw_sync_sgl_for_cpu)(struct xilly_endpoint *, + dma_addr_t, + size_t, + int); + void (*hw_sync_sgl_for_device)(struct xilly_endpoint *, + dma_addr_t, + size_t, + int); + int (*map_single)(struct xilly_endpoint *, + void *, + size_t, + int, + dma_addr_t *); +}; + +struct xilly_mapping { + void *device; + dma_addr_t dma_addr; + size_t size; + int direction; +}; + +irqreturn_t xillybus_isr(int irq, void *data); + +struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev, + struct device *dev, + struct xilly_endpoint_hardware + *ephw); + +int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint); + +void xillybus_endpoint_remove(struct xilly_endpoint *endpoint); + +#endif /* __XILLYBUS_H */ diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c new file mode 100644 index 000000000000..b827fa095f1b --- /dev/null +++ b/drivers/char/xillybus/xillybus_core.c @@ -0,0 +1,2103 @@ +/* + * linux/drivers/misc/xillybus_core.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework. + * + * This driver interfaces with a special IP core in an FPGA, setting up + * a pipe between a hardware FIFO in the programmable logic and a device + * file in the host. The number of such pipes and their attributes are + * set up on the logic. This driver detects these automatically and + * creates the device files accordingly. + * + * This program is free software; you can redistribute it and/or modify + * it under the smems of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/io.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/fs.h> +#include <linux/cdev.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/crc32.h> +#include <linux/poll.h> +#include <linux/delay.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus core functions"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_VERSION("1.07"); +MODULE_ALIAS("xillybus_core"); +MODULE_LICENSE("GPL v2"); + +/* General timeout is 100 ms, rx timeout is 10 ms */ +#define XILLY_RX_TIMEOUT (10*HZ/1000) +#define XILLY_TIMEOUT (100*HZ/1000) + +#define fpga_msg_ctrl_reg 0x0008 +#define fpga_dma_control_reg 0x0020 +#define fpga_dma_bufno_reg 0x0024 +#define fpga_dma_bufaddr_lowaddr_reg 0x0028 +#define fpga_dma_bufaddr_highaddr_reg 0x002c +#define fpga_buf_ctrl_reg 0x0030 +#define fpga_buf_offset_reg 0x0034 +#define fpga_endian_reg 0x0040 + +#define XILLYMSG_OPCODE_RELEASEBUF 1 +#define XILLYMSG_OPCODE_QUIESCEACK 2 +#define XILLYMSG_OPCODE_FIFOEOF 3 +#define XILLYMSG_OPCODE_FATAL_ERROR 4 +#define XILLYMSG_OPCODE_NONEMPTY 5 + +static const char xillyname[] = "xillybus"; + +static struct class *xillybus_class; + +/* + * ep_list_lock is the last lock to be taken; No other lock requests are + * allowed while holding it. It merely protects list_of_endpoints, and not + * the endpoints listed in it. + */ + +static LIST_HEAD(list_of_endpoints); +static struct mutex ep_list_lock; +static struct workqueue_struct *xillybus_wq; + +/* + * Locking scheme: Mutexes protect invocations of character device methods. + * If both locks are taken, wr_mutex is taken first, rd_mutex second. + * + * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the + * buffers' end_offset fields against changes made by IRQ handler (and in + * theory, other file request handlers, but the mutex handles that). Nothing + * else. + * They are held for short direct memory manipulations. Needless to say, + * no mutex locking is allowed when a spinlock is held. + * + * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. + * + * register_mutex is endpoint-specific, and is held when non-atomic + * register operations are performed. wr_mutex and rd_mutex may be + * held when register_mutex is taken, but none of the spinlocks. Note that + * register_mutex doesn't protect against sporadic buf_ctrl_reg writes + * which are unrelated to buf_offset_reg, since they are harmless. + * + * Blocking on the wait queues is allowed with mutexes held, but not with + * spinlocks. + * + * Only interruptible blocking is allowed on mutexes and wait queues. + * + * All in all, the locking order goes (with skips allowed, of course): + * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock + */ + +static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) +{ + int opcode; + int msg_channel, msg_bufno, msg_data, msg_dir; + + opcode = (buf[0] >> 24) & 0xff; + msg_dir = buf[0] & 1; + msg_channel = (buf[0] >> 1) & 0x7ff; + msg_bufno = (buf[0] >> 12) & 0x3ff; + msg_data = buf[1] & 0xfffffff; + + dev_warn(endpoint->dev, + "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", + opcode, msg_channel, msg_dir, msg_bufno, msg_data); +} + +/* + * xillybus_isr assumes the interrupt is allocated exclusively to it, + * which is the natural case MSI and several other hardware-oriented + * interrupts. Sharing is not allowed. + */ + +irqreturn_t xillybus_isr(int irq, void *data) +{ + struct xilly_endpoint *ep = data; + u32 *buf; + unsigned int buf_size; + int i; + int opcode; + unsigned int msg_channel, msg_bufno, msg_data, msg_dir; + struct xilly_channel *channel; + + buf = ep->msgbuf_addr; + buf_size = ep->msg_buf_size/sizeof(u32); + + ep->ephw->hw_sync_sgl_for_cpu(ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + for (i = 0; i < buf_size; i += 2) { + if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { + malformed_message(ep, &buf[i]); + dev_warn(ep->dev, + "Sending a NACK on counter %x (instead of %x) on entry %d\n", + ((buf[i+1] >> 28) & 0xf), + ep->msg_counter, + i/2); + + if (++ep->failed_messages > 10) { + dev_err(ep->dev, + "Lost sync with interrupt messages. Stopping.\n"); + } else { + ep->ephw->hw_sync_sgl_for_device( + ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + iowrite32(0x01, /* Message NACK */ + ep->registers + fpga_msg_ctrl_reg); + } + return IRQ_HANDLED; + } else if (buf[i] & (1 << 22)) /* Last message */ + break; + } + + if (i >= buf_size) { + dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); + return IRQ_HANDLED; + } + + buf_size = i + 2; + + for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ + opcode = (buf[i] >> 24) & 0xff; + + msg_dir = buf[i] & 1; + msg_channel = (buf[i] >> 1) & 0x7ff; + msg_bufno = (buf[i] >> 12) & 0x3ff; + msg_data = buf[i+1] & 0xfffffff; + + switch (opcode) { + case XILLYMSG_OPCODE_RELEASEBUF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0)) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_dir) { /* Write channel */ + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + channel->wr_buffers[msg_bufno]->end_offset = + msg_data; + channel->wr_fpga_buf_idx = msg_bufno; + channel->wr_empty = 0; + channel->wr_sleepy = 0; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + } else { + /* Read channel */ + + if (msg_bufno >= channel->num_rd_buffers) { + malformed_message(ep, &buf[i]); + break; + } + + spin_lock(&channel->rd_spinlock); + channel->rd_fpga_buf_idx = msg_bufno; + channel->rd_full = 0; + spin_unlock(&channel->rd_spinlock); + + wake_up_interruptible(&channel->rd_wait); + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + } + + break; + case XILLYMSG_OPCODE_NONEMPTY: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->wr_supports_nonempty) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + if (msg_bufno == channel->wr_host_buf_idx) + channel->wr_ready = 1; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_ready_wait); + + break; + case XILLYMSG_OPCODE_QUIESCEACK: + ep->idtlen = msg_data; + wake_up_interruptible(&ep->ep_wait); + + break; + case XILLYMSG_OPCODE_FIFOEOF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + channel = ep->channels[msg_channel]; + spin_lock(&channel->wr_spinlock); + channel->wr_eof = msg_bufno; + channel->wr_sleepy = 0; + + channel->wr_hangup = channel->wr_empty && + (channel->wr_host_buf_idx == msg_bufno); + + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + break; + case XILLYMSG_OPCODE_FATAL_ERROR: + ep->fatal_error = 1; + wake_up_interruptible(&ep->ep_wait); /* For select() */ + dev_err(ep->dev, + "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); + break; + default: + malformed_message(ep, &buf[i]); + break; + } + } + + ep->ephw->hw_sync_sgl_for_device(ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + ep->msg_counter = (ep->msg_counter + 1) & 0xf; + ep->failed_messages = 0; + iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ + + return IRQ_HANDLED; +} +EXPORT_SYMBOL(xillybus_isr); + +/* + * A few trivial memory management functions. + * NOTE: These functions are used only on probe and remove, and therefore + * no locks are applied! + */ + +static void xillybus_autoflush(struct work_struct *work); + +struct xilly_alloc_state { + void *salami; + int left_of_salami; + int nbuffer; + enum dma_data_direction direction; + u32 regdirection; +}; + +static int xilly_get_dma_buffers(struct xilly_endpoint *ep, + struct xilly_alloc_state *s, + struct xilly_buffer **buffers, + int bufnum, int bytebufsize) +{ + int i, rc; + dma_addr_t dma_addr; + struct device *dev = ep->dev; + struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ + + if (buffers) { /* Not the message buffer */ + this_buffer = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer), + GFP_KERNEL); + if (!this_buffer) + return -ENOMEM; + } + + for (i = 0; i < bufnum; i++) { + /* + * Buffers are expected in descending size order, so there + * is either enough space for this buffer or none at all. + */ + + if ((s->left_of_salami < bytebufsize) && + (s->left_of_salami > 0)) { + dev_err(ep->dev, + "Corrupt buffer allocation in IDT. Aborting.\n"); + return -ENODEV; + } + + if (s->left_of_salami == 0) { + int allocorder, allocsize; + + allocsize = PAGE_SIZE; + allocorder = 0; + while (bytebufsize > allocsize) { + allocsize *= 2; + allocorder++; + } + + s->salami = (void *) devm_get_free_pages( + dev, + GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, + allocorder); + if (!s->salami) + return -ENOMEM; + + s->left_of_salami = allocsize; + } + + rc = ep->ephw->map_single(ep, s->salami, + bytebufsize, s->direction, + &dma_addr); + if (rc) + return rc; + + iowrite32((u32) (dma_addr & 0xffffffff), + ep->registers + fpga_dma_bufaddr_lowaddr_reg); + iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), + ep->registers + fpga_dma_bufaddr_highaddr_reg); + + if (buffers) { /* Not the message buffer */ + this_buffer->addr = s->salami; + this_buffer->dma_addr = dma_addr; + buffers[i] = this_buffer++; + + iowrite32(s->regdirection | s->nbuffer++, + ep->registers + fpga_dma_bufno_reg); + } else { + ep->msgbuf_addr = s->salami; + ep->msgbuf_dma_addr = dma_addr; + ep->msg_buf_size = bytebufsize; + + iowrite32(s->regdirection, + ep->registers + fpga_dma_bufno_reg); + } + + s->left_of_salami -= bytebufsize; + s->salami += bytebufsize; + } + return 0; +} + +static int xilly_setupchannels(struct xilly_endpoint *ep, + unsigned char *chandesc, + int entries) +{ + struct device *dev = ep->dev; + int i, entry, rc; + struct xilly_channel *channel; + int channelnum, bufnum, bufsize, format, is_writebuf; + int bytebufsize; + int synchronous, allowpartial, exclusive_open, seekable; + int supports_nonempty; + int msg_buf_done = 0; + + struct xilly_alloc_state rd_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_TO_DEVICE, + .regdirection = 0, + }; + + struct xilly_alloc_state wr_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_FROM_DEVICE, + .regdirection = 0x80000000, + }; + + channel = devm_kcalloc(dev, ep->num_channels, + sizeof(struct xilly_channel), GFP_KERNEL); + if (!channel) + return -ENOMEM; + + ep->channels = devm_kcalloc(dev, ep->num_channels + 1, + sizeof(struct xilly_channel *), + GFP_KERNEL); + if (!ep->channels) + return -ENOMEM; + + ep->channels[0] = NULL; /* Channel 0 is message buf. */ + + /* Initialize all channels with defaults */ + + for (i = 1; i <= ep->num_channels; i++) { + channel->wr_buffers = NULL; + channel->rd_buffers = NULL; + channel->num_wr_buffers = 0; + channel->num_rd_buffers = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->rd_fpga_buf_idx = 0; + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_full = 0; + channel->wr_ref_count = 0; + channel->rd_ref_count = 0; + + spin_lock_init(&channel->wr_spinlock); + spin_lock_init(&channel->rd_spinlock); + mutex_init(&channel->wr_mutex); + mutex_init(&channel->rd_mutex); + init_waitqueue_head(&channel->rd_wait); + init_waitqueue_head(&channel->wr_wait); + init_waitqueue_head(&channel->wr_ready_wait); + + INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); + + channel->endpoint = ep; + channel->chan_num = i; + + channel->log2_element_size = 0; + + ep->channels[i] = channel++; + } + + for (entry = 0; entry < entries; entry++, chandesc += 4) { + struct xilly_buffer **buffers = NULL; + + is_writebuf = chandesc[0] & 0x01; + channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); + format = (chandesc[1] >> 4) & 0x03; + allowpartial = (chandesc[1] >> 6) & 0x01; + synchronous = (chandesc[1] >> 7) & 0x01; + bufsize = 1 << (chandesc[2] & 0x1f); + bufnum = 1 << (chandesc[3] & 0x0f); + exclusive_open = (chandesc[2] >> 7) & 0x01; + seekable = (chandesc[2] >> 6) & 0x01; + supports_nonempty = (chandesc[2] >> 5) & 0x01; + + if ((channelnum > ep->num_channels) || + ((channelnum == 0) && !is_writebuf)) { + dev_err(ep->dev, + "IDT requests channel out of range. Aborting.\n"); + return -ENODEV; + } + + channel = ep->channels[channelnum]; /* NULL for msg channel */ + + if (!is_writebuf || channelnum > 0) { + channel->log2_element_size = ((format > 2) ? + 2 : format); + + bytebufsize = channel->rd_buf_size = bufsize * + (1 << channel->log2_element_size); + + buffers = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer *), + GFP_KERNEL); + if (!buffers) + return -ENOMEM; + } else { + bytebufsize = bufsize << 2; + } + + if (!is_writebuf) { + channel->num_rd_buffers = bufnum; + channel->rd_allow_partial = allowpartial; + channel->rd_synchronous = synchronous; + channel->rd_exclusive_open = exclusive_open; + channel->seekable = seekable; + + channel->rd_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, + bufnum, bytebufsize); + } else if (channelnum > 0) { + channel->num_wr_buffers = bufnum; + + channel->seekable = seekable; + channel->wr_supports_nonempty = supports_nonempty; + + channel->wr_allow_partial = allowpartial; + channel->wr_synchronous = synchronous; + channel->wr_exclusive_open = exclusive_open; + + channel->wr_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, + bufnum, bytebufsize); + } else { + rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, + bufnum, bytebufsize); + msg_buf_done++; + } + + if (rc) + return -ENOMEM; + } + + if (!msg_buf_done) { + dev_err(ep->dev, + "Corrupt IDT: No message buffer. Aborting.\n"); + return -ENODEV; + } + return 0; +} + +static int xilly_scan_idt(struct xilly_endpoint *endpoint, + struct xilly_idt_handle *idt_handle) +{ + int count = 0; + unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; + unsigned char *end_of_idt = idt + endpoint->idtlen - 4; + unsigned char *scan; + int len; + + scan = idt; + idt_handle->idt = idt; + + scan++; /* Skip version number */ + + while ((scan <= end_of_idt) && *scan) { + while ((scan <= end_of_idt) && *scan++) + /* Do nothing, just scan thru string */; + count++; + } + + scan++; + + if (scan > end_of_idt) { + dev_err(endpoint->dev, + "IDT device name list overflow. Aborting.\n"); + return -ENODEV; + } + idt_handle->chandesc = scan; + + len = endpoint->idtlen - (3 + ((int) (scan - idt))); + + if (len & 0x03) { + dev_err(endpoint->dev, + "Corrupt IDT device name list. Aborting.\n"); + return -ENODEV; + } + + idt_handle->entries = len >> 2; + endpoint->num_channels = count; + + return 0; +} + +static int xilly_obtain_idt(struct xilly_endpoint *endpoint) +{ + struct xilly_channel *channel; + unsigned char *version; + long t; + + channel = endpoint->channels[1]; /* This should be generated ad-hoc */ + + channel->wr_sleepy = 1; + + iowrite32(1 | + (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ + endpoint->registers + fpga_buf_ctrl_reg); + + t = wait_event_interruptible_timeout(channel->wr_wait, + (!channel->wr_sleepy), + XILLY_TIMEOUT); + + if (t <= 0) { + dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); + + if (endpoint->fatal_error) + return -EIO; + + return -ENODEV; + } + + endpoint->ephw->hw_sync_sgl_for_cpu( + channel->endpoint, + channel->wr_buffers[0]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { + dev_err(endpoint->dev, + "IDT length mismatch (%d != %d). Aborting.\n", + channel->wr_buffers[0]->end_offset, endpoint->idtlen); + return -ENODEV; + } + + if (crc32_le(~0, channel->wr_buffers[0]->addr, + endpoint->idtlen+1) != 0) { + dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); + return -ENODEV; + } + + version = channel->wr_buffers[0]->addr; + + /* Check version number. Accept anything below 0x82 for now. */ + if (*version > 0x82) { + dev_err(endpoint->dev, + "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n", + *version); + return -ENODEV; + } + + return 0; +} + +static ssize_t xillybus_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + int no_time_left = 0; + long deadline, left_to_sleep; + struct xilly_channel *channel = filp->private_data; + + int empty, reached_eof, exhausted, ready; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int waiting_bufidx; + + if (channel->endpoint->fatal_error) + return -EIO; + + deadline = jiffies + 1 + XILLY_RX_TIMEOUT; + + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + + while (1) { /* Note that we may drop mutex within this loop */ + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->wr_spinlock, flags); + + empty = channel->wr_empty; + ready = !empty || channel->wr_ready; + + if (!empty) { + bufidx = channel->wr_host_buf_idx; + bufpos = channel->wr_host_buf_pos; + howmany = ((channel->wr_buffers[bufidx]->end_offset + + 1) << channel->log2_element_size) + - bufpos; + + /* Update wr_host_* to its post-operation state */ + if (howmany > bytes_to_do) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->wr_host_buf_pos += howmany; + } else { + bufferdone = 1; + + channel->wr_host_buf_pos = 0; + + if (bufidx == channel->wr_fpga_buf_idx) { + channel->wr_empty = 1; + channel->wr_sleepy = 1; + channel->wr_ready = 0; + } + + if (bufidx >= (channel->num_wr_buffers - 1)) + channel->wr_host_buf_idx = 0; + else + channel->wr_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * empty = empty before change + * exhasted = empty after possible change + */ + + reached_eof = channel->wr_empty && + (channel->wr_host_buf_idx == channel->wr_eof); + channel->wr_hangup = reached_eof; + exhausted = channel->wr_empty; + waiting_bufidx = channel->wr_host_buf_idx; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + if (!empty) { /* Go on, now without the spinlock */ + + if (bufpos == 0) /* Position zero means it's virgin */ + channel->endpoint->ephw->hw_sync_sgl_for_cpu( + channel->endpoint, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (copy_to_user( + userbuf, + channel->wr_buffers[bufidx]->addr + + bufpos, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + channel->endpoint->ephw->hw_sync_sgl_for_device( + channel->endpoint, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + /* + * Tell FPGA the buffer is done with. It's an + * atomic operation to the FPGA, so what + * happens with other channels doesn't matter, + * and the certain channel is protected with + * the channel-specific mutex. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + if (rc) { + mutex_unlock(&channel->wr_mutex); + return rc; + } + } + + /* This includes a zero-count return = EOF */ + if ((bytes_done >= count) || reached_eof) + break; + + if (!exhausted) + continue; /* More in RAM buffer(s)? Just go on. */ + + if ((bytes_done > 0) && + (no_time_left || + (channel->wr_synchronous && channel->wr_allow_partial))) + break; + + /* + * Nonblocking read: The "ready" flag tells us that the FPGA + * has data to send. In non-blocking mode, if it isn't on, + * just return. But if there is, we jump directly to the point + * where we ask for the FPGA to send all it has, and wait + * until that data arrives. So in a sense, we *do* block in + * nonblocking mode, but only for a very short time. + */ + + if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { + if (bytes_done > 0) + break; + + if (ready) + goto desperate; + + rc = -EAGAIN; + break; + } + + if (!no_time_left || (bytes_done > 0)) { + /* + * Note that in case of an element-misaligned read + * request, offsetlimit will include the last element, + * which will be partially read from. + */ + int offsetlimit = ((count - bytes_done) - 1) >> + channel->log2_element_size; + int buf_elements = channel->wr_buf_size >> + channel->log2_element_size; + + /* + * In synchronous mode, always send an offset limit. + * Just don't send a value too big. + */ + + if (channel->wr_synchronous) { + /* Don't request more than one buffer */ + if (channel->wr_allow_partial && + (offsetlimit >= buf_elements)) + offsetlimit = buf_elements - 1; + + /* Don't request more than all buffers */ + if (!channel->wr_allow_partial && + (offsetlimit >= + (buf_elements * channel->num_wr_buffers))) + offsetlimit = buf_elements * + channel->num_wr_buffers - 1; + } + + /* + * In asynchronous mode, force early flush of a buffer + * only if that will allow returning a full count. The + * "offsetlimit < ( ... )" rather than "<=" excludes + * requesting a full buffer, which would obviously + * cause a buffer transmission anyhow + */ + + if (channel->wr_synchronous || + (offsetlimit < (buf_elements - 1))) { + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(offsetlimit, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32(1 | (channel->chan_num << 1) | + (2 << 24) | /* 2 = offset limit */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + } + } + + /* + * If partial completion is disallowed, there is no point in + * timeout sleeping. Neither if no_time_left is set and + * there's no data. + */ + + if (!channel->wr_allow_partial || + (no_time_left && (bytes_done == 0))) { + /* + * This do-loop will run more than once if another + * thread reasserted wr_sleepy before we got the mutex + * back, so we try again. + */ + + do { + mutex_unlock(&channel->wr_mutex); + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + goto interrupted; + + if (mutex_lock_interruptible( + &channel->wr_mutex)) + goto interrupted; + } while (channel->wr_sleepy); + + continue; + +interrupted: /* Mutex is not held if got here */ + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; /* Don't admit snoozing */ + return -EINTR; + } + + left_to_sleep = deadline - ((long) jiffies); + + /* + * If our time is out, skip the waiting. We may miss wr_sleepy + * being deasserted but hey, almost missing the train is like + * missing it. + */ + + if (left_to_sleep > 0) { + left_to_sleep = + wait_event_interruptible_timeout( + channel->wr_wait, + (!channel->wr_sleepy), + left_to_sleep); + + if (left_to_sleep > 0) /* wr_sleepy deasserted */ + continue; + + if (left_to_sleep < 0) { /* Interrupt */ + mutex_unlock(&channel->wr_mutex); + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + +desperate: + no_time_left = 1; /* We're out of sleeping time. Desperate! */ + + if (bytes_done == 0) { + /* + * Reaching here means that we allow partial return, + * that we've run out of time, and that we have + * nothing to return. + * So tell the FPGA to send anything it has or gets. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (3 << 24) | /* Opcode 3, flush it all! */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + /* + * Reaching here means that we *do* have data in the buffer, + * but the "partial" flag disallows returning less than + * required. And we don't have as much. So loop again, + * which is likely to end up blocking indefinitely until + * enough data has arrived. + */ + } + + mutex_unlock(&channel->wr_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + return bytes_done; +} + +/* + * The timeout argument takes values as follows: + * >0 : Flush with timeout + * ==0 : Flush, and wait idefinitely for the flush to complete + * <0 : Autoflush: Flush only if there's a single buffer occupied + */ + +static int xillybus_myflush(struct xilly_channel *channel, long timeout) +{ + int rc; + unsigned long flags; + + int end_offset_plus1; + int bufidx, bufidx_minus1; + int i; + int empty; + int new_rd_host_buf_pos; + + if (channel->endpoint->fatal_error) + return -EIO; + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + /* + * Don't flush a closed channel. This can happen when the work queued + * autoflush thread fires off after the file has closed. This is not + * an error, just something to dismiss. + */ + + if (!channel->rd_ref_count) + goto done; + + bufidx = channel->rd_host_buf_idx; + + bufidx_minus1 = (bufidx == 0) ? + channel->num_rd_buffers - 1 : + bufidx - 1; + + end_offset_plus1 = channel->rd_host_buf_pos >> + channel->log2_element_size; + + new_rd_host_buf_pos = channel->rd_host_buf_pos - + (end_offset_plus1 << channel->log2_element_size); + + /* Submit the current buffer if it's nonempty */ + if (end_offset_plus1) { + unsigned char *tail = channel->rd_buffers[bufidx]->addr + + (end_offset_plus1 << channel->log2_element_size); + + /* Copy unflushed data, so we can put it in next buffer */ + for (i = 0; i < new_rd_host_buf_pos; i++) + channel->rd_leftovers[i] = *tail++; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + /* Autoflush only if a single buffer is occupied */ + + if ((timeout < 0) && + (channel->rd_full || + (bufidx_minus1 != channel->rd_fpga_buf_idx))) { + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + /* + * A new work item may be queued by the ISR exactly + * now, since the execution of a work item allows the + * queuing of a new one while it's running. + */ + goto done; + } + + /* The 4th element is never needed for data, so it's a flag */ + channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); + + /* Set up rd_full to reflect a certain moment's state */ + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + + channel->endpoint->ephw->hw_sync_sgl_for_device( + channel->endpoint, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (2 << 24) | /* Opcode 2, submit buffer */ + (bufidx << 12), + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + } else if (bufidx == 0) { + bufidx = channel->num_rd_buffers - 1; + } else { + bufidx--; + } + + channel->rd_host_buf_pos = new_rd_host_buf_pos; + + if (timeout < 0) + goto done; /* Autoflush */ + + /* + * bufidx is now the last buffer written to (or equal to + * rd_fpga_buf_idx if buffer was never written to), and + * channel->rd_host_buf_idx the one after it. + * + * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. + */ + + while (1) { /* Loop waiting for draining of buffers */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + + if (bufidx != channel->rd_fpga_buf_idx) + channel->rd_full = 1; /* + * Not really full, + * but needs waiting. + */ + + empty = !channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (empty) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing user should not be surprised if open() for write + * sleeps. + */ + if (timeout == 0) + wait_event_interruptible(channel->rd_wait, + (!channel->rd_full)); + + else if (wait_event_interruptible_timeout( + channel->rd_wait, + (!channel->rd_full), + timeout) == 0) { + dev_warn(channel->endpoint->dev, + "Timed out while flushing. Output data may be lost.\n"); + + rc = -ETIMEDOUT; + break; + } + + if (channel->rd_full) { + rc = -EINTR; + break; + } + } + +done: + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + return rc; +} + +static int xillybus_flush(struct file *filp, fl_owner_t id) +{ + if (!(filp->f_mode & FMODE_WRITE)) + return 0; + + return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ +} + +static void xillybus_autoflush(struct work_struct *work) +{ + struct delayed_work *workitem = container_of( + work, struct delayed_work, work); + struct xilly_channel *channel = container_of( + workitem, struct xilly_channel, rd_workitem); + int rc; + + rc = xillybus_myflush(channel, -1); + if (rc == -EINTR) + dev_warn(channel->endpoint->dev, + "Autoflush failed because work queue thread got a signal.\n"); + else if (rc) + dev_err(channel->endpoint->dev, + "Autoflush failed under weird circumstances.\n"); +} + +static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + struct xilly_channel *channel = filp->private_data; + + int full, exhausted; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int end_offset_plus1 = 0; + + if (channel->endpoint->fatal_error) + return -EIO; + + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + while (1) { + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + full = channel->rd_full; + + if (!full) { + bufidx = channel->rd_host_buf_idx; + bufpos = channel->rd_host_buf_pos; + howmany = channel->rd_buf_size - bufpos; + + /* + * Update rd_host_* to its state after this operation. + * count=0 means committing the buffer immediately, + * which is like flushing, but not necessarily block. + */ + + if ((howmany > bytes_to_do) && + (count || + ((bufpos >> channel->log2_element_size) == 0))) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->rd_host_buf_pos += howmany; + } else { + bufferdone = 1; + + if (count) { + end_offset_plus1 = + channel->rd_buf_size >> + channel->log2_element_size; + channel->rd_host_buf_pos = 0; + } else { + unsigned char *tail; + int i; + + end_offset_plus1 = bufpos >> + channel->log2_element_size; + + channel->rd_host_buf_pos -= + end_offset_plus1 << + channel->log2_element_size; + + tail = channel-> + rd_buffers[bufidx]->addr + + (end_offset_plus1 << + channel->log2_element_size); + + for (i = 0; + i < channel->rd_host_buf_pos; + i++) + channel->rd_leftovers[i] = + *tail++; + } + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * full = full before change + * exhasted = full after possible change + */ + + exhausted = channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (!full) { /* Go on, now without the spinlock */ + unsigned char *head = + channel->rd_buffers[bufidx]->addr; + int i; + + if ((bufpos == 0) || /* Zero means it's virgin */ + (channel->rd_leftovers[3] != 0)) { + channel->endpoint->ephw->hw_sync_sgl_for_cpu( + channel->endpoint, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + /* Virgin, but leftovers are due */ + for (i = 0; i < bufpos; i++) + *head++ = channel->rd_leftovers[i]; + + channel->rd_leftovers[3] = 0; /* Clear flag */ + } + + if (copy_from_user( + channel->rd_buffers[bufidx]->addr + bufpos, + userbuf, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + channel->endpoint->ephw->hw_sync_sgl_for_device( + channel->endpoint, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (2 << 24) | /* 2 = submit buffer */ + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + + channel->rd_leftovers[3] = + (channel->rd_host_buf_pos != 0); + } + + if (rc) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + return rc; + } + } + + if (bytes_done >= count) + break; + + if (!exhausted) + continue; /* If there's more space, just go on */ + + if ((bytes_done > 0) && channel->rd_allow_partial) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing, user should not be surprised if open() for write + * sleeps. + */ + + if (filp->f_flags & O_NONBLOCK) { + rc = -EAGAIN; + break; + } + + if (wait_event_interruptible(channel->rd_wait, + (!channel->rd_full))) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + + mutex_unlock(&channel->rd_mutex); + + if (!channel->rd_synchronous) + queue_delayed_work(xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + if ((channel->rd_synchronous) && (bytes_done > 0)) { + rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ + + if (rc && (rc != -EINTR)) + return rc; + } + + return bytes_done; +} + +static int xillybus_open(struct inode *inode, struct file *filp) +{ + int rc = 0; + unsigned long flags; + int minor = iminor(inode); + int major = imajor(inode); + struct xilly_endpoint *ep_iter, *endpoint = NULL; + struct xilly_channel *channel; + + mutex_lock(&ep_list_lock); + + list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) { + if ((ep_iter->major == major) && + (minor >= ep_iter->lowest_minor) && + (minor < (ep_iter->lowest_minor + + ep_iter->num_channels))) { + endpoint = ep_iter; + break; + } + } + mutex_unlock(&ep_list_lock); + + if (!endpoint) { + pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n", + major, minor); + return -ENODEV; + } + + if (endpoint->fatal_error) + return -EIO; + + channel = endpoint->channels[1 + minor - endpoint->lowest_minor]; + filp->private_data = channel; + + /* + * It gets complicated because: + * 1. We don't want to take a mutex we don't have to + * 2. We don't want to open one direction if the other will fail. + */ + + if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && + (channel->wr_synchronous || !channel->wr_allow_partial || + !channel->wr_supports_nonempty)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for read on this device\n"); + return -ENODEV; + } + + if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && + (channel->rd_synchronous || !channel->rd_allow_partial)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for write on this device\n"); + return -ENODEV; + } + + /* + * Note: open() may block on getting mutexes despite O_NONBLOCK. + * This shouldn't occur normally, since multiple open of the same + * file descriptor is almost always prohibited anyhow + * (*_exclusive_open is normally set in real-life systems). + */ + + if (filp->f_mode & FMODE_READ) { + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + } + + if (filp->f_mode & FMODE_WRITE) { + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + goto unlock_wr; + } + + if ((filp->f_mode & FMODE_READ) && + (channel->wr_ref_count != 0) && + (channel->wr_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if ((filp->f_mode & FMODE_WRITE) && + (channel->rd_ref_count != 0) && + (channel->rd_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if (filp->f_mode & FMODE_READ) { + if (channel->wr_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->wr_spinlock, flags); + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->wr_eof = -1; + channel->wr_hangup = 0; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + iowrite32(1 | (channel->chan_num << 1) | + (4 << 24) | /* Opcode 4, open channel */ + ((channel->wr_synchronous & 1) << 23), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->wr_ref_count++; + } + + if (filp->f_mode & FMODE_WRITE) { + if (channel->rd_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_leftovers[3] = 0; /* No leftovers. */ + channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; + channel->rd_full = 0; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + iowrite32((channel->chan_num << 1) | + (4 << 24), /* Opcode 4, open channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->rd_ref_count++; + } + +unlock: + if (filp->f_mode & FMODE_WRITE) + mutex_unlock(&channel->rd_mutex); +unlock_wr: + if (filp->f_mode & FMODE_READ) + mutex_unlock(&channel->wr_mutex); + + if (!rc && (!channel->seekable)) + return nonseekable_open(inode, filp); + + return rc; +} + +static int xillybus_release(struct inode *inode, struct file *filp) +{ + unsigned long flags; + struct xilly_channel *channel = filp->private_data; + + int buf_idx; + int eof; + + if (channel->endpoint->fatal_error) + return -EIO; + + if (filp->f_mode & FMODE_WRITE) { + mutex_lock(&channel->rd_mutex); + + channel->rd_ref_count--; + + if (channel->rd_ref_count == 0) { + /* + * We rely on the kernel calling flush() + * before we get here. + */ + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + mutex_unlock(&channel->rd_mutex); + } + + if (filp->f_mode & FMODE_READ) { + mutex_lock(&channel->wr_mutex); + + channel->wr_ref_count--; + + if (channel->wr_ref_count == 0) { + iowrite32(1 | (channel->chan_num << 1) | + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + /* + * This is crazily cautious: We make sure that not + * only that we got an EOF (be it because we closed + * the channel or because of a user's EOF), but verify + * that it's one beyond the last buffer arrived, so + * we have no leftover buffers pending before wrapping + * up (which can only happen in asynchronous channels, + * BTW) + */ + + while (1) { + spin_lock_irqsave(&channel->wr_spinlock, + flags); + buf_idx = channel->wr_fpga_buf_idx; + eof = channel->wr_eof; + channel->wr_sleepy = 1; + spin_unlock_irqrestore(&channel->wr_spinlock, + flags); + + /* + * Check if eof points at the buffer after + * the last one the FPGA submitted. Note that + * no EOF is marked by negative eof. + */ + + buf_idx++; + if (buf_idx == channel->num_wr_buffers) + buf_idx = 0; + + if (buf_idx == eof) + break; + + /* + * Steal extra 100 ms if awaken by interrupt. + * This is a simple workaround for an + * interrupt pending when entering, which would + * otherwise result in declaring the hardware + * non-responsive. + */ + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + msleep(100); + + if (channel->wr_sleepy) { + mutex_unlock(&channel->wr_mutex); + dev_warn(channel->endpoint->dev, + "Hardware failed to respond to close command, therefore left in messy state.\n"); + return -EINTR; + } + } + } + + mutex_unlock(&channel->wr_mutex); + } + + return 0; +} + +static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) +{ + struct xilly_channel *channel = filp->private_data; + loff_t pos = filp->f_pos; + int rc = 0; + + /* + * Take both mutexes not allowing interrupts, since it seems like + * common applications don't expect an -EINTR here. Besides, multiple + * access to a single file descriptor on seekable devices is a mess + * anyhow. + */ + + if (channel->endpoint->fatal_error) + return -EIO; + + mutex_lock(&channel->wr_mutex); + mutex_lock(&channel->rd_mutex); + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos += offset; + break; + case SEEK_END: + pos = offset; /* Going to the end => to the beginning */ + break; + default: + rc = -EINVAL; + goto end; + } + + /* In any case, we must finish on an element boundary */ + if (pos & ((1 << channel->log2_element_size) - 1)) { + rc = -EINVAL; + goto end; + } + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(pos >> channel->log2_element_size, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (6 << 24), /* Opcode 6, set address */ + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + +end: + mutex_unlock(&channel->rd_mutex); + mutex_unlock(&channel->wr_mutex); + + if (rc) /* Return error after releasing mutexes */ + return rc; + + filp->f_pos = pos; + + /* + * Since seekable devices are allowed only when the channel is + * synchronous, we assume that there is no data pending in either + * direction (which holds true as long as no concurrent access on the + * file descriptor takes place). + * The only thing we may need to throw away is leftovers from partial + * write() flush. + */ + + channel->rd_leftovers[3] = 0; + + return pos; +} + +static unsigned int xillybus_poll(struct file *filp, poll_table *wait) +{ + struct xilly_channel *channel = filp->private_data; + unsigned int mask = 0; + unsigned long flags; + + poll_wait(filp, &channel->endpoint->ep_wait, wait); + + /* + * poll() won't play ball regarding read() channels which + * aren't asynchronous and support the nonempty message. Allowing + * that will create situations where data has been delivered at + * the FPGA, and users expecting select() to wake up, which it may + * not. + */ + + if (!channel->wr_synchronous && channel->wr_supports_nonempty) { + poll_wait(filp, &channel->wr_wait, wait); + poll_wait(filp, &channel->wr_ready_wait, wait); + + spin_lock_irqsave(&channel->wr_spinlock, flags); + if (!channel->wr_empty || channel->wr_ready) + mask |= POLLIN | POLLRDNORM; + + if (channel->wr_hangup) + /* + * Not POLLHUP, because its behavior is in the + * mist, and POLLIN does what we want: Wake up + * the read file descriptor so it sees EOF. + */ + mask |= POLLIN | POLLRDNORM; + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + } + + /* + * If partial data write is disallowed on a write() channel, + * it's pointless to ever signal OK to write, because is could + * block despite some space being available. + */ + + if (channel->rd_allow_partial) { + poll_wait(filp, &channel->rd_wait, wait); + + spin_lock_irqsave(&channel->rd_spinlock, flags); + if (!channel->rd_full) + mask |= POLLOUT | POLLWRNORM; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + } + + if (channel->endpoint->fatal_error) + mask |= POLLERR; + + return mask; +} + +static const struct file_operations xillybus_fops = { + .owner = THIS_MODULE, + .read = xillybus_read, + .write = xillybus_write, + .open = xillybus_open, + .flush = xillybus_flush, + .release = xillybus_release, + .llseek = xillybus_llseek, + .poll = xillybus_poll, +}; + +static int xillybus_init_chrdev(struct xilly_endpoint *endpoint, + const unsigned char *idt) +{ + int rc; + dev_t dev; + int devnum, i, minor, major; + char devname[48]; + struct device *device; + + rc = alloc_chrdev_region(&dev, 0, /* minor start */ + endpoint->num_channels, + xillyname); + if (rc) { + dev_warn(endpoint->dev, "Failed to obtain major/minors"); + return rc; + } + + endpoint->major = major = MAJOR(dev); + endpoint->lowest_minor = minor = MINOR(dev); + + cdev_init(&endpoint->cdev, &xillybus_fops); + endpoint->cdev.owner = endpoint->ephw->owner; + rc = cdev_add(&endpoint->cdev, MKDEV(major, minor), + endpoint->num_channels); + if (rc) { + dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n"); + goto unregister_chrdev; + } + + idt++; + + for (i = minor, devnum = 0; + devnum < endpoint->num_channels; + devnum++, i++) { + snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt); + + devname[sizeof(devname)-1] = 0; /* Should never matter */ + + while (*idt++) + /* Skip to next */; + + device = device_create(xillybus_class, + NULL, + MKDEV(major, i), + NULL, + "%s", devname); + + if (IS_ERR(device)) { + dev_warn(endpoint->dev, + "Failed to create %s device. Aborting.\n", + devname); + rc = -ENODEV; + goto unroll_device_create; + } + } + + dev_info(endpoint->dev, "Created %d device files.\n", + endpoint->num_channels); + return 0; /* succeed */ + +unroll_device_create: + devnum--; i--; + for (; devnum >= 0; devnum--, i--) + device_destroy(xillybus_class, MKDEV(major, i)); + + cdev_del(&endpoint->cdev); +unregister_chrdev: + unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels); + + return rc; +} + +static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint) +{ + int minor; + + for (minor = endpoint->lowest_minor; + minor < (endpoint->lowest_minor + endpoint->num_channels); + minor++) + device_destroy(xillybus_class, MKDEV(endpoint->major, minor)); + cdev_del(&endpoint->cdev); + unregister_chrdev_region(MKDEV(endpoint->major, + endpoint->lowest_minor), + endpoint->num_channels); + + dev_info(endpoint->dev, "Removed %d device files.\n", + endpoint->num_channels); +} + +struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev, + struct device *dev, + struct xilly_endpoint_hardware + *ephw) +{ + struct xilly_endpoint *endpoint; + + endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); + if (!endpoint) + return NULL; + + endpoint->pdev = pdev; + endpoint->dev = dev; + endpoint->ephw = ephw; + endpoint->msg_counter = 0x0b; + endpoint->failed_messages = 0; + endpoint->fatal_error = 0; + + init_waitqueue_head(&endpoint->ep_wait); + mutex_init(&endpoint->register_mutex); + + return endpoint; +} +EXPORT_SYMBOL(xillybus_init_endpoint); + +static int xilly_quiesce(struct xilly_endpoint *endpoint) +{ + long t; + + endpoint->idtlen = -1; + + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, + "Failed to quiesce the device on exit.\n"); + return -ENODEV; + } + return 0; +} + +int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) +{ + int rc; + long t; + + void *bootstrap_resources; + int idtbuffersize = (1 << PAGE_SHIFT); + struct device *dev = endpoint->dev; + + /* + * The bogus IDT is used during bootstrap for allocating the initial + * message buffer, and then the message buffer and space for the IDT + * itself. The initial message buffer is of a single page's size, but + * it's soon replaced with a more modest one (and memory is freed). + */ + + unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, + 3, 192, PAGE_SHIFT, 0 }; + struct xilly_idt_handle idt_handle; + + /* + * Writing the value 0x00000001 to Endianness register signals which + * endianness this processor is using, so the FPGA can swap words as + * necessary. + */ + + iowrite32(1, endpoint->registers + fpga_endian_reg); + + /* Bootstrap phase I: Allocate temporary message buffer */ + + bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); + if (!bootstrap_resources) + return -ENOMEM; + + endpoint->num_channels = 0; + + rc = xilly_setupchannels(endpoint, bogus_idt, 1); + if (rc) + return rc; + + /* Clear the message subsystem (and counter in particular) */ + iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); + + endpoint->idtlen = -1; + + /* + * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT + * buffer size. + */ + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); + return -ENODEV; + } + + /* Enable DMA */ + iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), + endpoint->registers + fpga_dma_control_reg); + + /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ + while (endpoint->idtlen >= idtbuffersize) { + idtbuffersize *= 2; + bogus_idt[6]++; + } + + endpoint->num_channels = 1; + + rc = xilly_setupchannels(endpoint, bogus_idt, 2); + if (rc) + goto failed_idt; + + rc = xilly_obtain_idt(endpoint); + if (rc) + goto failed_idt; + + rc = xilly_scan_idt(endpoint, &idt_handle); + if (rc) + goto failed_idt; + + devres_close_group(dev, bootstrap_resources); + + /* Bootstrap phase III: Allocate buffers according to IDT */ + + rc = xilly_setupchannels(endpoint, + idt_handle.chandesc, + idt_handle.entries); + if (rc) + goto failed_idt; + + /* + * endpoint is now completely configured. We put it on the list + * available to open() before registering the char device(s) + */ + + mutex_lock(&ep_list_lock); + list_add_tail(&endpoint->ep_list, &list_of_endpoints); + mutex_unlock(&ep_list_lock); + + rc = xillybus_init_chrdev(endpoint, idt_handle.idt); + if (rc) + goto failed_chrdevs; + + devres_release_group(dev, bootstrap_resources); + + return 0; + +failed_chrdevs: + mutex_lock(&ep_list_lock); + list_del(&endpoint->ep_list); + mutex_unlock(&ep_list_lock); + +failed_idt: + xilly_quiesce(endpoint); + flush_workqueue(xillybus_wq); + + return rc; +} +EXPORT_SYMBOL(xillybus_endpoint_discovery); + +void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) +{ + xillybus_cleanup_chrdev(endpoint); + + mutex_lock(&ep_list_lock); + list_del(&endpoint->ep_list); + mutex_unlock(&ep_list_lock); + + xilly_quiesce(endpoint); + + /* + * Flushing is done upon endpoint release to prevent access to memory + * just about to be released. This makes the quiesce complete. + */ + flush_workqueue(xillybus_wq); +} +EXPORT_SYMBOL(xillybus_endpoint_remove); + +static int __init xillybus_init(void) +{ + mutex_init(&ep_list_lock); + + xillybus_class = class_create(THIS_MODULE, xillyname); + if (IS_ERR(xillybus_class)) + return PTR_ERR(xillybus_class); + + xillybus_wq = alloc_workqueue(xillyname, 0, 0); + if (!xillybus_wq) { + class_destroy(xillybus_class); + return -ENOMEM; + } + + return 0; +} + +static void __exit xillybus_exit(void) +{ + /* flush_workqueue() was called for each endpoint released */ + destroy_workqueue(xillybus_wq); + + class_destroy(xillybus_class); +} + +module_init(xillybus_init); +module_exit(xillybus_exit); diff --git a/drivers/char/xillybus/xillybus_of.c b/drivers/char/xillybus/xillybus_of.c new file mode 100644 index 000000000000..1ca0c7a4f1be --- /dev/null +++ b/drivers/char/xillybus/xillybus_of.c @@ -0,0 +1,187 @@ +/* + * linux/drivers/misc/xillybus_of.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework using Open Firmware. + * + * This program is free software; you can redistribute it and/or modify + * it under the smems of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/err.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus driver for Open Firmware"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_VERSION("1.06"); +MODULE_ALIAS("xillybus_of"); +MODULE_LICENSE("GPL v2"); + +static const char xillyname[] = "xillybus_of"; + +/* Match table for of_platform binding */ +static struct of_device_id xillybus_of_match[] = { + { .compatible = "xillybus,xillybus-1.00.a", }, + { .compatible = "xlnx,xillybus-1.00.a", }, /* Deprecated */ + {} +}; + +MODULE_DEVICE_TABLE(of, xillybus_of_match); + +static void xilly_dma_sync_single_for_cpu_of(struct xilly_endpoint *ep, + dma_addr_t dma_handle, + size_t size, + int direction) +{ + dma_sync_single_for_cpu(ep->dev, dma_handle, size, direction); +} + +static void xilly_dma_sync_single_for_device_of(struct xilly_endpoint *ep, + dma_addr_t dma_handle, + size_t size, + int direction) +{ + dma_sync_single_for_device(ep->dev, dma_handle, size, direction); +} + +static void xilly_dma_sync_single_nop(struct xilly_endpoint *ep, + dma_addr_t dma_handle, + size_t size, + int direction) +{ +} + +static void xilly_of_unmap(void *ptr) +{ + struct xilly_mapping *data = ptr; + + dma_unmap_single(data->device, data->dma_addr, + data->size, data->direction); + + kfree(ptr); +} + +static int xilly_map_single_of(struct xilly_endpoint *ep, + void *ptr, + size_t size, + int direction, + dma_addr_t *ret_dma_handle + ) +{ + dma_addr_t addr; + struct xilly_mapping *this; + int rc; + + this = kzalloc(sizeof(*this), GFP_KERNEL); + if (!this) + return -ENOMEM; + + addr = dma_map_single(ep->dev, ptr, size, direction); + + if (dma_mapping_error(ep->dev, addr)) { + kfree(this); + return -ENODEV; + } + + this->device = ep->dev; + this->dma_addr = addr; + this->size = size; + this->direction = direction; + + *ret_dma_handle = addr; + + rc = devm_add_action(ep->dev, xilly_of_unmap, this); + + if (rc) { + dma_unmap_single(ep->dev, addr, size, direction); + kfree(this); + return rc; + } + + return 0; +} + +static struct xilly_endpoint_hardware of_hw = { + .owner = THIS_MODULE, + .hw_sync_sgl_for_cpu = xilly_dma_sync_single_for_cpu_of, + .hw_sync_sgl_for_device = xilly_dma_sync_single_for_device_of, + .map_single = xilly_map_single_of, +}; + +static struct xilly_endpoint_hardware of_hw_coherent = { + .owner = THIS_MODULE, + .hw_sync_sgl_for_cpu = xilly_dma_sync_single_nop, + .hw_sync_sgl_for_device = xilly_dma_sync_single_nop, + .map_single = xilly_map_single_of, +}; + +static int xilly_drv_probe(struct platform_device *op) +{ + struct device *dev = &op->dev; + struct xilly_endpoint *endpoint; + int rc; + int irq; + struct resource res; + struct xilly_endpoint_hardware *ephw = &of_hw; + + if (of_property_read_bool(dev->of_node, "dma-coherent")) + ephw = &of_hw_coherent; + + endpoint = xillybus_init_endpoint(NULL, dev, ephw); + + if (!endpoint) + return -ENOMEM; + + dev_set_drvdata(dev, endpoint); + + rc = of_address_to_resource(dev->of_node, 0, &res); + endpoint->registers = devm_ioremap_resource(dev, &res); + + if (IS_ERR(endpoint->registers)) + return PTR_ERR(endpoint->registers); + + irq = irq_of_parse_and_map(dev->of_node, 0); + + rc = devm_request_irq(dev, irq, xillybus_isr, 0, xillyname, endpoint); + + if (rc) { + dev_err(endpoint->dev, + "Failed to register IRQ handler. Aborting.\n"); + return -ENODEV; + } + + return xillybus_endpoint_discovery(endpoint); +} + +static int xilly_drv_remove(struct platform_device *op) +{ + struct device *dev = &op->dev; + struct xilly_endpoint *endpoint = dev_get_drvdata(dev); + + xillybus_endpoint_remove(endpoint); + + return 0; +} + +static struct platform_driver xillybus_platform_driver = { + .probe = xilly_drv_probe, + .remove = xilly_drv_remove, + .driver = { + .name = xillyname, + .owner = THIS_MODULE, + .of_match_table = xillybus_of_match, + }, +}; + +module_platform_driver(xillybus_platform_driver); diff --git a/drivers/char/xillybus/xillybus_pcie.c b/drivers/char/xillybus/xillybus_pcie.c new file mode 100644 index 000000000000..d8266bc2ae35 --- /dev/null +++ b/drivers/char/xillybus/xillybus_pcie.c @@ -0,0 +1,228 @@ +/* + * linux/drivers/misc/xillybus_pcie.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework using PCI Express. + * + * This program is free software; you can redistribute it and/or modify + * it under the smems of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pci-aspm.h> +#include <linux/slab.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus driver for PCIe"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_VERSION("1.06"); +MODULE_ALIAS("xillybus_pcie"); +MODULE_LICENSE("GPL v2"); + +#define PCI_DEVICE_ID_XILLYBUS 0xebeb + +#define PCI_VENDOR_ID_ALTERA 0x1172 +#define PCI_VENDOR_ID_ACTEL 0x11aa +#define PCI_VENDOR_ID_LATTICE 0x1204 + +static const char xillyname[] = "xillybus_pcie"; + +static const struct pci_device_id xillyids[] = { + {PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_ALTERA, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_ACTEL, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_LATTICE, PCI_DEVICE_ID_XILLYBUS)}, + { /* End: all zeroes */ } +}; + +static int xilly_pci_direction(int direction) +{ + switch (direction) { + case DMA_TO_DEVICE: + return PCI_DMA_TODEVICE; + case DMA_FROM_DEVICE: + return PCI_DMA_FROMDEVICE; + default: + return PCI_DMA_BIDIRECTIONAL; + } +} + +static void xilly_dma_sync_single_for_cpu_pci(struct xilly_endpoint *ep, + dma_addr_t dma_handle, + size_t size, + int direction) +{ + pci_dma_sync_single_for_cpu(ep->pdev, + dma_handle, + size, + xilly_pci_direction(direction)); +} + +static void xilly_dma_sync_single_for_device_pci(struct xilly_endpoint *ep, + dma_addr_t dma_handle, + size_t size, + int direction) +{ + pci_dma_sync_single_for_device(ep->pdev, + dma_handle, + size, + xilly_pci_direction(direction)); +} + +static void xilly_pci_unmap(void *ptr) +{ + struct xilly_mapping *data = ptr; + + pci_unmap_single(data->device, data->dma_addr, + data->size, data->direction); + + kfree(ptr); +} + +/* + * Map either through the PCI DMA mapper or the non_PCI one. Behind the + * scenes exactly the same functions are called with the same parameters, + * but that can change. + */ + +static int xilly_map_single_pci(struct xilly_endpoint *ep, + void *ptr, + size_t size, + int direction, + dma_addr_t *ret_dma_handle + ) +{ + int pci_direction; + dma_addr_t addr; + struct xilly_mapping *this; + int rc; + + this = kzalloc(sizeof(*this), GFP_KERNEL); + if (!this) + return -ENOMEM; + + pci_direction = xilly_pci_direction(direction); + + addr = pci_map_single(ep->pdev, ptr, size, pci_direction); + + if (pci_dma_mapping_error(ep->pdev, addr)) { + kfree(this); + return -ENODEV; + } + + this->device = ep->pdev; + this->dma_addr = addr; + this->size = size; + this->direction = pci_direction; + + *ret_dma_handle = addr; + + rc = devm_add_action(ep->dev, xilly_pci_unmap, this); + if (rc) { + pci_unmap_single(ep->pdev, addr, size, pci_direction); + kfree(this); + return rc; + } + + return 0; +} + +static struct xilly_endpoint_hardware pci_hw = { + .owner = THIS_MODULE, + .hw_sync_sgl_for_cpu = xilly_dma_sync_single_for_cpu_pci, + .hw_sync_sgl_for_device = xilly_dma_sync_single_for_device_pci, + .map_single = xilly_map_single_pci, +}; + +static int xilly_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct xilly_endpoint *endpoint; + int rc; + + endpoint = xillybus_init_endpoint(pdev, &pdev->dev, &pci_hw); + + if (!endpoint) + return -ENOMEM; + + pci_set_drvdata(pdev, endpoint); + + rc = pcim_enable_device(pdev); + if (rc) { + dev_err(endpoint->dev, + "pcim_enable_device() failed. Aborting.\n"); + return rc; + } + + /* L0s has caused packet drops. No power saving, thank you. */ + + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S); + + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { + dev_err(endpoint->dev, + "Incorrect BAR configuration. Aborting.\n"); + return -ENODEV; + } + + rc = pcim_iomap_regions(pdev, 0x01, xillyname); + if (rc) { + dev_err(endpoint->dev, + "pcim_iomap_regions() failed. Aborting.\n"); + return rc; + } + + endpoint->registers = pcim_iomap_table(pdev)[0]; + + pci_set_master(pdev); + + /* Set up a single MSI interrupt */ + if (pci_enable_msi(pdev)) { + dev_err(endpoint->dev, + "Failed to enable MSI interrupts. Aborting.\n"); + return -ENODEV; + } + rc = devm_request_irq(&pdev->dev, pdev->irq, xillybus_isr, 0, + xillyname, endpoint); + if (rc) { + dev_err(endpoint->dev, + "Failed to register MSI handler. Aborting.\n"); + return -ENODEV; + } + + /* + * In theory, an attempt to set the DMA mask to 64 and dma_using_dac=1 + * is the right thing. But some unclever PCIe drivers report it's OK + * when the hardware drops those 64-bit PCIe packets. So trust + * nobody and use 32 bits DMA addressing in any case. + */ + + if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + endpoint->dma_using_dac = 0; + } else { + dev_err(endpoint->dev, "Failed to set DMA mask. Aborting.\n"); + return -ENODEV; + } + + return xillybus_endpoint_discovery(endpoint); +} + +static void xilly_remove(struct pci_dev *pdev) +{ + struct xilly_endpoint *endpoint = pci_get_drvdata(pdev); + + xillybus_endpoint_remove(endpoint); +} + +MODULE_DEVICE_TABLE(pci, xillyids); + +static struct pci_driver xillybus_driver = { + .name = xillyname, + .id_table = xillyids, + .probe = xilly_probe, + .remove = xilly_remove, +}; + +module_pci_driver(xillybus_driver); |