diff options
Diffstat (limited to 'drivers/misc/mic')
| -rw-r--r-- | drivers/misc/mic/Kconfig | 3 | ||||
| -rw-r--r-- | drivers/misc/mic/bus/scif_bus.h | 8 | ||||
| -rw-r--r-- | drivers/misc/mic/bus/vop_bus.h | 8 | ||||
| -rw-r--r-- | drivers/misc/mic/card/mic_debugfs.c | 24 | ||||
| -rw-r--r-- | drivers/misc/mic/card/mic_device.c | 8 | ||||
| -rw-r--r-- | drivers/misc/mic/cosm/cosm_debugfs.c | 39 | ||||
| -rw-r--r-- | drivers/misc/mic/host/mic_boot.c | 10 | ||||
| -rw-r--r-- | drivers/misc/mic/host/mic_debugfs.c | 62 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_debugfs.c | 44 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_dma.c | 11 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_fence.c | 22 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_map.h | 4 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_rma.c | 42 | ||||
| -rw-r--r-- | drivers/misc/mic/scif/scif_rma.h | 15 | ||||
| -rw-r--r-- | drivers/misc/mic/vop/vop_debugfs.c | 40 | ||||
| -rw-r--r-- | drivers/misc/mic/vop/vop_main.c | 124 | ||||
| -rw-r--r-- | drivers/misc/mic/vop/vop_vringh.c | 51 |
17 files changed, 208 insertions, 307 deletions
diff --git a/drivers/misc/mic/Kconfig b/drivers/misc/mic/Kconfig index 227cc7443671..242dcee14689 100644 --- a/drivers/misc/mic/Kconfig +++ b/drivers/misc/mic/Kconfig @@ -38,7 +38,6 @@ comment "VOP Bus Driver" config VOP_BUS tristate "VOP Bus Driver" - depends on 64BIT && PCI && X86 && X86_DEV_DMA_OPS help This option is selected by any driver which registers a device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST @@ -132,7 +131,7 @@ comment "VOP Driver" config VOP tristate "VOP Driver" - depends on 64BIT && PCI && X86 && VOP_BUS + depends on VOP_BUS select VHOST_RING select VIRTIO help diff --git a/drivers/misc/mic/bus/scif_bus.h b/drivers/misc/mic/bus/scif_bus.h index ff59568219ad..377a4f38cd7e 100644 --- a/drivers/misc/mic/bus/scif_bus.h +++ b/drivers/misc/mic/bus/scif_bus.h @@ -88,8 +88,8 @@ struct scif_driver { * @send_intr: Send an interrupt to the remote node on a specified doorbell. * @send_p2p_intr: Send an interrupt to the peer node on a specified doorbell * which is specifically targeted for a peer to peer node. - * @ioremap: Map a buffer with the specified physical address and length. - * @iounmap: Unmap a buffer previously mapped. + * @remap: Map a buffer with the specified physical address and length. + * @unmap: Unmap a buffer previously mapped. */ struct scif_hw_ops { int (*next_db)(struct scif_hw_dev *sdev); @@ -104,9 +104,9 @@ struct scif_hw_ops { void (*send_intr)(struct scif_hw_dev *sdev, int db); void (*send_p2p_intr)(struct scif_hw_dev *sdev, int db, struct mic_mw *mw); - void __iomem * (*ioremap)(struct scif_hw_dev *sdev, + void __iomem * (*remap)(struct scif_hw_dev *sdev, phys_addr_t pa, size_t len); - void (*iounmap)(struct scif_hw_dev *sdev, void __iomem *va); + void (*unmap)(struct scif_hw_dev *sdev, void __iomem *va); }; int scif_register_driver(struct scif_driver *driver); diff --git a/drivers/misc/mic/bus/vop_bus.h b/drivers/misc/mic/bus/vop_bus.h index fff7a865d721..cf5f3fae573c 100644 --- a/drivers/misc/mic/bus/vop_bus.h +++ b/drivers/misc/mic/bus/vop_bus.h @@ -87,8 +87,8 @@ struct vop_driver { * @get_dp: Get access to the virtio device page used by the self * node to add/remove/configure virtio devices. * @send_intr: Send an interrupt to the peer node on a specified doorbell. - * @ioremap: Map a buffer with the specified DMA address and length. - * @iounmap: Unmap a buffer previously mapped. + * @remap: Map a buffer with the specified DMA address and length. + * @unmap: Unmap a buffer previously mapped. * @dma_filter: The DMA filter function to use for obtaining access to * a DMA channel on the peer node. */ @@ -104,9 +104,9 @@ struct vop_hw_ops { void __iomem * (*get_remote_dp)(struct vop_device *vpdev); void * (*get_dp)(struct vop_device *vpdev); void (*send_intr)(struct vop_device *vpdev, int db); - void __iomem * (*ioremap)(struct vop_device *vpdev, + void __iomem * (*remap)(struct vop_device *vpdev, dma_addr_t pa, size_t len); - void (*iounmap)(struct vop_device *vpdev, void __iomem *va); + void (*unmap)(struct vop_device *vpdev, void __iomem *va); }; struct vop_device * diff --git a/drivers/misc/mic/card/mic_debugfs.c b/drivers/misc/mic/card/mic_debugfs.c index 421b3d7911df..7a4140874888 100644 --- a/drivers/misc/mic/card/mic_debugfs.c +++ b/drivers/misc/mic/card/mic_debugfs.c @@ -37,9 +37,9 @@ static struct dentry *mic_dbg; /** - * mic_intr_test - Send interrupts to host. + * mic_intr_show - Send interrupts to host. */ -static int mic_intr_test(struct seq_file *s, void *unused) +static int mic_intr_show(struct seq_file *s, void *unused) { struct mic_driver *mdrv = s->private; struct mic_device *mdev = &mdrv->mdev; @@ -56,23 +56,7 @@ static int mic_intr_test(struct seq_file *s, void *unused) return 0; } -static int mic_intr_test_open(struct inode *inode, struct file *file) -{ - return single_open(file, mic_intr_test, inode->i_private); -} - -static int mic_intr_test_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations intr_test_ops = { - .owner = THIS_MODULE, - .open = mic_intr_test_open, - .read = seq_read, - .llseek = seq_lseek, - .release = mic_intr_test_release -}; +DEFINE_SHOW_ATTRIBUTE(mic_intr); /** * mic_create_card_debug_dir - Initialize MIC debugfs entries. @@ -91,7 +75,7 @@ void __init mic_create_card_debug_dir(struct mic_driver *mdrv) } d = debugfs_create_file("intr_test", 0444, mdrv->dbg_dir, - mdrv, &intr_test_ops); + mdrv, &mic_intr_fops); if (!d) { dev_err(mdrv->dev, diff --git a/drivers/misc/mic/card/mic_device.c b/drivers/misc/mic/card/mic_device.c index e749af48f736..dcd07ef29801 100644 --- a/drivers/misc/mic/card/mic_device.c +++ b/drivers/misc/mic/card/mic_device.c @@ -245,8 +245,8 @@ static struct scif_hw_ops scif_hw_ops = { .next_db = ___mic_next_db, .send_intr = ___mic_send_intr, .send_p2p_intr = ___mic_send_p2p_intr, - .ioremap = ___mic_ioremap, - .iounmap = ___mic_iounmap, + .remap = ___mic_ioremap, + .unmap = ___mic_iounmap, }; static inline struct mic_driver *vpdev_to_mdrv(struct vop_device *vpdev) @@ -316,8 +316,8 @@ static struct vop_hw_ops vop_hw_ops = { .next_db = __mic_next_db, .get_remote_dp = __mic_get_remote_dp, .send_intr = __mic_send_intr, - .ioremap = __mic_ioremap, - .iounmap = __mic_iounmap, + .remap = __mic_ioremap, + .unmap = __mic_iounmap, }; static int mic_request_dma_chans(struct mic_driver *mdrv) diff --git a/drivers/misc/mic/cosm/cosm_debugfs.c b/drivers/misc/mic/cosm/cosm_debugfs.c index 216cb3cd2fe3..71c216d0504d 100644 --- a/drivers/misc/mic/cosm/cosm_debugfs.c +++ b/drivers/misc/mic/cosm/cosm_debugfs.c @@ -28,12 +28,12 @@ static struct dentry *cosm_dbg; /** - * cosm_log_buf_show - Display MIC kernel log buffer + * log_buf_show - Display MIC kernel log buffer * * log_buf addr/len is read from System.map by user space * and populated in sysfs entries. */ -static int cosm_log_buf_show(struct seq_file *s, void *unused) +static int log_buf_show(struct seq_file *s, void *unused) { void __iomem *log_buf_va; int __iomem *log_buf_len_va; @@ -78,26 +78,15 @@ done: return 0; } -static int cosm_log_buf_open(struct inode *inode, struct file *file) -{ - return single_open(file, cosm_log_buf_show, inode->i_private); -} - -static const struct file_operations log_buf_ops = { - .owner = THIS_MODULE, - .open = cosm_log_buf_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release -}; +DEFINE_SHOW_ATTRIBUTE(log_buf); /** - * cosm_force_reset_show - Force MIC reset + * force_reset_show - Force MIC reset * * Invokes the force_reset COSM bus op instead of the standard reset * op in case a force reset of the MIC device is required */ -static int cosm_force_reset_show(struct seq_file *s, void *pos) +static int force_reset_show(struct seq_file *s, void *pos) { struct cosm_device *cdev = s->private; @@ -105,18 +94,7 @@ static int cosm_force_reset_show(struct seq_file *s, void *pos) return 0; } -static int cosm_force_reset_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, cosm_force_reset_show, inode->i_private); -} - -static const struct file_operations force_reset_ops = { - .owner = THIS_MODULE, - .open = cosm_force_reset_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release -}; +DEFINE_SHOW_ATTRIBUTE(force_reset); void cosm_create_debug_dir(struct cosm_device *cdev) { @@ -130,9 +108,10 @@ void cosm_create_debug_dir(struct cosm_device *cdev) if (!cdev->dbg_dir) return; - debugfs_create_file("log_buf", 0444, cdev->dbg_dir, cdev, &log_buf_ops); + debugfs_create_file("log_buf", 0444, cdev->dbg_dir, cdev, + &log_buf_fops); debugfs_create_file("force_reset", 0444, cdev->dbg_dir, cdev, - &force_reset_ops); + &force_reset_fops); } void cosm_delete_debug_dir(struct cosm_device *cdev) diff --git a/drivers/misc/mic/host/mic_boot.c b/drivers/misc/mic/host/mic_boot.c index c327985c9523..079c36f0ce6e 100644 --- a/drivers/misc/mic/host/mic_boot.c +++ b/drivers/misc/mic/host/mic_boot.c @@ -133,8 +133,8 @@ static struct vop_hw_ops vop_hw_ops = { .get_dp = __mic_get_dp, .get_remote_dp = __mic_get_remote_dp, .send_intr = __mic_send_intr, - .ioremap = __mic_ioremap, - .iounmap = __mic_iounmap, + .remap = __mic_ioremap, + .unmap = __mic_iounmap, }; static inline struct mic_device *scdev_to_mdev(struct scif_hw_dev *scdev) @@ -149,7 +149,7 @@ static void *__mic_dma_alloc(struct device *dev, size_t size, struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); dma_addr_t tmp; - void *va = kmalloc(size, gfp); + void *va = kmalloc(size, gfp | __GFP_ZERO); if (va) { tmp = mic_map_single(mdev, va, size); @@ -315,8 +315,8 @@ static struct scif_hw_ops scif_hw_ops = { .ack_interrupt = ___mic_ack_interrupt, .next_db = ___mic_next_db, .send_intr = ___mic_send_intr, - .ioremap = ___mic_ioremap, - .iounmap = ___mic_iounmap, + .remap = ___mic_ioremap, + .unmap = ___mic_iounmap, }; static inline struct mic_device *mbdev_to_mdev(struct mbus_device *mbdev) diff --git a/drivers/misc/mic/host/mic_debugfs.c b/drivers/misc/mic/host/mic_debugfs.c index 0a9daba8bb5d..c6e3c764699f 100644 --- a/drivers/misc/mic/host/mic_debugfs.c +++ b/drivers/misc/mic/host/mic_debugfs.c @@ -54,23 +54,7 @@ static int mic_smpt_show(struct seq_file *s, void *pos) return 0; } -static int mic_smpt_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, mic_smpt_show, inode->i_private); -} - -static int mic_smpt_debug_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations smpt_file_ops = { - .owner = THIS_MODULE, - .open = mic_smpt_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = mic_smpt_debug_release -}; +DEFINE_SHOW_ATTRIBUTE(mic_smpt); static int mic_post_code_show(struct seq_file *s, void *pos) { @@ -81,23 +65,7 @@ static int mic_post_code_show(struct seq_file *s, void *pos) return 0; } -static int mic_post_code_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, mic_post_code_show, inode->i_private); -} - -static int mic_post_code_debug_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations post_code_ops = { - .owner = THIS_MODULE, - .open = mic_post_code_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = mic_post_code_debug_release -}; +DEFINE_SHOW_ATTRIBUTE(mic_post_code); static int mic_msi_irq_info_show(struct seq_file *s, void *pos) { @@ -143,24 +111,7 @@ static int mic_msi_irq_info_show(struct seq_file *s, void *pos) return 0; } -static int mic_msi_irq_info_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, mic_msi_irq_info_show, inode->i_private); -} - -static int -mic_msi_irq_info_debug_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations msi_irq_info_ops = { - .owner = THIS_MODULE, - .open = mic_msi_irq_info_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = mic_msi_irq_info_debug_release -}; +DEFINE_SHOW_ATTRIBUTE(mic_msi_irq_info); /** * mic_create_debug_dir - Initialize MIC debugfs entries. @@ -177,13 +128,14 @@ void mic_create_debug_dir(struct mic_device *mdev) if (!mdev->dbg_dir) return; - debugfs_create_file("smpt", 0444, mdev->dbg_dir, mdev, &smpt_file_ops); + debugfs_create_file("smpt", 0444, mdev->dbg_dir, mdev, + &mic_smpt_fops); debugfs_create_file("post_code", 0444, mdev->dbg_dir, mdev, - &post_code_ops); + &mic_post_code_fops); debugfs_create_file("msi_irq_info", 0444, mdev->dbg_dir, mdev, - &msi_irq_info_ops); + &mic_msi_irq_info_fops); } /** diff --git a/drivers/misc/mic/scif/scif_debugfs.c b/drivers/misc/mic/scif/scif_debugfs.c index 6884dad97e17..cca5e980c710 100644 --- a/drivers/misc/mic/scif/scif_debugfs.c +++ b/drivers/misc/mic/scif/scif_debugfs.c @@ -24,7 +24,7 @@ /* Debugfs parent dir */ static struct dentry *scif_dbg; -static int scif_dev_test(struct seq_file *s, void *unused) +static int scif_dev_show(struct seq_file *s, void *unused) { int node; @@ -44,23 +44,7 @@ static int scif_dev_test(struct seq_file *s, void *unused) return 0; } -static int scif_dev_test_open(struct inode *inode, struct file *file) -{ - return single_open(file, scif_dev_test, inode->i_private); -} - -static int scif_dev_test_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations scif_dev_ops = { - .owner = THIS_MODULE, - .open = scif_dev_test_open, - .read = seq_read, - .llseek = seq_lseek, - .release = scif_dev_test_release -}; +DEFINE_SHOW_ATTRIBUTE(scif_dev); static void scif_display_window(struct scif_window *window, struct seq_file *s) { @@ -104,7 +88,7 @@ static void scif_display_all_windows(struct list_head *head, struct seq_file *s) } } -static int scif_rma_test(struct seq_file *s, void *unused) +static int scif_rma_show(struct seq_file *s, void *unused) { struct scif_endpt *ep; struct list_head *pos; @@ -123,23 +107,7 @@ static int scif_rma_test(struct seq_file *s, void *unused) return 0; } -static int scif_rma_test_open(struct inode *inode, struct file *file) -{ - return single_open(file, scif_rma_test, inode->i_private); -} - -static int scif_rma_test_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations scif_rma_ops = { - .owner = THIS_MODULE, - .open = scif_rma_test_open, - .read = seq_read, - .llseek = seq_lseek, - .release = scif_rma_test_release -}; +DEFINE_SHOW_ATTRIBUTE(scif_rma); void __init scif_init_debugfs(void) { @@ -150,8 +118,8 @@ void __init scif_init_debugfs(void) return; } - debugfs_create_file("scif_dev", 0444, scif_dbg, NULL, &scif_dev_ops); - debugfs_create_file("scif_rma", 0444, scif_dbg, NULL, &scif_rma_ops); + debugfs_create_file("scif_dev", 0444, scif_dbg, NULL, &scif_dev_fops); + debugfs_create_file("scif_rma", 0444, scif_dbg, NULL, &scif_rma_fops); debugfs_create_u8("en_msg_log", 0666, scif_dbg, &scif_info.en_msg_log); debugfs_create_u8("p2p_enable", 0666, scif_dbg, &scif_info.p2p_enable); } diff --git a/drivers/misc/mic/scif/scif_dma.c b/drivers/misc/mic/scif/scif_dma.c index 18b8ed57c4ac..e0d97044d0e9 100644 --- a/drivers/misc/mic/scif/scif_dma.c +++ b/drivers/misc/mic/scif/scif_dma.c @@ -201,23 +201,18 @@ static void scif_mmu_notifier_release(struct mmu_notifier *mn, } static int scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, - unsigned long end, - bool blockable) + const struct mmu_notifier_range *range) { struct scif_mmu_notif *mmn; mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier); - scif_rma_destroy_tcw(mmn, start, end - start); + scif_rma_destroy_tcw(mmn, range->start, range->end - range->start); return 0; } static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, - unsigned long end) + const struct mmu_notifier_range *range) { /* * Nothing to do here, everything needed was done in diff --git a/drivers/misc/mic/scif/scif_fence.c b/drivers/misc/mic/scif/scif_fence.c index 7bb929f05d85..2e7ce6ae9dd2 100644 --- a/drivers/misc/mic/scif/scif_fence.c +++ b/drivers/misc/mic/scif/scif_fence.c @@ -195,10 +195,11 @@ static inline void *scif_get_local_va(off_t off, struct scif_window *window) static void scif_prog_signal_cb(void *arg) { - struct scif_status *status = arg; + struct scif_cb_arg *cb_arg = arg; - dma_pool_free(status->ep->remote_dev->signal_pool, status, - status->src_dma_addr); + dma_pool_free(cb_arg->ep->remote_dev->signal_pool, cb_arg->status, + cb_arg->src_dma_addr); + kfree(cb_arg); } static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val) @@ -209,6 +210,7 @@ static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val) bool x100 = !is_dma_copy_aligned(chan->device, 1, 1, 1); struct dma_async_tx_descriptor *tx; struct scif_status *status = NULL; + struct scif_cb_arg *cb_arg = NULL; dma_addr_t src; dma_cookie_t cookie; int err; @@ -257,8 +259,16 @@ static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val) goto dma_fail; } if (!x100) { + cb_arg = kmalloc(sizeof(*cb_arg), GFP_KERNEL); + if (!cb_arg) { + err = -ENOMEM; + goto dma_fail; + } + cb_arg->src_dma_addr = src; + cb_arg->status = status; + cb_arg->ep = ep; tx->callback = scif_prog_signal_cb; - tx->callback_param = status; + tx->callback_param = cb_arg; } cookie = tx->tx_submit(tx); if (dma_submit_error(cookie)) { @@ -270,9 +280,11 @@ static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val) dma_async_issue_pending(chan); return 0; dma_fail: - if (!x100) + if (!x100) { dma_pool_free(ep->remote_dev->signal_pool, status, src - offsetof(struct scif_status, val)); + kfree(cb_arg); + } alloc_fail: return err; } diff --git a/drivers/misc/mic/scif/scif_map.h b/drivers/misc/mic/scif/scif_map.h index 3e86360ba5a6..7b380534eba1 100644 --- a/drivers/misc/mic/scif/scif_map.h +++ b/drivers/misc/mic/scif/scif_map.h @@ -97,7 +97,7 @@ scif_ioremap(dma_addr_t phys, size_t size, struct scif_dev *scifdev) out_virt = phys_to_virt(phys); else out_virt = (void __force *) - sdev->hw_ops->ioremap(sdev, phys, size); + sdev->hw_ops->remap(sdev, phys, size); return out_virt; } @@ -107,7 +107,7 @@ scif_iounmap(void *virt, size_t len, struct scif_dev *scifdev) if (!scifdev_self(scifdev)) { struct scif_hw_dev *sdev = scifdev->sdev; - sdev->hw_ops->iounmap(sdev, (void __force __iomem *)virt); + sdev->hw_ops->unmap(sdev, (void __force __iomem *)virt); } } diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index 0e4193cb08cf..0fb9b440dc70 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -15,7 +15,7 @@ * Intel SCIF driver. * */ -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> @@ -272,21 +272,12 @@ static inline void __scif_release_mm(struct mm_struct *mm) static inline int __scif_dec_pinned_vm_lock(struct mm_struct *mm, - int nr_pages, bool try_lock) + int nr_pages) { if (!mm || !nr_pages || !scif_ulimit_check) return 0; - if (try_lock) { - if (!down_write_trylock(&mm->mmap_sem)) { - dev_err(scif_info.mdev.this_device, - "%s %d err\n", __func__, __LINE__); - return -1; - } - } else { - down_write(&mm->mmap_sem); - } - mm->pinned_vm -= nr_pages; - up_write(&mm->mmap_sem); + + atomic64_sub(nr_pages, &mm->pinned_vm); return 0; } @@ -298,16 +289,16 @@ static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm, if (!mm || !nr_pages || !scif_ulimit_check) return 0; - locked = nr_pages; - locked += mm->pinned_vm; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + locked = atomic64_add_return(nr_pages, &mm->pinned_vm); + if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) { + atomic64_sub(nr_pages, &mm->pinned_vm); dev_err(scif_info.mdev.this_device, "locked(%lu) > lock_limit(%lu)\n", locked, lock_limit); return -ENOMEM; } - mm->pinned_vm = locked; return 0; } @@ -326,7 +317,7 @@ int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window) might_sleep(); if (!window->temp && window->mm) { - __scif_dec_pinned_vm_lock(window->mm, window->nr_pages, 0); + __scif_dec_pinned_vm_lock(window->mm, window->nr_pages); __scif_release_mm(window->mm); window->mm = NULL; } @@ -672,8 +663,8 @@ int scif_unregister_window(struct scif_window *window) { window->unreg_state = OP_IN_PROGRESS; send_msg = true; - /* fall through */ } + /* fall through */ case OP_IN_PROGRESS: { scif_get_window(window, 1); @@ -737,7 +728,7 @@ done: ep->rma_info.dma_chan); } else { if (!__scif_dec_pinned_vm_lock(window->mm, - window->nr_pages, 1)) { + window->nr_pages)) { __scif_release_mm(window->mm); window->mm = NULL; } @@ -1385,28 +1376,23 @@ int __scif_pin_pages(void *addr, size_t len, int *out_prot, prot |= SCIF_PROT_WRITE; retry: mm = current->mm; - down_write(&mm->mmap_sem); if (ulimit) { err = __scif_check_inc_pinned_vm(mm, nr_pages); if (err) { - up_write(&mm->mmap_sem); pinned_pages->nr_pages = 0; goto error_unmap; } } - pinned_pages->nr_pages = get_user_pages( + pinned_pages->nr_pages = get_user_pages_fast( (u64)addr, nr_pages, (prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0, - pinned_pages->pages, - NULL); - up_write(&mm->mmap_sem); + pinned_pages->pages); if (nr_pages != pinned_pages->nr_pages) { if (try_upgrade) { if (ulimit) - __scif_dec_pinned_vm_lock(mm, - nr_pages, 0); + __scif_dec_pinned_vm_lock(mm, nr_pages); /* Roll back any pinned pages */ for (i = 0; i < pinned_pages->nr_pages; i++) { if (pinned_pages->pages[i]) @@ -1433,7 +1419,7 @@ retry: return err; dec_pinned: if (ulimit) - __scif_dec_pinned_vm_lock(mm, nr_pages, 0); + __scif_dec_pinned_vm_lock(mm, nr_pages); /* Something went wrong! Rollback */ error_unmap: pinned_pages->nr_pages = nr_pages; diff --git a/drivers/misc/mic/scif/scif_rma.h b/drivers/misc/mic/scif/scif_rma.h index fa6722279196..964dd0fc3657 100644 --- a/drivers/misc/mic/scif/scif_rma.h +++ b/drivers/misc/mic/scif/scif_rma.h @@ -53,7 +53,7 @@ #ifndef SCIF_RMA_H #define SCIF_RMA_H -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/mmu_notifier.h> #include "../bus/scif_bus.h" @@ -206,6 +206,19 @@ struct scif_status { }; /* + * struct scif_cb_arg - Stores the argument of the callback func + * + * @src_dma_addr: Source buffer DMA address + * @status: DMA status + * @ep: SCIF endpoint + */ +struct scif_cb_arg { + dma_addr_t src_dma_addr; + struct scif_status *status; + struct scif_endpt *ep; +}; + +/* * struct scif_window - Registration Window for Self and Remote * * @nr_pages: Number of pages which is defined as a s64 instead of an int diff --git a/drivers/misc/mic/vop/vop_debugfs.c b/drivers/misc/mic/vop/vop_debugfs.c index ab43884e5cd7..2ccef52aca23 100644 --- a/drivers/misc/mic/vop/vop_debugfs.c +++ b/drivers/misc/mic/vop/vop_debugfs.c @@ -101,23 +101,7 @@ static int vop_dp_show(struct seq_file *s, void *pos) return 0; } -static int vop_dp_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, vop_dp_show, inode->i_private); -} - -static int vop_dp_debug_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations dp_ops = { - .owner = THIS_MODULE, - .open = vop_dp_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = vop_dp_debug_release -}; +DEFINE_SHOW_ATTRIBUTE(vop_dp); static int vop_vdev_info_show(struct seq_file *s, void *unused) { @@ -194,23 +178,7 @@ static int vop_vdev_info_show(struct seq_file *s, void *unused) return 0; } -static int vop_vdev_info_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, vop_vdev_info_show, inode->i_private); -} - -static int vop_vdev_info_debug_release(struct inode *inode, struct file *file) -{ - return single_release(inode, file); -} - -static const struct file_operations vdev_info_ops = { - .owner = THIS_MODULE, - .open = vop_vdev_info_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = vop_vdev_info_debug_release -}; +DEFINE_SHOW_ATTRIBUTE(vop_vdev_info); void vop_init_debugfs(struct vop_info *vi) { @@ -222,8 +190,8 @@ void vop_init_debugfs(struct vop_info *vi) pr_err("can't create debugfs dir vop\n"); return; } - debugfs_create_file("dp", 0444, vi->dbg, vi, &dp_ops); - debugfs_create_file("vdev_info", 0444, vi->dbg, vi, &vdev_info_ops); + debugfs_create_file("dp", 0444, vi->dbg, vi, &vop_dp_fops); + debugfs_create_file("vdev_info", 0444, vi->dbg, vi, &vop_vdev_info_fops); } void vop_exit_debugfs(struct vop_info *vi) diff --git a/drivers/misc/mic/vop/vop_main.c b/drivers/misc/mic/vop/vop_main.c index 3633202e18f4..e37b2c2152a2 100644 --- a/drivers/misc/mic/vop/vop_main.c +++ b/drivers/misc/mic/vop/vop_main.c @@ -34,6 +34,7 @@ #include <linux/module.h> #include <linux/sched.h> #include <linux/dma-mapping.h> +#include <linux/io-64-nonatomic-lo-hi.h> #include "vop_main.h" @@ -47,7 +48,8 @@ * @dc: Virtio device control * @vpdev: VOP device which is the parent for this virtio device * @vr: Buffer for accessing the VRING - * @used: Buffer for used + * @used_virt: Virtual address of used ring + * @used: DMA address of used ring * @used_size: Size of the used buffer * @reset_done: Track whether VOP reset is complete * @virtio_cookie: Cookie returned upon requesting a interrupt @@ -61,6 +63,7 @@ struct _vop_vdev { struct mic_device_ctrl __iomem *dc; struct vop_device *vpdev; void __iomem *vr[VOP_MAX_VRINGS]; + void *used_virt[VOP_MAX_VRINGS]; dma_addr_t used[VOP_MAX_VRINGS]; int used_size[VOP_MAX_VRINGS]; struct completion reset_done; @@ -116,7 +119,7 @@ _vop_total_desc_size(struct mic_device_desc __iomem *desc) static u64 vop_get_features(struct virtio_device *vdev) { unsigned int i, bits; - u32 features = 0; + u64 features = 0; struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc; u8 __iomem *in_features = _vop_vq_features(desc); int feature_len = ioread8(&desc->feature_len); @@ -124,11 +127,21 @@ static u64 vop_get_features(struct virtio_device *vdev) bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8; for (i = 0; i < bits; i++) if (ioread8(&in_features[i / 8]) & (BIT(i % 8))) - features |= BIT(i); + features |= BIT_ULL(i); return features; } +static void vop_transport_features(struct virtio_device *vdev) +{ + /* + * Packed ring isn't enabled on virtio_vop for now, + * because virtio_vop uses vring_new_virtqueue() which + * creates virtio rings on preallocated memory. + */ + __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED); +} + static int vop_finalize_features(struct virtio_device *vdev) { unsigned int i, bits; @@ -141,6 +154,9 @@ static int vop_finalize_features(struct virtio_device *vdev) /* Give virtio_ring a chance to accept features. */ vring_transport_features(vdev); + /* Give virtio_vop a chance to accept features. */ + vop_transport_features(vdev); + memset_io(out_features, 0, feature_len); bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8; @@ -213,7 +229,7 @@ static void vop_reset_inform_host(struct virtio_device *dev) if (ioread8(&dc->host_ack)) break; msleep(100); - }; + } dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry); @@ -247,14 +263,14 @@ static bool vop_notify(struct virtqueue *vq) static void vop_del_vq(struct virtqueue *vq, int n) { struct _vop_vdev *vdev = to_vopvdev(vq->vdev); - struct vring *vr = (struct vring *)(vq + 1); struct vop_device *vpdev = vdev->vpdev; dma_unmap_single(&vpdev->dev, vdev->used[n], vdev->used_size[n], DMA_BIDIRECTIONAL); - free_pages((unsigned long)vr->used, get_order(vdev->used_size[n])); + free_pages((unsigned long)vdev->used_virt[n], + get_order(vdev->used_size[n])); vring_del_virtqueue(vq); - vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]); + vpdev->hw_ops->unmap(vpdev, vdev->vr[n]); vdev->vr[n] = NULL; } @@ -270,6 +286,26 @@ static void vop_del_vqs(struct virtio_device *dev) vop_del_vq(vq, idx++); } +static struct virtqueue *vop_new_virtqueue(unsigned int index, + unsigned int num, + struct virtio_device *vdev, + bool context, + void *pages, + bool (*notify)(struct virtqueue *vq), + void (*callback)(struct virtqueue *vq), + const char *name, + void *used) +{ + bool weak_barriers = false; + struct vring vring; + + vring_init(&vring, num, pages, MIC_VIRTIO_RING_ALIGN); + vring.used = used; + + return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, + notify, callback, name); +} + /* * This routine will assign vring's allocated in host/io memory. Code in * virtio_ring.c however continues to access this io memory as if it were local @@ -289,7 +325,6 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev, struct _mic_vring_info __iomem *info; void *used; int vr_size, _vr_size, err, magic; - struct vring *vr; u8 type = ioread8(&vdev->desc->type); if (index >= ioread8(&vdev->desc->num_vq)) @@ -303,23 +338,12 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev, memcpy_fromio(&config, vqconfig, sizeof(config)); _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN); vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info)); - va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address), - vr_size); + va = vpdev->hw_ops->remap(vpdev, le64_to_cpu(config.address), vr_size); if (!va) return ERR_PTR(-ENOMEM); vdev->vr[index] = va; memset_io(va, 0x0, _vr_size); - vq = vring_new_virtqueue( - index, - le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN, - dev, - false, - ctx, - (void __force *)va, vop_notify, callback, name); - if (!vq) { - err = -ENOMEM; - goto unmap; - } + info = va + _vr_size; magic = ioread32(&info->magic); @@ -328,18 +352,27 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev, goto unmap; } - /* Allocate and reassign used ring now */ vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * le16_to_cpu(config.num)); used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(vdev->used_size[index])); + vdev->used_virt[index] = used; if (!used) { err = -ENOMEM; dev_err(_vop_dev(vdev), "%s %d err %d\n", __func__, __LINE__, err); - goto del_vq; + goto unmap; + } + + vq = vop_new_virtqueue(index, le16_to_cpu(config.num), dev, ctx, + (void __force *)va, vop_notify, callback, + name, used); + if (!vq) { + err = -ENOMEM; + goto free_used; } + vdev->used[index] = dma_map_single(&vpdev->dev, used, vdev->used_size[index], DMA_BIDIRECTIONAL); @@ -347,28 +380,19 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev, err = -ENOMEM; dev_err(_vop_dev(vdev), "%s %d err %d\n", __func__, __LINE__, err); - goto free_used; + goto del_vq; } writeq(vdev->used[index], &vqconfig->used_address); - /* - * To reassign the used ring here we are directly accessing - * struct vring_virtqueue which is a private data structure - * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in - * vring_new_virtqueue() would ensure that - * (&vq->vring == (struct vring *) (&vq->vq + 1)); - */ - vr = (struct vring *)(vq + 1); - vr->used = used; vq->priv = vdev; return vq; +del_vq: + vring_del_virtqueue(vq); free_used: free_pages((unsigned long)used, get_order(vdev->used_size[index])); -del_vq: - vring_del_virtqueue(vq); unmap: - vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]); + vpdev->hw_ops->unmap(vpdev, vdev->vr[index]); return ERR_PTR(err); } @@ -381,16 +405,21 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs, struct _vop_vdev *vdev = to_vopvdev(dev); struct vop_device *vpdev = vdev->vpdev; struct mic_device_ctrl __iomem *dc = vdev->dc; - int i, err, retry; + int i, err, retry, queue_idx = 0; /* We must have this many virtqueues. */ if (nvqs > ioread8(&vdev->desc->num_vq)) return -ENOENT; for (i = 0; i < nvqs; ++i) { + if (!names[i]) { + vqs[i] = NULL; + continue; + } + dev_dbg(_vop_dev(vdev), "%s: %d: %s\n", __func__, i, names[i]); - vqs[i] = vop_find_vq(dev, i, callbacks[i], names[i], + vqs[i] = vop_find_vq(dev, queue_idx++, callbacks[i], names[i], ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); @@ -408,7 +437,7 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs, if (!ioread8(&dc->used_address_updated)) break; msleep(100); - }; + } dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry); if (!retry) { @@ -484,7 +513,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, vdev->desc = d; vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d); vdev->dnode = dnode; - vdev->vdev.priv = (void *)(u64)dnode; + vdev->vdev.priv = (void *)(unsigned long)dnode; init_completion(&vdev->reset_done); vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev); @@ -506,7 +535,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, offset, type); goto free_irq; } - writeq((u64)vdev, &vdev->dc->vdev); + writeq((unsigned long)vdev, &vdev->dc->vdev); dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n", __func__, offset, type, vdev); @@ -533,13 +562,18 @@ static int vop_match_desc(struct device *dev, void *data) return vdev->desc == (void __iomem *)data; } +static struct _vop_vdev *vop_dc_to_vdev(struct mic_device_ctrl *dc) +{ + return (struct _vop_vdev *)(unsigned long)readq(&dc->vdev); +} + static void _vop_handle_config_change(struct mic_device_desc __iomem *d, unsigned int offset, struct vop_device *vpdev) { struct mic_device_ctrl __iomem *dc = (void __iomem *)d + _vop_aligned_desc_size(d); - struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev); + struct _vop_vdev *vdev = vop_dc_to_vdev(dc); if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED) return; @@ -558,11 +592,13 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d, { struct mic_device_ctrl __iomem *dc = (void __iomem *)d + _vop_aligned_desc_size(d); - struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev); + struct _vop_vdev *vdev = vop_dc_to_vdev(dc); u8 status; int ret = -1; if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) { + struct device *dev = get_device(&vdev->vdev.dev); + dev_dbg(&vpdev->dev, "%s %d config_change %d type %d vdev %p\n", __func__, __LINE__, @@ -574,7 +610,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d, iowrite8(-1, &dc->h2c_vdev_db); if (status & VIRTIO_CONFIG_S_DRIVER_OK) wait_for_completion(&vdev->reset_done); - put_device(&vdev->vdev.dev); + put_device(dev); iowrite8(1, &dc->guest_ack); dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n", __func__, __LINE__, ioread8(&dc->guest_ack)); diff --git a/drivers/misc/mic/vop/vop_vringh.c b/drivers/misc/mic/vop/vop_vringh.c index cbc8ebcff5cf..3632fce40590 100644 --- a/drivers/misc/mic/vop/vop_vringh.c +++ b/drivers/misc/mic/vop/vop_vringh.c @@ -80,7 +80,7 @@ static void vop_virtio_init_post(struct vop_vdev *vdev) continue; } vdev->vvr[i].vrh.vring.used = - (void __force *)vpdev->hw_ops->ioremap( + (void __force *)vpdev->hw_ops->remap( vpdev, le64_to_cpu(vqconfig[i].used_address), used_size); @@ -528,15 +528,15 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, int vr_idx) { struct vop_device *vpdev = vdev->vpdev; - void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len); + void __iomem *dbuf = vpdev->hw_ops->remap(vpdev, daddr, len); struct vop_vringh *vvr = &vdev->vvr[vr_idx]; struct vop_info *vi = dev_get_drvdata(&vpdev->dev); - size_t dma_alignment = 1 << vi->dma_ch->device->copy_align; - bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); + size_t dma_alignment; + bool x200; size_t dma_offset, partlen; int err; - if (!VOP_USE_DMA) { + if (!VOP_USE_DMA || !vi->dma_ch) { if (copy_to_user(ubuf, (void __force *)dbuf, len)) { err = -EFAULT; dev_err(vop_dev(vdev), "%s %d err %d\n", @@ -548,6 +548,9 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, goto err; } + dma_alignment = 1 << vi->dma_ch->device->copy_align; + x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); + dma_offset = daddr - round_down(daddr, dma_alignment); daddr -= dma_offset; len += dma_offset; @@ -585,9 +588,9 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, } err = 0; err: - vpdev->hw_ops->iounmap(vpdev, dbuf); + vpdev->hw_ops->unmap(vpdev, dbuf); dev_dbg(vop_dev(vdev), - "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n", + "%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n", __func__, ubuf, dbuf, len, vr_idx); return err; } @@ -603,21 +606,26 @@ static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf, int vr_idx) { struct vop_device *vpdev = vdev->vpdev; - void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len); + void __iomem *dbuf = vpdev->hw_ops->remap(vpdev, daddr, len); struct vop_vringh *vvr = &vdev->vvr[vr_idx]; struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev); - size_t dma_alignment = 1 << vi->dma_ch->device->copy_align; - bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); + size_t dma_alignment; + bool x200; size_t partlen; - bool dma = VOP_USE_DMA; + bool dma = VOP_USE_DMA && vi->dma_ch; int err = 0; - if (daddr & (dma_alignment - 1)) { - vdev->tx_dst_unaligned += len; - dma = false; - } else if (ALIGN(len, dma_alignment) > dlen) { - vdev->tx_len_unaligned += len; - dma = false; + if (dma) { + dma_alignment = 1 << vi->dma_ch->device->copy_align; + x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); + + if (daddr & (dma_alignment - 1)) { + vdev->tx_dst_unaligned += len; + dma = false; + } else if (ALIGN(len, dma_alignment) > dlen) { + vdev->tx_len_unaligned += len; + dma = false; + } } if (!dma) @@ -668,9 +676,9 @@ memcpy: vdev->out_bytes += len; err = 0; err: - vpdev->hw_ops->iounmap(vpdev, dbuf); + vpdev->hw_ops->unmap(vpdev, dbuf); dev_dbg(vop_dev(vdev), - "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n", + "%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n", __func__, ubuf, dbuf, len, vr_idx); return err; } @@ -704,16 +712,17 @@ static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov, while (len && iov->i < iov->used) { struct kvec *kiov = &iov->iov[iov->i]; + unsigned long daddr = (unsigned long)kiov->iov_base; partlen = min(kiov->iov_len, len); if (read) ret = vop_virtio_copy_to_user(vdev, ubuf, partlen, - (u64)kiov->iov_base, + daddr, kiov->iov_len, vr_idx); else ret = vop_virtio_copy_from_user(vdev, ubuf, partlen, - (u64)kiov->iov_base, + daddr, kiov->iov_len, vr_idx); if (ret) { |