diff options
Diffstat (limited to 'drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c')
-rw-r--r-- | drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c | 324 |
1 files changed, 181 insertions, 143 deletions
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c index 1091b9f1dd07..2b500d85cebc 100644 --- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c +++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c @@ -45,16 +45,8 @@ #include <asm/pgtable.h> #include <soc/bcm2835/raspberrypi-firmware.h> -#define dmac_map_area __glue(_CACHE,_dma_map_area) -#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area) - -extern void dmac_map_area(const void *, size_t, int); -extern void dmac_unmap_area(const void *, size_t, int); - #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32) -#define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset)) - #include "vchiq_arm.h" #include "vchiq_2835.h" #include "vchiq_connected.h" @@ -70,13 +62,25 @@ typedef struct vchiq_2835_state_struct { VCHIQ_ARM_STATE_T arm_state; } VCHIQ_2835_ARM_STATE_T; +struct vchiq_pagelist_info { + PAGELIST_T *pagelist; + size_t pagelist_buffer_size; + dma_addr_t dma_addr; + enum dma_data_direction dma_dir; + unsigned int num_pages; + unsigned int pages_need_release; + struct page **pages; + struct scatterlist *scatterlist; + unsigned int scatterlist_mapped; +}; + static void __iomem *g_regs; static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE); static unsigned int g_fragments_size; static char *g_fragments_base; static char *g_free_fragments; static struct semaphore g_free_fragments_sema; -static unsigned long g_virt_to_bus_offset; +static struct device *g_dev; extern int vchiq_arm_log_level; @@ -85,12 +89,13 @@ static DEFINE_SEMAPHORE(g_free_fragments_mutex); static irqreturn_t vchiq_doorbell_irq(int irq, void *dev_id); -static int +static struct vchiq_pagelist_info * create_pagelist(char __user *buf, size_t count, unsigned short type, - struct task_struct *task, PAGELIST_T ** ppagelist); + struct task_struct *task); static void -free_pagelist(PAGELIST_T *pagelist, int actual); +free_pagelist(struct vchiq_pagelist_info *pagelistinfo, + int actual); int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state) { @@ -104,7 +109,14 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state) int slot_mem_size, frag_mem_size; int err, irq, i; - g_virt_to_bus_offset = virt_to_dma(dev, (void *)0); + /* + * VCHI messages between the CPU and firmware use + * 32-bit bus addresses. + */ + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + + if (err < 0) + return err; (void)of_property_read_u32(dev->of_node, "cache-line-size", &g_cache_line_size); @@ -121,7 +133,7 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state) return -ENOMEM; } - WARN_ON(((int)slot_mem & (PAGE_SIZE - 1)) != 0); + WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0); vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size); if (!vchiq_slot_zero) @@ -173,9 +185,10 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state) return err ? : -ENXIO; } + g_dev = dev; vchiq_log_info(vchiq_arm_log_level, - "vchiq_init - done (slots %x, phys %pad)", - (unsigned int)vchiq_slot_zero, &slot_phys); + "vchiq_init - done (slots %pK, phys %pad)", + vchiq_slot_zero, &slot_phys); vchiq_call_connected_callbacks(); @@ -213,47 +226,37 @@ remote_event_signal(REMOTE_EVENT_T *event) event->fired = 1; - dsb(); /* data barrier operation */ + dsb(sy); /* data barrier operation */ if (event->armed) writel(0, g_regs + BELL2); /* trigger vc interrupt */ } -int -vchiq_copy_from_user(void *dst, const void *src, int size) -{ - if ((uint32_t)src < TASK_SIZE) { - return copy_from_user(dst, src, size); - } else { - memcpy(dst, src, size); - return 0; - } -} - VCHIQ_STATUS_T vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle, void *offset, int size, int dir) { - PAGELIST_T *pagelist; - int ret; + struct vchiq_pagelist_info *pagelistinfo; WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID); - ret = create_pagelist((char __user *)offset, size, - (dir == VCHIQ_BULK_RECEIVE) - ? PAGELIST_READ - : PAGELIST_WRITE, - current, - &pagelist); - if (ret != 0) + pagelistinfo = create_pagelist((char __user *)offset, size, + (dir == VCHIQ_BULK_RECEIVE) + ? PAGELIST_READ + : PAGELIST_WRITE, + current); + + if (!pagelistinfo) return VCHIQ_ERROR; bulk->handle = memhandle; - bulk->data = VCHIQ_ARM_ADDRESS(pagelist); + bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr; - /* Store the pagelist address in remote_data, which isn't used by the - slave. */ - bulk->remote_data = pagelist; + /* + * Store the pagelistinfo address in remote_data, + * which isn't used by the slave. + */ + bulk->remote_data = pagelistinfo; return VCHIQ_SUCCESS; } @@ -262,7 +265,8 @@ void vchiq_complete_bulk(VCHIQ_BULK_T *bulk) { if (bulk && bulk->remote_data && bulk->actual) - free_pagelist((PAGELIST_T *)bulk->remote_data, bulk->actual); + free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data, + bulk->actual); } void @@ -350,57 +354,93 @@ vchiq_doorbell_irq(int irq, void *dev_id) return ret; } +static void +cleaup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo) +{ + if (pagelistinfo->scatterlist_mapped) { + dma_unmap_sg(g_dev, pagelistinfo->scatterlist, + pagelistinfo->num_pages, pagelistinfo->dma_dir); + } + + if (pagelistinfo->pages_need_release) { + unsigned int i; + + for (i = 0; i < pagelistinfo->num_pages; i++) + put_page(pagelistinfo->pages[i]); + } + + dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size, + pagelistinfo->pagelist, pagelistinfo->dma_addr); +} + /* There is a potential problem with partial cache lines (pages?) ** at the ends of the block when reading. If the CPU accessed anything in ** the same line (page?) then it may have pulled old data into the cache, ** obscuring the new data underneath. We can solve this by transferring the ** partial cache lines separately, and allowing the ARM to copy into the ** cached area. - -** N.B. This implementation plays slightly fast and loose with the Linux -** driver programming rules, e.g. its use of dmac_map_area instead of -** dma_map_single, but it isn't a multi-platform driver and it benefits -** from increased speed as a result. */ -static int +static struct vchiq_pagelist_info * create_pagelist(char __user *buf, size_t count, unsigned short type, - struct task_struct *task, PAGELIST_T ** ppagelist) + struct task_struct *task) { PAGELIST_T *pagelist; + struct vchiq_pagelist_info *pagelistinfo; struct page **pages; - unsigned long *addrs; - unsigned int num_pages, offset, i; - char *addr, *base_addr, *next_addr; - int run, addridx, actual_pages; - unsigned long *need_release; - - offset = (unsigned int)buf & (PAGE_SIZE - 1); + u32 *addrs; + unsigned int num_pages, offset, i, k; + int actual_pages; + size_t pagelist_size; + struct scatterlist *scatterlist, *sg; + int dma_buffers; + dma_addr_t dma_addr; + + offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1)); num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE; - *ppagelist = NULL; + pagelist_size = sizeof(PAGELIST_T) + + (num_pages * sizeof(u32)) + + (num_pages * sizeof(pages[0]) + + (num_pages * sizeof(struct scatterlist))) + + sizeof(struct vchiq_pagelist_info); /* Allocate enough storage to hold the page pointers and the page ** list */ - pagelist = kmalloc(sizeof(PAGELIST_T) + - (num_pages * sizeof(unsigned long)) + - sizeof(unsigned long) + - (num_pages * sizeof(pages[0])), - GFP_KERNEL); - - vchiq_log_trace(vchiq_arm_log_level, - "create_pagelist - %x", (unsigned int)pagelist); + pagelist = dma_zalloc_coherent(g_dev, + pagelist_size, + &dma_addr, + GFP_KERNEL); + + vchiq_log_trace(vchiq_arm_log_level, "create_pagelist - %pK", + pagelist); if (!pagelist) - return -ENOMEM; + return NULL; - addrs = pagelist->addrs; - need_release = (unsigned long *)(addrs + num_pages); - pages = (struct page **)(addrs + num_pages + 1); + addrs = pagelist->addrs; + pages = (struct page **)(addrs + num_pages); + scatterlist = (struct scatterlist *)(pages + num_pages); + pagelistinfo = (struct vchiq_pagelist_info *) + (scatterlist + num_pages); + + pagelist->length = count; + pagelist->type = type; + pagelist->offset = offset; + + /* Populate the fields of the pagelistinfo structure */ + pagelistinfo->pagelist = pagelist; + pagelistinfo->pagelist_buffer_size = pagelist_size; + pagelistinfo->dma_addr = dma_addr; + pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ? + DMA_TO_DEVICE : DMA_FROM_DEVICE; + pagelistinfo->num_pages = num_pages; + pagelistinfo->pages_need_release = 0; + pagelistinfo->pages = pages; + pagelistinfo->scatterlist = scatterlist; + pagelistinfo->scatterlist_mapped = 0; if (is_vmalloc_addr(buf)) { - int dir = (type == PAGELIST_WRITE) ? - DMA_TO_DEVICE : DMA_FROM_DEVICE; unsigned long length = count; unsigned int off = offset; @@ -413,14 +453,13 @@ create_pagelist(char __user *buf, size_t count, unsigned short type, if (bytes > length) bytes = length; pages[actual_pages] = pg; - dmac_map_area(page_address(pg) + off, bytes, dir); length -= bytes; off = 0; } - *need_release = 0; /* do not try and release vmalloc pages */ + /* do not try and release vmalloc pages */ } else { down_read(&task->mm->mmap_sem); - actual_pages = get_user_pages(task, task->mm, + actual_pages = get_user_pages( (unsigned long)buf & ~(PAGE_SIZE - 1), num_pages, (type == PAGELIST_READ) ? FOLL_WRITE : 0, @@ -438,44 +477,59 @@ create_pagelist(char __user *buf, size_t count, unsigned short type, while (actual_pages > 0) { actual_pages--; - page_cache_release(pages[actual_pages]); + put_page(pages[actual_pages]); } - kfree(pagelist); - if (actual_pages == 0) - actual_pages = -ENOMEM; - return actual_pages; + cleaup_pagelistinfo(pagelistinfo); + return NULL; } - *need_release = 1; /* release user pages */ + /* release user pages */ + pagelistinfo->pages_need_release = 1; } - pagelist->length = count; - pagelist->type = type; - pagelist->offset = offset; - - /* Group the pages into runs of contiguous pages */ - - base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0])); - next_addr = base_addr + PAGE_SIZE; - addridx = 0; - run = 0; + /* + * Initialize the scatterlist so that the magic cookie + * is filled if debugging is enabled + */ + sg_init_table(scatterlist, num_pages); + /* Now set the pages for each scatterlist */ + for (i = 0; i < num_pages; i++) + sg_set_page(scatterlist + i, pages[i], PAGE_SIZE, 0); + + dma_buffers = dma_map_sg(g_dev, + scatterlist, + num_pages, + pagelistinfo->dma_dir); + + if (dma_buffers == 0) { + cleaup_pagelistinfo(pagelistinfo); + return NULL; + } - for (i = 1; i < num_pages; i++) { - addr = VCHIQ_ARM_ADDRESS(page_address(pages[i])); - if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) { - next_addr += PAGE_SIZE; - run++; + pagelistinfo->scatterlist_mapped = 1; + + /* Combine adjacent blocks for performance */ + k = 0; + for_each_sg(scatterlist, sg, dma_buffers, i) { + u32 len = sg_dma_len(sg); + u32 addr = sg_dma_address(sg); + + /* Note: addrs is the address + page_count - 1 + * The firmware expects the block to be page + * aligned and a multiple of the page size + */ + WARN_ON(len == 0); + WARN_ON(len & ~PAGE_MASK); + WARN_ON(addr & ~PAGE_MASK); + if (k > 0 && + ((addrs[k - 1] & PAGE_MASK) | + ((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT) + == addr) { + addrs[k - 1] += (len >> PAGE_SHIFT); } else { - addrs[addridx] = (unsigned long)base_addr + run; - addridx++; - base_addr = addr; - next_addr = addr + PAGE_SIZE; - run = 0; + addrs[k++] = addr | ((len >> PAGE_SHIFT) - 1); } } - addrs[addridx] = (unsigned long)base_addr + run; - addridx++; - /* Partial cache lines (fragments) require special measures */ if ((type == PAGELIST_READ) && ((pagelist->offset & (g_cache_line_size - 1)) || @@ -484,8 +538,8 @@ create_pagelist(char __user *buf, size_t count, unsigned short type, char *fragments; if (down_interruptible(&g_free_fragments_sema) != 0) { - kfree(pagelist); - return -EINTR; + cleaup_pagelistinfo(pagelistinfo); + return NULL; } WARN_ON(g_free_fragments == NULL); @@ -499,29 +553,28 @@ create_pagelist(char __user *buf, size_t count, unsigned short type, (fragments - g_fragments_base) / g_fragments_size; } - dmac_flush_range(pagelist, addrs + num_pages); - - *ppagelist = pagelist; - - return 0; + return pagelistinfo; } static void -free_pagelist(PAGELIST_T *pagelist, int actual) +free_pagelist(struct vchiq_pagelist_info *pagelistinfo, + int actual) { - unsigned long *need_release; - struct page **pages; - unsigned int num_pages, i; - - vchiq_log_trace(vchiq_arm_log_level, - "free_pagelist - %x, %d", (unsigned int)pagelist, actual); + unsigned int i; + PAGELIST_T *pagelist = pagelistinfo->pagelist; + struct page **pages = pagelistinfo->pages; + unsigned int num_pages = pagelistinfo->num_pages; - num_pages = - (pagelist->length + pagelist->offset + PAGE_SIZE - 1) / - PAGE_SIZE; + vchiq_log_trace(vchiq_arm_log_level, "free_pagelist - %pK, %d", + pagelistinfo->pagelist, actual); - need_release = (unsigned long *)(pagelist->addrs + num_pages); - pages = (struct page **)(pagelist->addrs + num_pages + 1); + /* + * NOTE: dma_unmap_sg must be called before the + * cpu can touch any of the data/pages. + */ + dma_unmap_sg(g_dev, pagelistinfo->scatterlist, + pagelistinfo->num_pages, pagelistinfo->dma_dir); + pagelistinfo->scatterlist_mapped = 0; /* Deal with any partial cache lines (fragments) */ if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) { @@ -559,27 +612,12 @@ free_pagelist(PAGELIST_T *pagelist, int actual) up(&g_free_fragments_sema); } - if (*need_release) { - unsigned int length = pagelist->length; - unsigned int offset = pagelist->offset; - - for (i = 0; i < num_pages; i++) { - struct page *pg = pages[i]; - - if (pagelist->type != PAGELIST_WRITE) { - unsigned int bytes = PAGE_SIZE - offset; - - if (bytes > length) - bytes = length; - dmac_unmap_area(page_address(pg) + offset, - bytes, DMA_FROM_DEVICE); - length -= bytes; - offset = 0; - set_page_dirty(pg); - } - page_cache_release(pg); - } + /* Need to mark all the pages dirty. */ + if (pagelist->type != PAGELIST_WRITE && + pagelistinfo->pages_need_release) { + for (i = 0; i < num_pages; i++) + set_page_dirty(pages[i]); } - kfree(pagelist); + cleaup_pagelistinfo(pagelistinfo); } |