From 2f718ffc16c43a435d12919c75dbfad518abd056 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Tue, 16 Oct 2007 01:24:59 -0700 Subject: mm: buffered write iterator Add an iterator data structure to operate over an iovec. Add usercopy operators needed by generic_file_buffered_write, and convert that function over. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 144 ++++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 27 deletions(-) (limited to 'mm/filemap.c') diff --git a/mm/filemap.c b/mm/filemap.c index 557fd887254f..67a03a0a9aee 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -30,7 +30,7 @@ #include #include #include -#include "filemap.h" +#include /* for BUG_ON(!in_atomic()) only */ #include "internal.h" /* @@ -1635,8 +1635,7 @@ int remove_suid(struct dentry *dentry) } EXPORT_SYMBOL(remove_suid); -size_t -__filemap_copy_from_user_iovec_inatomic(char *vaddr, +static size_t __iovec_copy_from_user_inatomic(char *vaddr, const struct iovec *iov, size_t base, size_t bytes) { size_t copied = 0, left = 0; @@ -1658,6 +1657,110 @@ __filemap_copy_from_user_iovec_inatomic(char *vaddr, return copied - left; } +/* + * Copy as much as we can into the page and return the number of bytes which + * were sucessfully copied. If a fault is encountered then return the number of + * bytes which were copied. + */ +size_t iov_iter_copy_from_user_atomic(struct page *page, + struct iov_iter *i, unsigned long offset, size_t bytes) +{ + char *kaddr; + size_t copied; + + BUG_ON(!in_atomic()); + kaddr = kmap_atomic(page, KM_USER0); + if (likely(i->nr_segs == 1)) { + int left; + char __user *buf = i->iov->iov_base + i->iov_offset; + left = __copy_from_user_inatomic_nocache(kaddr + offset, + buf, bytes); + copied = bytes - left; + } else { + copied = __iovec_copy_from_user_inatomic(kaddr + offset, + i->iov, i->iov_offset, bytes); + } + kunmap_atomic(kaddr, KM_USER0); + + return copied; +} + +/* + * This has the same sideeffects and return value as + * iov_iter_copy_from_user_atomic(). + * The difference is that it attempts to resolve faults. + * Page must not be locked. + */ +size_t iov_iter_copy_from_user(struct page *page, + struct iov_iter *i, unsigned long offset, size_t bytes) +{ + char *kaddr; + size_t copied; + + kaddr = kmap(page); + if (likely(i->nr_segs == 1)) { + int left; + char __user *buf = i->iov->iov_base + i->iov_offset; + left = __copy_from_user_nocache(kaddr + offset, buf, bytes); + copied = bytes - left; + } else { + copied = __iovec_copy_from_user_inatomic(kaddr + offset, + i->iov, i->iov_offset, bytes); + } + kunmap(page); + return copied; +} + +static void __iov_iter_advance_iov(struct iov_iter *i, size_t bytes) +{ + if (likely(i->nr_segs == 1)) { + i->iov_offset += bytes; + } else { + const struct iovec *iov = i->iov; + size_t base = i->iov_offset; + + while (bytes) { + int copy = min(bytes, iov->iov_len - base); + + bytes -= copy; + base += copy; + if (iov->iov_len == base) { + iov++; + base = 0; + } + } + i->iov = iov; + i->iov_offset = base; + } +} + +void iov_iter_advance(struct iov_iter *i, size_t bytes) +{ + BUG_ON(i->count < bytes); + + __iov_iter_advance_iov(i, bytes); + i->count -= bytes; +} + +int iov_iter_fault_in_readable(struct iov_iter *i) +{ + size_t seglen = min(i->iov->iov_len - i->iov_offset, i->count); + char __user *buf = i->iov->iov_base + i->iov_offset; + return fault_in_pages_readable(buf, seglen); +} + +/* + * Return the count of just the current iov_iter segment. + */ +size_t iov_iter_single_seg_count(struct iov_iter *i) +{ + const struct iovec *iov = i->iov; + if (i->nr_segs == 1) + return i->count; + else + return min(i->count, iov->iov_len - i->iov_offset); +} + /* * Performs necessary checks before doing a write * @@ -1816,30 +1919,22 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, const struct address_space_operations *a_ops = mapping->a_ops; struct inode *inode = mapping->host; long status = 0; - const struct iovec *cur_iov = iov; /* current iovec */ - size_t iov_offset = 0; /* offset in the current iovec */ - char __user *buf; + struct iov_iter i; - /* - * handle partial DIO write. Adjust cur_iov if needed. - */ - filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, written); + iov_iter_init(&i, iov, nr_segs, count, written); do { struct page *src_page; struct page *page; pgoff_t index; /* Pagecache index for current page */ unsigned long offset; /* Offset into pagecache page */ - unsigned long seglen; /* Bytes remaining in current iovec */ unsigned long bytes; /* Bytes to write to page */ size_t copied; /* Bytes copied from user */ - buf = cur_iov->iov_base + iov_offset; offset = (pos & (PAGE_CACHE_SIZE - 1)); index = pos >> PAGE_CACHE_SHIFT; - bytes = PAGE_CACHE_SIZE - offset; - if (bytes > count) - bytes = count; + bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, + iov_iter_count(&i)); /* * a non-NULL src_page indicates that we're doing the @@ -1847,10 +1942,6 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, */ src_page = NULL; - seglen = cur_iov->iov_len - iov_offset; - if (seglen > bytes) - seglen = bytes; - /* * Bring in the user page that we will copy from _first_. * Otherwise there's a nasty deadlock on copying from the @@ -1861,7 +1952,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, * to check that the address is actually valid, when atomic * usercopies are used, below. */ - if (unlikely(fault_in_pages_readable(buf, seglen))) { + if (unlikely(iov_iter_fault_in_readable(&i))) { status = -EFAULT; break; } @@ -1892,8 +1983,8 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, * same reason as we can't take a page fault with a * page locked (as explained below). */ - copied = filemap_copy_from_user(src_page, offset, - cur_iov, nr_segs, iov_offset, bytes); + copied = iov_iter_copy_from_user(src_page, &i, + offset, bytes); if (unlikely(copied == 0)) { status = -EFAULT; page_cache_release(page); @@ -1939,8 +2030,8 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, * really matter. */ pagefault_disable(); - copied = filemap_copy_from_user_atomic(page, offset, - cur_iov, nr_segs, iov_offset, bytes); + copied = iov_iter_copy_from_user_atomic(page, &i, + offset, bytes); pagefault_enable(); } else { void *src, *dst; @@ -1965,10 +2056,9 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, if (src_page) page_cache_release(src_page); + iov_iter_advance(&i, copied); written += copied; - count -= copied; pos += copied; - filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, copied); balance_dirty_pages_ratelimited(mapping); cond_resched(); @@ -1992,7 +2082,7 @@ fs_write_aop_error: continue; else break; - } while (count); + } while (iov_iter_count(&i)); *ppos = pos; /* -- cgit v1.2.3-59-g8ed1b