From 01a658e1e92745255bfcae1977adc2f563a159ac Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 Dec 2017 14:25:02 +0100 Subject: aio: don't print the page size at boot time The page size is in no way related to the aio code, and printing it in the (debug) dmesg at every boot serves no purpose. Signed-off-by: Christoph Hellwig Acked-by: Jeff Moyer Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/aio.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 88d7927ffbc6..add46b06be86 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -264,9 +264,6 @@ static int __init aio_setup(void) kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); - - pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page)); - return 0; } __initcall(aio_setup); -- cgit v1.2.3-59-g8ed1b From c213dc826c591cec9e1a458f17ae474af6c8e00c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 23 Nov 2017 19:53:40 +0100 Subject: aio: remove an outdated BUG_ON and comment in aio_complete These days we don't treat sync iocbs special in the aio completion code as they never use it. Remove the old comment and BUG_ON given that the current definition of is_sync_kiocb makes it impossible to hit. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- fs/aio.c | 9 --------- 1 file changed, 9 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index add46b06be86..7c1855afd723 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1107,15 +1107,6 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2) file_end_write(file); } - /* - * Special case handling for sync iocbs: - * - events go directly into the iocb for fast handling - * - the sync task with the iocb in its stack holds the single iocb - * ref, no other paths have a way to get another ref - * - the sync task helpfully left a reference to itself in the iocb - */ - BUG_ON(is_sync_kiocb(kiocb)); - if (iocb->ki_list.next) { unsigned long flags; -- cgit v1.2.3-59-g8ed1b From 75321b50a37a5ba612125a04bfc9e43e3da5b305 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 9 Apr 2018 14:57:56 +0200 Subject: aio: sanitize ki_list handling Instead of handcoded non-null checks always initialize ki_list to an empty list and use list_empty / list_empty_careful on it. While we're at it also error out on a double call to kiocb_set_cancel_fn instead of ignoring it. Signed-off-by: Christoph Hellwig Acked-by: Jeff Moyer Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/aio.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 7c1855afd723..18507743757a 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -553,13 +553,12 @@ void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) struct kioctx *ctx = req->ki_ctx; unsigned long flags; - spin_lock_irqsave(&ctx->ctx_lock, flags); - - if (!req->ki_list.next) - list_add(&req->ki_list, &ctx->active_reqs); + if (WARN_ON_ONCE(!list_empty(&req->ki_list))) + return; + spin_lock_irqsave(&ctx->ctx_lock, flags); + list_add_tail(&req->ki_list, &ctx->active_reqs); req->ki_cancel = cancel; - spin_unlock_irqrestore(&ctx->ctx_lock, flags); } EXPORT_SYMBOL(kiocb_set_cancel_fn); @@ -1039,7 +1038,7 @@ static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) goto out_put; percpu_ref_get(&ctx->reqs); - + INIT_LIST_HEAD(&req->ki_list); req->ki_ctx = ctx; return req; out_put: @@ -1107,7 +1106,7 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2) file_end_write(file); } - if (iocb->ki_list.next) { + if (!list_empty_careful(&iocb->ki_list)) { unsigned long flags; spin_lock_irqsave(&ctx->ctx_lock, flags); -- cgit v1.2.3-59-g8ed1b From 92ce4728563ad1fc42466f9bbecc1ac31d675894 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 6 Apr 2018 09:28:17 +0200 Subject: aio: remove the extra get_file/fput pair in io_submit_one If we release the lockdep write protection token before calling into ->write_iter and thus never access the file pointer after an -EIOCBQUEUED return from ->write_iter or ->read_iter we don't need this extra reference. Signed-off-by: Christoph Hellwig --- fs/aio.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 18507743757a..e5866e2fc331 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1515,16 +1515,19 @@ static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored, return ret; ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) { - req->ki_flags |= IOCB_WRITE; - file_start_write(file); - ret = aio_ret(req, call_write_iter(file, req, &iter)); /* - * We release freeze protection in aio_complete(). Fool lockdep - * by telling it the lock got released so that it doesn't - * complain about held lock when we return to userspace. + * Open-code file_start_write here to grab freeze protection, + * which will be released by another thread in aio_complete(). + * Fool lockdep by telling it the lock got released so that it + * doesn't complain about the held lock when we return to + * userspace. */ - if (S_ISREG(file_inode(file)->i_mode)) + if (S_ISREG(file_inode(file)->i_mode)) { + __sb_start_write(file_inode(file)->i_sb, SB_FREEZE_WRITE, true); __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE); + } + req->ki_flags |= IOCB_WRITE; + ret = aio_ret(req, call_write_iter(file, req, &iter)); } kfree(iovec); return ret; @@ -1599,7 +1602,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, req->ki_user_iocb = user_iocb; req->ki_user_data = iocb->aio_data; - get_file(file); switch (iocb->aio_lio_opcode) { case IOCB_CMD_PREAD: ret = aio_read(&req->common, iocb, false, compat); @@ -1618,8 +1620,13 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, ret = -EINVAL; break; } - fput(file); + /* + * If ret is -EIOCBQUEUED, ownership of the file reference acquired + * above passed to the file system, which at this point might have + * dropped the reference, so we must be careful to not reference it + * once we have called into the file system. + */ if (ret && ret != -EIOCBQUEUED) goto out_put_req; return 0; -- cgit v1.2.3-59-g8ed1b From 54843f875f7a9f802bbb0d9895c3266b4a0b2f37 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 2 May 2018 19:57:21 +0200 Subject: aio: refactor read/write iocb setup Don't reference the kiocb structure from the common aio code, and move any use of it into helper specific to the read/write path. This is in preparation for aio_poll support that wants to use the space for different fields. Signed-off-by: Christoph Hellwig Acked-by: Jeff Moyer Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/aio.c | 161 ++++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 92 insertions(+), 69 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index e5866e2fc331..d07c27b3fe88 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -170,7 +170,9 @@ struct kioctx { #define KIOCB_CANCELLED ((void *) (~0ULL)) struct aio_kiocb { - struct kiocb common; + union { + struct kiocb rw; + }; struct kioctx *ki_ctx; kiocb_cancel_fn *ki_cancel; @@ -549,7 +551,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) { - struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common); + struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw); struct kioctx *ctx = req->ki_ctx; unsigned long flags; @@ -581,7 +583,7 @@ static int kiocb_cancel(struct aio_kiocb *kiocb) cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED); } while (cancel != old); - return cancel(&kiocb->common); + return cancel(&kiocb->rw); } /* @@ -1046,15 +1048,6 @@ out_put: return NULL; } -static void kiocb_free(struct aio_kiocb *req) -{ - if (req->common.ki_filp) - fput(req->common.ki_filp); - if (req->ki_eventfd != NULL) - eventfd_ctx_put(req->ki_eventfd); - kmem_cache_free(kiocb_cachep, req); -} - static struct kioctx *lookup_ioctx(unsigned long ctx_id) { struct aio_ring __user *ring = (void __user *)ctx_id; @@ -1085,27 +1078,14 @@ out: /* aio_complete * Called when the io request on the given iocb is complete. */ -static void aio_complete(struct kiocb *kiocb, long res, long res2) +static void aio_complete(struct aio_kiocb *iocb, long res, long res2) { - struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, common); struct kioctx *ctx = iocb->ki_ctx; struct aio_ring *ring; struct io_event *ev_page, *event; unsigned tail, pos, head; unsigned long flags; - if (kiocb->ki_flags & IOCB_WRITE) { - struct file *file = kiocb->ki_filp; - - /* - * Tell lockdep we inherited freeze protection from submission - * thread. - */ - if (S_ISREG(file_inode(file)->i_mode)) - __sb_writers_acquired(file_inode(file)->i_sb, SB_FREEZE_WRITE); - file_end_write(file); - } - if (!list_empty_careful(&iocb->ki_list)) { unsigned long flags; @@ -1167,11 +1147,12 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2) * eventfd. The eventfd_signal() function is safe to be called * from IRQ context. */ - if (iocb->ki_eventfd != NULL) + if (iocb->ki_eventfd) { eventfd_signal(iocb->ki_eventfd, 1); + eventfd_ctx_put(iocb->ki_eventfd); + } - /* everything turned out well, dispose of the aiocb. */ - kiocb_free(iocb); + kmem_cache_free(kiocb_cachep, iocb); /* * We have to order our ring_info tail store above and test @@ -1434,6 +1415,45 @@ SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) return -EINVAL; } +static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) +{ + struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); + + if (kiocb->ki_flags & IOCB_WRITE) { + struct inode *inode = file_inode(kiocb->ki_filp); + + /* + * Tell lockdep we inherited freeze protection from submission + * thread. + */ + if (S_ISREG(inode->i_mode)) + __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); + file_end_write(kiocb->ki_filp); + } + + fput(kiocb->ki_filp); + aio_complete(iocb, res, res2); +} + +static int aio_prep_rw(struct kiocb *req, struct iocb *iocb) +{ + int ret; + + req->ki_filp = fget(iocb->aio_fildes); + if (unlikely(!req->ki_filp)) + return -EBADF; + req->ki_complete = aio_complete_rw; + req->ki_pos = iocb->aio_offset; + req->ki_flags = iocb_flags(req->ki_filp); + if (iocb->aio_flags & IOCB_FLAG_RESFD) + req->ki_flags |= IOCB_EVENTFD; + req->ki_hint = file_write_hint(req->ki_filp); + ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); + if (unlikely(ret)) + fput(req->ki_filp); + return ret; +} + static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec, bool vectored, bool compat, struct iov_iter *iter) { @@ -1453,7 +1473,7 @@ static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec, return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter); } -static inline ssize_t aio_ret(struct kiocb *req, ssize_t ret) +static inline ssize_t aio_rw_ret(struct kiocb *req, ssize_t ret) { switch (ret) { case -EIOCBQUEUED: @@ -1469,7 +1489,7 @@ static inline ssize_t aio_ret(struct kiocb *req, ssize_t ret) ret = -EINTR; /*FALLTHRU*/ default: - aio_complete(req, ret, 0); + aio_complete_rw(req, ret, 0); return 0; } } @@ -1477,59 +1497,79 @@ static inline ssize_t aio_ret(struct kiocb *req, ssize_t ret) static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored, bool compat) { - struct file *file = req->ki_filp; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iov_iter iter; + struct file *file; ssize_t ret; + ret = aio_prep_rw(req, iocb); + if (ret) + return ret; + file = req->ki_filp; + + ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_READ))) - return -EBADF; + goto out_fput; + ret = -EINVAL; if (unlikely(!file->f_op->read_iter)) - return -EINVAL; + goto out_fput; ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); if (ret) - return ret; + goto out_fput; ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) - ret = aio_ret(req, call_read_iter(file, req, &iter)); + ret = aio_rw_ret(req, call_read_iter(file, req, &iter)); kfree(iovec); +out_fput: + if (unlikely(ret && ret != -EIOCBQUEUED)) + fput(file); return ret; } static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored, bool compat) { - struct file *file = req->ki_filp; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iov_iter iter; + struct file *file; ssize_t ret; + ret = aio_prep_rw(req, iocb); + if (ret) + return ret; + file = req->ki_filp; + + ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_WRITE))) - return -EBADF; + goto out_fput; + ret = -EINVAL; if (unlikely(!file->f_op->write_iter)) - return -EINVAL; + goto out_fput; ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); if (ret) - return ret; + goto out_fput; ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) { /* * Open-code file_start_write here to grab freeze protection, - * which will be released by another thread in aio_complete(). - * Fool lockdep by telling it the lock got released so that it - * doesn't complain about the held lock when we return to - * userspace. + * which will be released by another thread in + * aio_complete_rw(). Fool lockdep by telling it the lock got + * released so that it doesn't complain about the held lock when + * we return to userspace. */ if (S_ISREG(file_inode(file)->i_mode)) { __sb_start_write(file_inode(file)->i_sb, SB_FREEZE_WRITE, true); __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE); } req->ki_flags |= IOCB_WRITE; - ret = aio_ret(req, call_write_iter(file, req, &iter)); + ret = aio_rw_ret(req, call_write_iter(file, req, &iter)); } kfree(iovec); +out_fput: + if (unlikely(ret && ret != -EIOCBQUEUED)) + fput(file); return ret; } @@ -1537,7 +1577,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, struct iocb *iocb, bool compat) { struct aio_kiocb *req; - struct file *file; ssize_t ret; /* enforce forwards compatibility on users */ @@ -1560,16 +1599,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, if (unlikely(!req)) return -EAGAIN; - req->common.ki_filp = file = fget(iocb->aio_fildes); - if (unlikely(!req->common.ki_filp)) { - ret = -EBADF; - goto out_put_req; - } - req->common.ki_pos = iocb->aio_offset; - req->common.ki_complete = aio_complete; - req->common.ki_flags = iocb_flags(req->common.ki_filp); - req->common.ki_hint = file_write_hint(file); - if (iocb->aio_flags & IOCB_FLAG_RESFD) { /* * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an @@ -1583,14 +1612,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, req->ki_eventfd = NULL; goto out_put_req; } - - req->common.ki_flags |= IOCB_EVENTFD; - } - - ret = kiocb_set_rw_flags(&req->common, iocb->aio_rw_flags); - if (unlikely(ret)) { - pr_debug("EINVAL: aio_rw_flags\n"); - goto out_put_req; } ret = put_user(KIOCB_KEY, &user_iocb->aio_key); @@ -1604,16 +1625,16 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, switch (iocb->aio_lio_opcode) { case IOCB_CMD_PREAD: - ret = aio_read(&req->common, iocb, false, compat); + ret = aio_read(&req->rw, iocb, false, compat); break; case IOCB_CMD_PWRITE: - ret = aio_write(&req->common, iocb, false, compat); + ret = aio_write(&req->rw, iocb, false, compat); break; case IOCB_CMD_PREADV: - ret = aio_read(&req->common, iocb, true, compat); + ret = aio_read(&req->rw, iocb, true, compat); break; case IOCB_CMD_PWRITEV: - ret = aio_write(&req->common, iocb, true, compat); + ret = aio_write(&req->rw, iocb, true, compat); break; default: pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); @@ -1633,7 +1654,9 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, out_put_req: put_reqs_available(ctx, 1); percpu_ref_put(&ctx->reqs); - kiocb_free(req); + if (req->ki_eventfd) + eventfd_ctx_put(req->ki_eventfd); + kmem_cache_free(kiocb_cachep, req); return ret; } -- cgit v1.2.3-59-g8ed1b From a3c0d439e4d92411c2b4b21a526a4de720d0806b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 27 Mar 2018 19:18:57 +0200 Subject: aio: implement IOCB_CMD_FSYNC and IOCB_CMD_FDSYNC Simple workqueue offload for now, but prepared for adding a real aio_fsync method if the need arises. Based on an earlier patch from Dave Chinner. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/aio.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index d07c27b3fe88..61d2e6942951 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -156,6 +156,12 @@ struct kioctx { unsigned id; }; +struct fsync_iocb { + struct work_struct work; + struct file *file; + bool datasync; +}; + /* * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either * cancelled or completed (this makes a certain amount of sense because @@ -172,6 +178,7 @@ struct kioctx { struct aio_kiocb { union { struct kiocb rw; + struct fsync_iocb fsync; }; struct kioctx *ki_ctx; @@ -1573,6 +1580,36 @@ out_fput: return ret; } +static void aio_fsync_work(struct work_struct *work) +{ + struct fsync_iocb *req = container_of(work, struct fsync_iocb, work); + int ret; + + ret = vfs_fsync(req->file, req->datasync); + fput(req->file); + aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); +} + +static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync) +{ + if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || + iocb->aio_rw_flags)) + return -EINVAL; + + req->file = fget(iocb->aio_fildes); + if (unlikely(!req->file)) + return -EBADF; + if (unlikely(!req->file->f_op->fsync)) { + fput(req->file); + return -EINVAL; + } + + req->datasync = datasync; + INIT_WORK(&req->work, aio_fsync_work); + schedule_work(&req->work); + return -EIOCBQUEUED; +} + static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, struct iocb *iocb, bool compat) { @@ -1636,6 +1673,12 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, case IOCB_CMD_PWRITEV: ret = aio_write(&req->rw, iocb, true, compat); break; + case IOCB_CMD_FSYNC: + ret = aio_fsync(&req->fsync, iocb, false); + break; + case IOCB_CMD_FDSYNC: + ret = aio_fsync(&req->fsync, iocb, true); + break; default: pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); ret = -EINVAL; -- cgit v1.2.3-59-g8ed1b From 7a074e96dee62586c935c80cecd931431bfdd0be Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 2 May 2018 19:51:00 +0200 Subject: aio: implement io_pgetevents This is the io_getevents equivalent of ppoll/pselect and allows to properly mix signals and aio completions (especially with IOCB_CMD_POLL) and atomically executes the following sequence: sigset_t origmask; pthread_sigmask(SIG_SETMASK, &sigmask, &origmask); ret = io_getevents(ctx, min_nr, nr, events, timeout); pthread_sigmask(SIG_SETMASK, &origmask, NULL); Note that unlike many other signal related calls we do not pass a sigmask size, as that would get us to 7 arguments, which aren't easily supported by the syscall infrastructure. It seems a lot less painful to just add a new syscall variant in the unlikely case we're going to increase the sigset size. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + fs/aio.c | 114 ++++++++++++++++++++++++++++++--- include/linux/compat.h | 7 ++ include/linux/syscalls.h | 6 ++ include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/aio_abi.h | 6 ++ kernel/sys_ni.c | 2 + 8 files changed, 130 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index d6b27dab1b30..14a2f996e543 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -396,3 +396,4 @@ 382 i386 pkey_free sys_pkey_free __ia32_sys_pkey_free 383 i386 statx sys_statx __ia32_sys_statx 384 i386 arch_prctl sys_arch_prctl __ia32_compat_sys_arch_prctl +385 i386 io_pgetevents sys_io_pgetevents __ia32_compat_sys_io_pgetevents diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 4dfe42666d0c..cd36232ab62f 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -341,6 +341,7 @@ 330 common pkey_alloc __x64_sys_pkey_alloc 331 common pkey_free __x64_sys_pkey_free 332 common statx __x64_sys_statx +333 common io_pgetevents __x64_sys_io_pgetevents # # x32-specific system call numbers start at 512 to avoid cache impact diff --git a/fs/aio.c b/fs/aio.c index 61d2e6942951..f3eae5d5771b 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1303,10 +1303,6 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr, wait_event_interruptible_hrtimeout(ctx->wait, aio_read_events(ctx, min_nr, nr, event, &ret), until); - - if (!ret && signal_pending(current)) - ret = -EINTR; - return ret; } @@ -1921,13 +1917,60 @@ SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, struct timespec __user *, timeout) { struct timespec64 ts; + int ret; + + if (timeout && unlikely(get_timespec64(&ts, timeout))) + return -EFAULT; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + if (!ret && signal_pending(current)) + ret = -EINTR; + return ret; +} - if (timeout) { - if (unlikely(get_timespec64(&ts, timeout))) +SYSCALL_DEFINE6(io_pgetevents, + aio_context_t, ctx_id, + long, min_nr, + long, nr, + struct io_event __user *, events, + struct timespec __user *, timeout, + const struct __aio_sigset __user *, usig) +{ + struct __aio_sigset ksig = { NULL, }; + sigset_t ksigmask, sigsaved; + struct timespec64 ts; + int ret; + + if (timeout && unlikely(get_timespec64(&ts, timeout))) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + if (ksig.sigmask) { + if (ksig.sigsetsize != sizeof(sigset_t)) + return -EINVAL; + if (copy_from_user(&ksigmask, ksig.sigmask, sizeof(ksigmask))) return -EFAULT; + sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP)); + sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved); + } + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + if (signal_pending(current)) { + if (ksig.sigmask) { + current->saved_sigmask = sigsaved; + set_restore_sigmask(); + } + + if (!ret) + ret = -ERESTARTNOHAND; + } else { + if (ksig.sigmask) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); } - return do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + return ret; } #ifdef CONFIG_COMPAT @@ -1938,13 +1981,64 @@ COMPAT_SYSCALL_DEFINE5(io_getevents, compat_aio_context_t, ctx_id, struct compat_timespec __user *, timeout) { struct timespec64 t; + int ret; + + if (timeout && compat_get_timespec64(&t, timeout)) + return -EFAULT; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + if (!ret && signal_pending(current)) + ret = -EINTR; + return ret; +} + - if (timeout) { - if (compat_get_timespec64(&t, timeout)) +struct __compat_aio_sigset { + compat_sigset_t __user *sigmask; + compat_size_t sigsetsize; +}; + +COMPAT_SYSCALL_DEFINE6(io_pgetevents, + compat_aio_context_t, ctx_id, + compat_long_t, min_nr, + compat_long_t, nr, + struct io_event __user *, events, + struct compat_timespec __user *, timeout, + const struct __compat_aio_sigset __user *, usig) +{ + struct __compat_aio_sigset ksig = { NULL, }; + sigset_t ksigmask, sigsaved; + struct timespec64 t; + int ret; + + if (timeout && compat_get_timespec64(&t, timeout)) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + if (ksig.sigmask) { + if (ksig.sigsetsize != sizeof(compat_sigset_t)) + return -EINVAL; + if (get_compat_sigset(&ksigmask, ksig.sigmask)) return -EFAULT; + sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP)); + sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved); + } + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + if (signal_pending(current)) { + if (ksig.sigmask) { + current->saved_sigmask = sigsaved; + set_restore_sigmask(); + } + if (!ret) + ret = -ERESTARTNOHAND; + } else { + if (ksig.sigmask) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); } - return do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + return ret; } #endif diff --git a/include/linux/compat.h b/include/linux/compat.h index 081281ad5772..ad192057b887 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -330,6 +330,7 @@ extern int put_compat_rusage(const struct rusage *, struct compat_rusage __user *); struct compat_siginfo; +struct __compat_aio_sigset; struct compat_dirent { u32 d_ino; @@ -553,6 +554,12 @@ asmlinkage long compat_sys_io_getevents(compat_aio_context_t ctx_id, compat_long_t nr, struct io_event __user *events, struct compat_timespec __user *timeout); +asmlinkage long compat_sys_io_pgetevents(compat_aio_context_t ctx_id, + compat_long_t min_nr, + compat_long_t nr, + struct io_event __user *events, + struct compat_timespec __user *timeout, + const struct __compat_aio_sigset __user *usig); /* fs/cookies.c */ asmlinkage long compat_sys_lookup_dcookie(u32, u32, char __user *, compat_size_t); diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 70fcda1a9049..811172fcb916 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -290,6 +290,12 @@ asmlinkage long sys_io_getevents(aio_context_t ctx_id, long nr, struct io_event __user *events, struct timespec __user *timeout); +asmlinkage long sys_io_pgetevents(aio_context_t ctx_id, + long min_nr, + long nr, + struct io_event __user *events, + struct timespec __user *timeout, + const struct __aio_sigset *sig); /* fs/xattr.c */ asmlinkage long sys_setxattr(const char __user *path, const char __user *name, diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 8bcb186c6f67..42990676a55e 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -732,9 +732,11 @@ __SYSCALL(__NR_pkey_alloc, sys_pkey_alloc) __SYSCALL(__NR_pkey_free, sys_pkey_free) #define __NR_statx 291 __SYSCALL(__NR_statx, sys_statx) +#define __NR_io_pgetevents 292 +__SC_COMP(__NR_io_pgetevents, sys_io_pgetevents, compat_sys_io_pgetevents) #undef __NR_syscalls -#define __NR_syscalls 292 +#define __NR_syscalls 293 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/aio_abi.h b/include/uapi/linux/aio_abi.h index a04adbc70ddf..2c0a3415beee 100644 --- a/include/uapi/linux/aio_abi.h +++ b/include/uapi/linux/aio_abi.h @@ -29,6 +29,7 @@ #include #include +#include #include typedef __kernel_ulong_t aio_context_t; @@ -108,5 +109,10 @@ struct iocb { #undef IFBIG #undef IFLITTLE +struct __aio_sigset { + sigset_t __user *sigmask; + size_t sigsetsize; +}; + #endif /* __LINUX__AIO_ABI_H */ diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 9791364925dc..183169c2a75b 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -43,7 +43,9 @@ COND_SYSCALL(io_submit); COND_SYSCALL_COMPAT(io_submit); COND_SYSCALL(io_cancel); COND_SYSCALL(io_getevents); +COND_SYSCALL(io_pgetevents); COND_SYSCALL_COMPAT(io_getevents); +COND_SYSCALL_COMPAT(io_pgetevents); /* fs/xattr.c */ -- cgit v1.2.3-59-g8ed1b From 8f546ae1fc5ce8396827d4868c7eee1f1cc6947a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 11 Jan 2018 12:23:05 +0100 Subject: fs: unexport poll_schedule_timeout No users outside of select.c. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/select.c | 3 +-- include/linux/poll.h | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/select.c b/fs/select.c index ba879c51288f..a87f396f0313 100644 --- a/fs/select.c +++ b/fs/select.c @@ -233,7 +233,7 @@ static void __pollwait(struct file *filp, wait_queue_head_t *wait_address, add_wait_queue(wait_address, &entry->wait); } -int poll_schedule_timeout(struct poll_wqueues *pwq, int state, +static int poll_schedule_timeout(struct poll_wqueues *pwq, int state, ktime_t *expires, unsigned long slack) { int rc = -EINTR; @@ -258,7 +258,6 @@ int poll_schedule_timeout(struct poll_wqueues *pwq, int state, return rc; } -EXPORT_SYMBOL(poll_schedule_timeout); /** * poll_select_set_timeout - helper function to setup the timeout value diff --git a/include/linux/poll.h b/include/linux/poll.h index f45ebd017eaa..a3576da63377 100644 --- a/include/linux/poll.h +++ b/include/linux/poll.h @@ -96,8 +96,6 @@ struct poll_wqueues { extern void poll_initwait(struct poll_wqueues *pwq); extern void poll_freewait(struct poll_wqueues *pwq); -extern int poll_schedule_timeout(struct poll_wqueues *pwq, int state, - ktime_t *expires, unsigned long slack); extern u64 select_estimate_accuracy(struct timespec64 *tv); #define MAX_INT64_SECONDS (((s64)(~((u64)0)>>1)/HZ)-1) -- cgit v1.2.3-59-g8ed1b From a0f8dcfc607d8d113090ff36eee2ee406463bb0e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 5 Mar 2018 07:15:25 -0800 Subject: fs: cleanup do_pollfd Use straightline code with failure handling gotos instead of a lot of nested conditionals. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- fs/select.c | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/select.c b/fs/select.c index a87f396f0313..25da26253485 100644 --- a/fs/select.c +++ b/fs/select.c @@ -812,34 +812,32 @@ static inline __poll_t do_pollfd(struct pollfd *pollfd, poll_table *pwait, bool *can_busy_poll, __poll_t busy_flag) { - __poll_t mask; - int fd; - - mask = 0; - fd = pollfd->fd; - if (fd >= 0) { - struct fd f = fdget(fd); - mask = EPOLLNVAL; - if (f.file) { - /* userland u16 ->events contains POLL... bitmap */ - __poll_t filter = demangle_poll(pollfd->events) | - EPOLLERR | EPOLLHUP; - mask = DEFAULT_POLLMASK; - if (f.file->f_op->poll) { - pwait->_key = filter; - pwait->_key |= busy_flag; - mask = f.file->f_op->poll(f.file, pwait); - if (mask & busy_flag) - *can_busy_poll = true; - } - /* Mask out unneeded events. */ - mask &= filter; - fdput(f); - } + int fd = pollfd->fd; + __poll_t mask = 0, filter; + struct fd f; + + if (fd < 0) + goto out; + mask = EPOLLNVAL; + f = fdget(fd); + if (!f.file) + goto out; + + /* userland u16 ->events contains POLL... bitmap */ + filter = demangle_poll(pollfd->events) | EPOLLERR | EPOLLHUP; + mask = DEFAULT_POLLMASK; + if (f.file->f_op->poll) { + pwait->_key = filter | busy_flag; + mask = f.file->f_op->poll(f.file, pwait); + if (mask & busy_flag) + *can_busy_poll = true; } + mask &= filter; /* Mask out unneeded events. */ + fdput(f); + +out: /* ... and so does ->revents */ pollfd->revents = mangle_poll(mask); - return mask; } -- cgit v1.2.3-59-g8ed1b From 9965ed174e7d38896e5d2582159d8ef31ecd4cb5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 5 Mar 2018 07:26:05 -0800 Subject: fs: add new vfs_poll and file_can_poll helpers These abstract out calls to the poll method in preparation for changes in how we poll. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- drivers/staging/comedi/drivers/serial2002.c | 4 ++-- drivers/vfio/virqfd.c | 2 +- drivers/vhost/vhost.c | 2 +- fs/eventpoll.c | 5 ++--- fs/select.c | 23 ++++++++--------------- include/linux/poll.h | 12 ++++++++++++ mm/memcontrol.c | 2 +- net/9p/trans_fd.c | 18 ++++-------------- virt/kvm/eventfd.c | 2 +- 9 files changed, 32 insertions(+), 38 deletions(-) (limited to 'fs') diff --git a/drivers/staging/comedi/drivers/serial2002.c b/drivers/staging/comedi/drivers/serial2002.c index b3f3b4a201af..5471b2212a62 100644 --- a/drivers/staging/comedi/drivers/serial2002.c +++ b/drivers/staging/comedi/drivers/serial2002.c @@ -113,7 +113,7 @@ static void serial2002_tty_read_poll_wait(struct file *f, int timeout) long elapsed; __poll_t mask; - mask = f->f_op->poll(f, &table.pt); + mask = vfs_poll(f, &table.pt); if (mask & (EPOLLRDNORM | EPOLLRDBAND | EPOLLIN | EPOLLHUP | EPOLLERR)) { break; @@ -136,7 +136,7 @@ static int serial2002_tty_read(struct file *f, int timeout) result = -1; if (!IS_ERR(f)) { - if (f->f_op->poll) { + if (file_can_poll(f)) { serial2002_tty_read_poll_wait(f, timeout); if (kernel_read(f, &ch, 1, &pos) == 1) diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c index 085700f1be10..2a1be859ee71 100644 --- a/drivers/vfio/virqfd.c +++ b/drivers/vfio/virqfd.c @@ -166,7 +166,7 @@ int vfio_virqfd_enable(void *opaque, init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup); init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc); - events = irqfd.file->f_op->poll(irqfd.file, &virqfd->pt); + events = vfs_poll(irqfd.file, &virqfd->pt); /* * Check if there was an event already pending on the eventfd diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index f3bd8e941224..f6022881f147 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -208,7 +208,7 @@ int vhost_poll_start(struct vhost_poll *poll, struct file *file) if (poll->wqh) return 0; - mask = file->f_op->poll(file, &poll->table); + mask = vfs_poll(file, &poll->table); if (mask) vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask)); if (mask & EPOLLERR) { diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 602ca4285b2e..67db22fe99c5 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -884,8 +884,7 @@ static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, pt->_key = epi->event.events; if (!is_file_epoll(epi->ffd.file)) - return epi->ffd.file->f_op->poll(epi->ffd.file, pt) & - epi->event.events; + return vfs_poll(epi->ffd.file, pt) & epi->event.events; ep = epi->ffd.file->private_data; poll_wait(epi->ffd.file, &ep->poll_wait, pt); @@ -2025,7 +2024,7 @@ SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd, /* The target file descriptor must support poll */ error = -EPERM; - if (!tf.file->f_op->poll) + if (!file_can_poll(tf.file)) goto error_tgt_fput; /* Check if EPOLLWAKEUP is allowed */ diff --git a/fs/select.c b/fs/select.c index 25da26253485..e30def680b2e 100644 --- a/fs/select.c +++ b/fs/select.c @@ -502,14 +502,10 @@ static int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time) continue; f = fdget(i); if (f.file) { - const struct file_operations *f_op; - f_op = f.file->f_op; - mask = DEFAULT_POLLMASK; - if (f_op->poll) { - wait_key_set(wait, in, out, - bit, busy_flag); - mask = (*f_op->poll)(f.file, wait); - } + wait_key_set(wait, in, out, bit, + busy_flag); + mask = vfs_poll(f.file, wait); + fdput(f); if ((mask & POLLIN_SET) && (in & bit)) { res_in |= bit; @@ -825,13 +821,10 @@ static inline __poll_t do_pollfd(struct pollfd *pollfd, poll_table *pwait, /* userland u16 ->events contains POLL... bitmap */ filter = demangle_poll(pollfd->events) | EPOLLERR | EPOLLHUP; - mask = DEFAULT_POLLMASK; - if (f.file->f_op->poll) { - pwait->_key = filter | busy_flag; - mask = f.file->f_op->poll(f.file, pwait); - if (mask & busy_flag) - *can_busy_poll = true; - } + pwait->_key = filter | busy_flag; + mask = vfs_poll(f.file, pwait); + if (mask & busy_flag) + *can_busy_poll = true; mask &= filter; /* Mask out unneeded events. */ fdput(f); diff --git a/include/linux/poll.h b/include/linux/poll.h index a3576da63377..7e0fdcf905d2 100644 --- a/include/linux/poll.h +++ b/include/linux/poll.h @@ -74,6 +74,18 @@ static inline void init_poll_funcptr(poll_table *pt, poll_queue_proc qproc) pt->_key = ~(__poll_t)0; /* all events enabled */ } +static inline bool file_can_poll(struct file *file) +{ + return file->f_op->poll; +} + +static inline __poll_t vfs_poll(struct file *file, struct poll_table_struct *pt) +{ + if (unlikely(!file->f_op->poll)) + return DEFAULT_POLLMASK; + return file->f_op->poll(file, pt); +} + struct poll_table_entry { struct file *filp; __poll_t key; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2bd3df3d101a..1695f38630f1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3849,7 +3849,7 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, if (ret) goto out_put_css; - efile.file->f_op->poll(efile.file, &event->pt); + vfs_poll(efile.file, &event->pt); spin_lock(&memcg->event_list_lock); list_add(&event->list, &memcg->event_list); diff --git a/net/9p/trans_fd.c b/net/9p/trans_fd.c index 0cfba919d167..3811775692d0 100644 --- a/net/9p/trans_fd.c +++ b/net/9p/trans_fd.c @@ -231,7 +231,7 @@ static void p9_conn_cancel(struct p9_conn *m, int err) static __poll_t p9_fd_poll(struct p9_client *client, struct poll_table_struct *pt, int *err) { - __poll_t ret, n; + __poll_t ret; struct p9_trans_fd *ts = NULL; if (client && client->status == Connected) @@ -243,19 +243,9 @@ p9_fd_poll(struct p9_client *client, struct poll_table_struct *pt, int *err) return EPOLLERR; } - if (!ts->rd->f_op->poll) - ret = DEFAULT_POLLMASK; - else - ret = ts->rd->f_op->poll(ts->rd, pt); - - if (ts->rd != ts->wr) { - if (!ts->wr->f_op->poll) - n = DEFAULT_POLLMASK; - else - n = ts->wr->f_op->poll(ts->wr, pt); - ret = (ret & ~EPOLLOUT) | (n & ~EPOLLIN); - } - + ret = vfs_poll(ts->rd, pt); + if (ts->rd != ts->wr) + ret = (ret & ~EPOLLOUT) | (vfs_poll(ts->wr, pt) & ~EPOLLIN); return ret; } diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 6e865e8b5b10..90d30fbe95ae 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -397,7 +397,7 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) * Check if there was an event already pending on the eventfd * before we registered, and trigger it as if we didn't miss it. */ - events = f.file->f_op->poll(f.file, &irqfd->pt); + events = vfs_poll(f.file, &irqfd->pt); if (events & EPOLLIN) schedule_work(&irqfd->inject); -- cgit v1.2.3-59-g8ed1b From 3deb642f0de4c14f37437dd247f9c77839f043f8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 15:29:24 +0100 Subject: fs: introduce new ->get_poll_head and ->poll_mask methods ->get_poll_head returns the waitqueue that the poll operation is going to sleep on. Note that this means we can only use a single waitqueue for the poll, unlike some current drivers that use two waitqueues for different events. But now that we have keyed wakeups and heavily use those for poll there aren't that many good reason left to keep the multiple waitqueues, and if there are any ->poll is still around, the driver just won't support aio poll. Signed-off-by: Christoph Hellwig Reviewed-by: Greg Kroah-Hartman Reviewed-by: Darrick J. Wong --- Documentation/filesystems/Locking | 7 ++++++- Documentation/filesystems/vfs.txt | 13 +++++++++++++ fs/select.c | 23 +++++++++++++++++++++++ include/linux/fs.h | 2 ++ include/linux/poll.h | 12 ++++++------ 5 files changed, 50 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking index 220bba28f72b..6d227f9d7bd9 100644 --- a/Documentation/filesystems/Locking +++ b/Documentation/filesystems/Locking @@ -440,6 +440,8 @@ prototypes: ssize_t (*write_iter) (struct kiocb *, struct iov_iter *); int (*iterate) (struct file *, struct dir_context *); __poll_t (*poll) (struct file *, struct poll_table_struct *); + struct wait_queue_head * (*get_poll_head)(struct file *, __poll_t); + __poll_t (*poll_mask) (struct file *, __poll_t); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); @@ -470,7 +472,7 @@ prototypes: }; locking rules: - All may block. + All except for ->poll_mask may block. ->llseek() locking has moved from llseek to the individual llseek implementations. If your fs is not using generic_file_llseek, you @@ -498,6 +500,9 @@ in sys_read() and friends. the lease within the individual filesystem to record the result of the operation +->poll_mask can be called with or without the waitqueue lock for the waitqueue +returned from ->get_poll_head. + --------------------------- dquot_operations ------------------------------- prototypes: int (*write_dquot) (struct dquot *); diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt index f608180ad59d..829a7b7857a4 100644 --- a/Documentation/filesystems/vfs.txt +++ b/Documentation/filesystems/vfs.txt @@ -857,6 +857,8 @@ struct file_operations { ssize_t (*write_iter) (struct kiocb *, struct iov_iter *); int (*iterate) (struct file *, struct dir_context *); __poll_t (*poll) (struct file *, struct poll_table_struct *); + struct wait_queue_head * (*get_poll_head)(struct file *, __poll_t); + __poll_t (*poll_mask) (struct file *, __poll_t); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); @@ -901,6 +903,17 @@ otherwise noted. activity on this file and (optionally) go to sleep until there is activity. Called by the select(2) and poll(2) system calls + get_poll_head: Returns the struct wait_queue_head that callers can + wait on. Callers need to check the returned events using ->poll_mask + once woken. Can return NULL to indicate polling is not supported, + or any error code using the ERR_PTR convention to indicate that a + grave error occured and ->poll_mask shall not be called. + + poll_mask: return the mask of EPOLL* values describing the file descriptor + state. Called either before going to sleep on the waitqueue returned by + get_poll_head, or after it has been woken. If ->get_poll_head and + ->poll_mask are implemented ->poll does not need to be implement. + unlocked_ioctl: called by the ioctl(2) system call. compat_ioctl: called by the ioctl(2) system call when 32 bit system calls diff --git a/fs/select.c b/fs/select.c index e30def680b2e..bc3cc0f98896 100644 --- a/fs/select.c +++ b/fs/select.c @@ -34,6 +34,29 @@ #include +__poll_t vfs_poll(struct file *file, struct poll_table_struct *pt) +{ + if (file->f_op->poll) { + return file->f_op->poll(file, pt); + } else if (file_has_poll_mask(file)) { + unsigned int events = poll_requested_events(pt); + struct wait_queue_head *head; + + if (pt && pt->_qproc) { + head = file->f_op->get_poll_head(file, events); + if (!head) + return DEFAULT_POLLMASK; + if (IS_ERR(head)) + return EPOLLERR; + pt->_qproc(file, head, pt); + } + + return file->f_op->poll_mask(file, events); + } else { + return DEFAULT_POLLMASK; + } +} +EXPORT_SYMBOL_GPL(vfs_poll); /* * Estimate expected accuracy in ns from a timeval. diff --git a/include/linux/fs.h b/include/linux/fs.h index 760d8da1b6c7..4b6045ebb2f2 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1711,6 +1711,8 @@ struct file_operations { int (*iterate) (struct file *, struct dir_context *); int (*iterate_shared) (struct file *, struct dir_context *); __poll_t (*poll) (struct file *, struct poll_table_struct *); + struct wait_queue_head * (*get_poll_head)(struct file *, __poll_t); + __poll_t (*poll_mask) (struct file *, __poll_t); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); diff --git a/include/linux/poll.h b/include/linux/poll.h index 7e0fdcf905d2..fdf86b4cbc71 100644 --- a/include/linux/poll.h +++ b/include/linux/poll.h @@ -74,18 +74,18 @@ static inline void init_poll_funcptr(poll_table *pt, poll_queue_proc qproc) pt->_key = ~(__poll_t)0; /* all events enabled */ } -static inline bool file_can_poll(struct file *file) +static inline bool file_has_poll_mask(struct file *file) { - return file->f_op->poll; + return file->f_op->get_poll_head && file->f_op->poll_mask; } -static inline __poll_t vfs_poll(struct file *file, struct poll_table_struct *pt) +static inline bool file_can_poll(struct file *file) { - if (unlikely(!file->f_op->poll)) - return DEFAULT_POLLMASK; - return file->f_op->poll(file, pt); + return file->f_op->poll || file_has_poll_mask(file); } +__poll_t vfs_poll(struct file *file, struct poll_table_struct *pt); + struct poll_table_entry { struct file *filp; __poll_t key; -- cgit v1.2.3-59-g8ed1b From f3a2752a43de18eb2eafa869d06f6e423ff0ce61 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 30 Mar 2018 11:19:25 +0200 Subject: aio: simplify KIOCB_KEY handling No need to pass the key field to lookup_iocb to compare it with KIOCB_KEY, as we can do that right after retrieving it from userspace. Also move the KIOCB_KEY definition to aio.c as it is an internal value not used by any other place in the kernel. Signed-off-by: Christoph Hellwig --- fs/aio.c | 14 +++++++------- include/linux/aio.h | 2 -- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 1c383bb44b2d..50a90e5581ed 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -46,6 +46,8 @@ #include "internal.h" +#define KIOCB_KEY 0 + #define AIO_RING_MAGIC 0xa10a10a1 #define AIO_RING_COMPAT_FEATURES 1 #define AIO_RING_INCOMPAT_FEATURES 0 @@ -1811,15 +1813,12 @@ COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, * Finds a given iocb for cancellation. */ static struct aio_kiocb * -lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb, u32 key) +lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb) { struct aio_kiocb *kiocb; assert_spin_locked(&ctx->ctx_lock); - if (key != KIOCB_KEY) - return NULL; - /* TODO: use a hash or array, this sucks. */ list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { if (kiocb->ki_user_iocb == iocb) @@ -1846,9 +1845,10 @@ SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, u32 key; int ret; - ret = get_user(key, &iocb->aio_key); - if (unlikely(ret)) + if (unlikely(get_user(key, &iocb->aio_key))) return -EFAULT; + if (unlikely(key != KIOCB_KEY)) + return -EINVAL; ctx = lookup_ioctx(ctx_id); if (unlikely(!ctx)) @@ -1856,7 +1856,7 @@ SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, spin_lock_irq(&ctx->ctx_lock); - kiocb = lookup_kiocb(ctx, iocb, key); + kiocb = lookup_kiocb(ctx, iocb); if (kiocb) ret = kiocb_cancel(kiocb); else diff --git a/include/linux/aio.h b/include/linux/aio.h index 9d8aabecfe2d..b83e68dd006f 100644 --- a/include/linux/aio.h +++ b/include/linux/aio.h @@ -8,8 +8,6 @@ struct kioctx; struct kiocb; struct mm_struct; -#define KIOCB_KEY 0 - typedef int (kiocb_cancel_fn)(struct kiocb *); /* prototypes */ -- cgit v1.2.3-59-g8ed1b From 888933f8fdf06ecbd5a476b312a37176ab29fb5e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 23 May 2018 14:11:02 +0200 Subject: aio: simplify cancellation With the current aio code there is no need for the magic KIOCB_CANCELLED value, as a cancelation just kicks the driver to queue the completion ASAP, with all actual completion handling done in another thread. Given that both the completion path and cancelation take the context lock there is no need for magic cmpxchg loops either. If we remove iocbs from the active list after calling ->ki_cancel (but with ctx_lock still held), we can also rely on the invariant thay anything found on the list has a ->ki_cancel callback and can be cancelled, further simplifing the code. Signed-off-by: Christoph Hellwig --- fs/aio.c | 48 ++++++------------------------------------------ 1 file changed, 6 insertions(+), 42 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 50a90e5581ed..0633cf3b325c 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -164,19 +164,6 @@ struct fsync_iocb { bool datasync; }; -/* - * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either - * cancelled or completed (this makes a certain amount of sense because - * successful cancellation - io_cancel() - does deliver the completion to - * userspace). - * - * And since most things don't implement kiocb cancellation and we'd really like - * kiocb completion to be lockless when possible, we use ki_cancel to - * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED - * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel(). - */ -#define KIOCB_CANCELLED ((void *) (~0ULL)) - struct aio_kiocb { union { struct kiocb rw; @@ -574,27 +561,6 @@ void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) } EXPORT_SYMBOL(kiocb_set_cancel_fn); -static int kiocb_cancel(struct aio_kiocb *kiocb) -{ - kiocb_cancel_fn *old, *cancel; - - /* - * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it - * actually has a cancel function, hence the cmpxchg() - */ - - cancel = READ_ONCE(kiocb->ki_cancel); - do { - if (!cancel || cancel == KIOCB_CANCELLED) - return -EINVAL; - - old = cancel; - cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED); - } while (cancel != old); - - return cancel(&kiocb->rw); -} - /* * free_ioctx() should be RCU delayed to synchronize against the RCU * protected lookup_ioctx() and also needs process context to call @@ -641,7 +607,7 @@ static void free_ioctx_users(struct percpu_ref *ref) while (!list_empty(&ctx->active_reqs)) { req = list_first_entry(&ctx->active_reqs, struct aio_kiocb, ki_list); - kiocb_cancel(req); + req->ki_cancel(&req->rw); list_del_init(&req->ki_list); } @@ -1842,8 +1808,8 @@ SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, { struct kioctx *ctx; struct aio_kiocb *kiocb; + int ret = -EINVAL; u32 key; - int ret; if (unlikely(get_user(key, &iocb->aio_key))) return -EFAULT; @@ -1855,13 +1821,11 @@ SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, return -EINVAL; spin_lock_irq(&ctx->ctx_lock); - kiocb = lookup_kiocb(ctx, iocb); - if (kiocb) - ret = kiocb_cancel(kiocb); - else - ret = -EINVAL; - + if (kiocb) { + ret = kiocb->ki_cancel(&kiocb->rw); + list_del_init(&kiocb->ki_list); + } spin_unlock_irq(&ctx->ctx_lock); if (!ret) { -- cgit v1.2.3-59-g8ed1b From 2c14fa838cbefc23cf1c73ca167ed85b274b2913 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 20 Mar 2018 16:34:36 +0100 Subject: aio: implement IOCB_CMD_POLL Simple one-shot poll through the io_submit() interface. To poll for a file descriptor the application should submit an iocb of type IOCB_CMD_POLL. It will poll the fd for the events specified in the the first 32 bits of the aio_buf field of the iocb. Unlike poll or epoll without EPOLLONESHOT this interface always works in one shot mode, that is once the iocb is completed, it will have to be resubmitted. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- fs/aio.c | 134 ++++++++++++++++++++++++++++++++++++++++++- include/uapi/linux/aio_abi.h | 6 +- 2 files changed, 135 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 0633cf3b325c..bd711cfa4a1f 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -5,6 +5,7 @@ * Implements an efficient asynchronous io interface. * * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. + * Copyright 2018 Christoph Hellwig. * * See ../COPYING for licensing terms. */ @@ -164,10 +165,22 @@ struct fsync_iocb { bool datasync; }; +struct poll_iocb { + struct file *file; + __poll_t events; + struct wait_queue_head *head; + + union { + struct wait_queue_entry wait; + struct work_struct work; + }; +}; + struct aio_kiocb { union { struct kiocb rw; struct fsync_iocb fsync; + struct poll_iocb poll; }; struct kioctx *ki_ctx; @@ -1558,7 +1571,6 @@ static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync) if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)) return -EINVAL; - req->file = fget(iocb->aio_fildes); if (unlikely(!req->file)) return -EBADF; @@ -1573,6 +1585,124 @@ static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync) return -EIOCBQUEUED; } +/* need to use list_del_init so we can check if item was present */ +static inline bool __aio_poll_remove(struct poll_iocb *req) +{ + if (list_empty(&req->wait.entry)) + return false; + list_del_init(&req->wait.entry); + return true; +} + +static inline void __aio_poll_complete(struct poll_iocb *req, __poll_t mask) +{ + struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); + struct file *file = req->file; + + aio_complete(iocb, mangle_poll(mask), 0); + fput(file); +} + +static void aio_poll_work(struct work_struct *work) +{ + struct poll_iocb *req = container_of(work, struct poll_iocb, work); + + __aio_poll_complete(req, req->events); +} + +static int aio_poll_cancel(struct kiocb *iocb) +{ + struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw); + struct poll_iocb *req = &aiocb->poll; + struct wait_queue_head *head = req->head; + bool found = false; + + spin_lock(&head->lock); + found = __aio_poll_remove(req); + spin_unlock(&head->lock); + + if (found) { + req->events = 0; + INIT_WORK(&req->work, aio_poll_work); + schedule_work(&req->work); + } + return 0; +} + +static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, + void *key) +{ + struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); + struct file *file = req->file; + __poll_t mask = key_to_poll(key); + + assert_spin_locked(&req->head->lock); + + /* for instances that support it check for an event match first: */ + if (mask && !(mask & req->events)) + return 0; + + mask = file->f_op->poll_mask(file, req->events); + if (!mask) + return 0; + + __aio_poll_remove(req); + + req->events = mask; + INIT_WORK(&req->work, aio_poll_work); + schedule_work(&req->work); + return 1; +} + +static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb) +{ + struct kioctx *ctx = aiocb->ki_ctx; + struct poll_iocb *req = &aiocb->poll; + __poll_t mask; + + /* reject any unknown events outside the normal event mask. */ + if ((u16)iocb->aio_buf != iocb->aio_buf) + return -EINVAL; + /* reject fields that are not defined for poll */ + if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) + return -EINVAL; + + req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; + req->file = fget(iocb->aio_fildes); + if (unlikely(!req->file)) + return -EBADF; + if (!file_has_poll_mask(req->file)) + goto out_fail; + + req->head = req->file->f_op->get_poll_head(req->file, req->events); + if (!req->head) + goto out_fail; + if (IS_ERR(req->head)) { + mask = EPOLLERR; + goto done; + } + + init_waitqueue_func_entry(&req->wait, aio_poll_wake); + aiocb->ki_cancel = aio_poll_cancel; + + spin_lock_irq(&ctx->ctx_lock); + spin_lock(&req->head->lock); + mask = req->file->f_op->poll_mask(req->file, req->events); + if (!mask) { + __add_wait_queue(req->head, &req->wait); + list_add_tail(&aiocb->ki_list, &ctx->active_reqs); + } + spin_unlock(&req->head->lock); + spin_unlock_irq(&ctx->ctx_lock); +done: + if (mask) + __aio_poll_complete(req, mask); + return -EIOCBQUEUED; +out_fail: + fput(req->file); + return -EINVAL; /* same as no support for IOCB_CMD_POLL */ +} + static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, struct iocb *iocb, bool compat) { @@ -1641,6 +1771,8 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, break; case IOCB_CMD_FDSYNC: ret = aio_fsync(&req->fsync, iocb, true); + case IOCB_CMD_POLL: + ret = aio_poll(req, iocb); break; default: pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); diff --git a/include/uapi/linux/aio_abi.h b/include/uapi/linux/aio_abi.h index 2c0a3415beee..ed0185945bb2 100644 --- a/include/uapi/linux/aio_abi.h +++ b/include/uapi/linux/aio_abi.h @@ -39,10 +39,8 @@ enum { IOCB_CMD_PWRITE = 1, IOCB_CMD_FSYNC = 2, IOCB_CMD_FDSYNC = 3, - /* These two are experimental. - * IOCB_CMD_PREADX = 4, - * IOCB_CMD_POLL = 5, - */ + /* 4 was the experimental IOCB_CMD_PREADX */ + IOCB_CMD_POLL = 5, IOCB_CMD_NOOP = 6, IOCB_CMD_PREADV = 7, IOCB_CMD_PWRITEV = 8, -- cgit v1.2.3-59-g8ed1b From 1962da0d2198a00a056a2915fbc6ec684c3bef0a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 20 May 2018 19:19:30 +0200 Subject: aio: try to complete poll iocbs without context switch If we can acquire ctx_lock without spinning we can just remove our iocb from the active_reqs list, and thus complete the iocbs from the wakeup context. Signed-off-by: Christoph Hellwig --- fs/aio.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index bd711cfa4a1f..8274d09d44a2 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1633,6 +1633,7 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, void *key) { struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); + struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); struct file *file = req->file; __poll_t mask = key_to_poll(key); @@ -1648,9 +1649,22 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, __aio_poll_remove(req); - req->events = mask; - INIT_WORK(&req->work, aio_poll_work); - schedule_work(&req->work); + /* + * Try completing without a context switch if we can acquire ctx_lock + * without spinning. Otherwise we need to defer to a workqueue to + * avoid a deadlock due to the lock order. + */ + if (spin_trylock(&iocb->ki_ctx->ctx_lock)) { + list_del_init(&iocb->ki_list); + spin_unlock(&iocb->ki_ctx->ctx_lock); + + __aio_poll_complete(req, mask); + } else { + req->events = mask; + INIT_WORK(&req->work, aio_poll_work); + schedule_work(&req->work); + } + return 1; } -- cgit v1.2.3-59-g8ed1b From dd67081b36a1fb41f632e2fe69992786396c9e77 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 31 Dec 2017 16:42:12 +0100 Subject: pipe: convert to ->poll_mask Signed-off-by: Christoph Hellwig --- fs/pipe.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/pipe.c b/fs/pipe.c index 39d6f431da83..bb0840e234f3 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -509,19 +509,22 @@ static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } -/* No kernel lock held - fine */ -static __poll_t -pipe_poll(struct file *filp, poll_table *wait) +static struct wait_queue_head * +pipe_get_poll_head(struct file *filp, __poll_t events) { - __poll_t mask; struct pipe_inode_info *pipe = filp->private_data; - int nrbufs; - poll_wait(filp, &pipe->wait, wait); + return &pipe->wait; +} + +/* No kernel lock held - fine */ +static __poll_t pipe_poll_mask(struct file *filp, __poll_t events) +{ + struct pipe_inode_info *pipe = filp->private_data; + int nrbufs = pipe->nrbufs; + __poll_t mask = 0; /* Reading only -- no need for acquiring the semaphore. */ - nrbufs = pipe->nrbufs; - mask = 0; if (filp->f_mode & FMODE_READ) { mask = (nrbufs > 0) ? EPOLLIN | EPOLLRDNORM : 0; if (!pipe->writers && filp->f_version != pipe->w_counter) @@ -1020,7 +1023,8 @@ const struct file_operations pipefifo_fops = { .llseek = no_llseek, .read_iter = pipe_read, .write_iter = pipe_write, - .poll = pipe_poll, + .get_poll_head = pipe_get_poll_head, + .poll_mask = pipe_poll_mask, .unlocked_ioctl = pipe_ioctl, .release = pipe_release, .fasync = pipe_fasync, -- cgit v1.2.3-59-g8ed1b From 9e42f195f5b5cae4cf8ec71432c0fdaf641d1b73 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 31 Dec 2017 16:42:48 +0100 Subject: eventfd: switch to ->poll_mask Signed-off-by: Christoph Hellwig --- fs/eventfd.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/eventfd.c b/fs/eventfd.c index 08d3bd602f73..61c9514da5e9 100644 --- a/fs/eventfd.c +++ b/fs/eventfd.c @@ -101,14 +101,20 @@ static int eventfd_release(struct inode *inode, struct file *file) return 0; } -static __poll_t eventfd_poll(struct file *file, poll_table *wait) +static struct wait_queue_head * +eventfd_get_poll_head(struct file *file, __poll_t events) +{ + struct eventfd_ctx *ctx = file->private_data; + + return &ctx->wqh; +} + +static __poll_t eventfd_poll_mask(struct file *file, __poll_t eventmask) { struct eventfd_ctx *ctx = file->private_data; __poll_t events = 0; u64 count; - poll_wait(file, &ctx->wqh, wait); - /* * All writes to ctx->count occur within ctx->wqh.lock. This read * can be done outside ctx->wqh.lock because we know that poll_wait @@ -305,7 +311,8 @@ static const struct file_operations eventfd_fops = { .show_fdinfo = eventfd_show_fdinfo, #endif .release = eventfd_release, - .poll = eventfd_poll, + .get_poll_head = eventfd_get_poll_head, + .poll_mask = eventfd_poll_mask, .read = eventfd_read, .write = eventfd_write, .llseek = noop_llseek, -- cgit v1.2.3-59-g8ed1b From 652fe8e87621d95a94edd25a08d5754abd878161 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 5 Mar 2018 07:23:51 -0800 Subject: timerfd: convert to ->poll_mask Signed-off-by: Christoph Hellwig --- fs/timerfd.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/timerfd.c b/fs/timerfd.c index cdad49da3ff7..d84a2bee4f82 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -226,21 +226,20 @@ static int timerfd_release(struct inode *inode, struct file *file) kfree_rcu(ctx, rcu); return 0; } - -static __poll_t timerfd_poll(struct file *file, poll_table *wait) + +static struct wait_queue_head *timerfd_get_poll_head(struct file *file, + __poll_t eventmask) { struct timerfd_ctx *ctx = file->private_data; - __poll_t events = 0; - unsigned long flags; - poll_wait(file, &ctx->wqh, wait); + return &ctx->wqh; +} - spin_lock_irqsave(&ctx->wqh.lock, flags); - if (ctx->ticks) - events |= EPOLLIN; - spin_unlock_irqrestore(&ctx->wqh.lock, flags); +static __poll_t timerfd_poll_mask(struct file *file, __poll_t eventmask) +{ + struct timerfd_ctx *ctx = file->private_data; - return events; + return ctx->ticks ? EPOLLIN : 0; } static ssize_t timerfd_read(struct file *file, char __user *buf, size_t count, @@ -364,7 +363,8 @@ static long timerfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg static const struct file_operations timerfd_fops = { .release = timerfd_release, - .poll = timerfd_poll, + .get_poll_head = timerfd_get_poll_head, + .poll_mask = timerfd_poll_mask, .read = timerfd_read, .llseek = noop_llseek, .show_fdinfo = timerfd_show, -- cgit v1.2.3-59-g8ed1b From ac060cbaa84999eb18e1045775d8fe7662d76476 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 28 May 2018 07:19:49 +0200 Subject: aio: add missing break for the IOCB_CMD_FDSYNC case Looks like this got lost in a merge. Signed-off-by: Christoph Hellwig Signed-off-by: Al Viro --- fs/aio.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 8274d09d44a2..e0b2f183fa1c 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1785,6 +1785,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, break; case IOCB_CMD_FDSYNC: ret = aio_fsync(&req->fsync, iocb, true); + break; case IOCB_CMD_POLL: ret = aio_poll(req, iocb); break; -- cgit v1.2.3-59-g8ed1b From 3c96c7f4caeb044da53a85092903f9192f4e2342 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 28 May 2018 13:37:43 -0400 Subject: aio: take list removal to (some) callers of aio_complete() We really want iocb out of io_cancel(2) reach before we start tearing it down. Signed-off-by: Al Viro --- fs/aio.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index e0b2f183fa1c..ae5977563b7e 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1073,14 +1073,6 @@ static void aio_complete(struct aio_kiocb *iocb, long res, long res2) unsigned tail, pos, head; unsigned long flags; - if (!list_empty_careful(&iocb->ki_list)) { - unsigned long flags; - - spin_lock_irqsave(&ctx->ctx_lock, flags); - list_del(&iocb->ki_list); - spin_unlock_irqrestore(&ctx->ctx_lock, flags); - } - /* * Add a completion event to the ring buffer. Must be done holding * ctx->completion_lock to prevent other code from messing with the tail @@ -1398,10 +1390,23 @@ SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) return -EINVAL; } +static void aio_remove_iocb(struct aio_kiocb *iocb) +{ + struct kioctx *ctx = iocb->ki_ctx; + unsigned long flags; + + spin_lock_irqsave(&ctx->ctx_lock, flags); + list_del(&iocb->ki_list); + spin_unlock_irqrestore(&ctx->ctx_lock, flags); +} + static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) { struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); + if (!list_empty_careful(&iocb->ki_list)) + aio_remove_iocb(iocb); + if (kiocb->ki_flags & IOCB_WRITE) { struct inode *inode = file_inode(kiocb->ki_filp); @@ -1594,20 +1599,19 @@ static inline bool __aio_poll_remove(struct poll_iocb *req) return true; } -static inline void __aio_poll_complete(struct poll_iocb *req, __poll_t mask) +static inline void __aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask) { - struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); - struct file *file = req->file; - + fput(iocb->poll.file); aio_complete(iocb, mangle_poll(mask), 0); - fput(file); } static void aio_poll_work(struct work_struct *work) { - struct poll_iocb *req = container_of(work, struct poll_iocb, work); + struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, poll.work); - __aio_poll_complete(req, req->events); + if (!list_empty_careful(&iocb->ki_list)) + aio_remove_iocb(iocb); + __aio_poll_complete(iocb, iocb->poll.events); } static int aio_poll_cancel(struct kiocb *iocb) @@ -1658,7 +1662,7 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, list_del_init(&iocb->ki_list); spin_unlock(&iocb->ki_ctx->ctx_lock); - __aio_poll_complete(req, mask); + __aio_poll_complete(iocb, mask); } else { req->events = mask; INIT_WORK(&req->work, aio_poll_work); @@ -1710,7 +1714,7 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb) spin_unlock_irq(&ctx->ctx_lock); done: if (mask) - __aio_poll_complete(req, mask); + __aio_poll_complete(aiocb, mask); return -EIOCBQUEUED; out_fail: fput(req->file); -- cgit v1.2.3-59-g8ed1b From 9061d14a8a97009085242b284b5b5f668af2bf97 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 26 May 2018 19:11:40 -0400 Subject: aio: all callers of aio_{read,write,fsync,poll} treat 0 and -EIOCBQUEUED the same way ... so just make them return 0 when caller does not need to destroy iocb Reviewed-by: Christoph Hellwig Signed-off-by: Al Viro --- fs/aio.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index ae5977563b7e..6098b6ba8671 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1461,11 +1461,11 @@ static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec, return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter); } -static inline ssize_t aio_rw_ret(struct kiocb *req, ssize_t ret) +static inline void aio_rw_done(struct kiocb *req, ssize_t ret) { switch (ret) { case -EIOCBQUEUED: - return ret; + break; case -ERESTARTSYS: case -ERESTARTNOINTR: case -ERESTARTNOHAND: @@ -1478,7 +1478,6 @@ static inline ssize_t aio_rw_ret(struct kiocb *req, ssize_t ret) /*FALLTHRU*/ default: aio_complete_rw(req, ret, 0); - return 0; } } @@ -1507,10 +1506,10 @@ static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored, goto out_fput; ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) - ret = aio_rw_ret(req, call_read_iter(file, req, &iter)); + aio_rw_done(req, call_read_iter(file, req, &iter)); kfree(iovec); out_fput: - if (unlikely(ret && ret != -EIOCBQUEUED)) + if (unlikely(ret)) fput(file); return ret; } @@ -1552,11 +1551,11 @@ static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored, __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE); } req->ki_flags |= IOCB_WRITE; - ret = aio_rw_ret(req, call_write_iter(file, req, &iter)); + aio_rw_done(req, call_write_iter(file, req, &iter)); } kfree(iovec); out_fput: - if (unlikely(ret && ret != -EIOCBQUEUED)) + if (unlikely(ret)) fput(file); return ret; } @@ -1587,7 +1586,7 @@ static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync) req->datasync = datasync; INIT_WORK(&req->work, aio_fsync_work); schedule_work(&req->work); - return -EIOCBQUEUED; + return 0; } /* need to use list_del_init so we can check if item was present */ @@ -1715,7 +1714,7 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb) done: if (mask) __aio_poll_complete(aiocb, mask); - return -EIOCBQUEUED; + return 0; out_fail: fput(req->file); return -EINVAL; /* same as no support for IOCB_CMD_POLL */ @@ -1800,12 +1799,11 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, } /* - * If ret is -EIOCBQUEUED, ownership of the file reference acquired - * above passed to the file system, which at this point might have - * dropped the reference, so we must be careful to not reference it - * once we have called into the file system. + * If ret is 0, we'd either done aio_complete() ourselves or have + * arranged for that to be done asynchronously. Anything non-zero + * means that we need to destroy req ourselves. */ - if (ret && ret != -EIOCBQUEUED) + if (ret) goto out_put_req; return 0; out_put_req: -- cgit v1.2.3-59-g8ed1b From d2988bd412fac87859f0b68f5ea328fb0f09c216 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 26 May 2018 19:13:10 -0400 Subject: aio_read_events_ring(): make a bit more readable The logics for 'avail' is * not past the tail of cyclic buffer * no more than asked * not past the end of buffer * not past the end of a page Unobfuscate the last part. Signed-off-by: Al Viro --- fs/aio.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 6098b6ba8671..06761d5aac1e 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1197,14 +1197,13 @@ static long aio_read_events_ring(struct kioctx *ctx, if (head == tail) break; - avail = min(avail, nr - ret); - avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - - ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE)); - pos = head + AIO_EVENTS_OFFSET; page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]; pos %= AIO_EVENTS_PER_PAGE; + avail = min(avail, nr - ret); + avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); + ev = kmap(page); copy_ret = copy_to_user(event + ret, ev + pos, sizeof(*ev) * avail); -- cgit v1.2.3-59-g8ed1b From 95af8496ac48263badf5b8dde5e06ef35aaace2b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 26 May 2018 19:43:16 -0400 Subject: aio: shift copyin of iocb into io_submit_one() Reviewed-by: Christoph Hellwig Signed-off-by: Al Viro --- fs/aio.c | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 06761d5aac1e..90b46560f0ce 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1720,22 +1720,26 @@ out_fail: } static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, - struct iocb *iocb, bool compat) + bool compat) { struct aio_kiocb *req; + struct iocb iocb; ssize_t ret; + if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) + return -EFAULT; + /* enforce forwards compatibility on users */ - if (unlikely(iocb->aio_reserved2)) { + if (unlikely(iocb.aio_reserved2)) { pr_debug("EINVAL: reserve field set\n"); return -EINVAL; } /* prevent overflows */ if (unlikely( - (iocb->aio_buf != (unsigned long)iocb->aio_buf) || - (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) || - ((ssize_t)iocb->aio_nbytes < 0) + (iocb.aio_buf != (unsigned long)iocb.aio_buf) || + (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) || + ((ssize_t)iocb.aio_nbytes < 0) )) { pr_debug("EINVAL: overflow check\n"); return -EINVAL; @@ -1745,14 +1749,14 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, if (unlikely(!req)) return -EAGAIN; - if (iocb->aio_flags & IOCB_FLAG_RESFD) { + if (iocb.aio_flags & IOCB_FLAG_RESFD) { /* * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an * instance of the file* now. The file descriptor must be * an eventfd() fd, and will be signaled for each completed * event using the eventfd_signal() function. */ - req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd); + req->ki_eventfd = eventfd_ctx_fdget((int) iocb.aio_resfd); if (IS_ERR(req->ki_eventfd)) { ret = PTR_ERR(req->ki_eventfd); req->ki_eventfd = NULL; @@ -1767,32 +1771,32 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, } req->ki_user_iocb = user_iocb; - req->ki_user_data = iocb->aio_data; + req->ki_user_data = iocb.aio_data; - switch (iocb->aio_lio_opcode) { + switch (iocb.aio_lio_opcode) { case IOCB_CMD_PREAD: - ret = aio_read(&req->rw, iocb, false, compat); + ret = aio_read(&req->rw, &iocb, false, compat); break; case IOCB_CMD_PWRITE: - ret = aio_write(&req->rw, iocb, false, compat); + ret = aio_write(&req->rw, &iocb, false, compat); break; case IOCB_CMD_PREADV: - ret = aio_read(&req->rw, iocb, true, compat); + ret = aio_read(&req->rw, &iocb, true, compat); break; case IOCB_CMD_PWRITEV: - ret = aio_write(&req->rw, iocb, true, compat); + ret = aio_write(&req->rw, &iocb, true, compat); break; case IOCB_CMD_FSYNC: - ret = aio_fsync(&req->fsync, iocb, false); + ret = aio_fsync(&req->fsync, &iocb, false); break; case IOCB_CMD_FDSYNC: - ret = aio_fsync(&req->fsync, iocb, true); + ret = aio_fsync(&req->fsync, &iocb, true); break; case IOCB_CMD_POLL: - ret = aio_poll(req, iocb); + ret = aio_poll(req, &iocb); break; default: - pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); + pr_debug("invalid aio operation %d\n", iocb.aio_lio_opcode); ret = -EINVAL; break; } @@ -1845,19 +1849,13 @@ static long do_io_submit(aio_context_t ctx_id, long nr, */ for (i=0; i Date: Sat, 26 May 2018 20:10:07 -0400 Subject: aio: fold do_io_submit() into callers get rid of insane "copy array of 32bit pointers into an array of native ones" glue. Reviewed-by: Christoph Hellwig Signed-off-by: Al Viro --- fs/aio.c | 99 +++++++++++++++++++++++++++++----------------------------------- 1 file changed, 45 insertions(+), 54 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index 90b46560f0ce..da859574c953 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1818,8 +1818,20 @@ out_put_req: return ret; } -static long do_io_submit(aio_context_t ctx_id, long nr, - struct iocb __user *__user *iocbpp, bool compat) +/* sys_io_submit: + * Queue the nr iocbs pointed to by iocbpp for processing. Returns + * the number of iocbs queued. May return -EINVAL if the aio_context + * specified by ctx_id is invalid, if nr is < 0, if the iocb at + * *iocbpp[0] is not properly initialized, if the operation specified + * is invalid for the file descriptor in the iocb. May fail with + * -EFAULT if any of the data structures point to invalid data. May + * fail with -EBADF if the file descriptor specified in the first + * iocb is invalid. May fail with -EAGAIN if insufficient resources + * are available to queue any iocbs. Will return 0 if nr is 0. Will + * fail with -ENOSYS if not implemented. + */ +SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, + struct iocb __user * __user *, iocbpp) { struct kioctx *ctx; long ret = 0; @@ -1832,9 +1844,6 @@ static long do_io_submit(aio_context_t ctx_id, long nr, if (unlikely(nr > LONG_MAX/sizeof(*iocbpp))) nr = LONG_MAX/sizeof(*iocbpp); - if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp))))) - return -EFAULT; - ctx = lookup_ioctx(ctx_id); if (unlikely(!ctx)) { pr_debug("EINVAL: invalid context id\n"); @@ -1842,20 +1851,15 @@ static long do_io_submit(aio_context_t ctx_id, long nr, } blk_start_plug(&plug); - - /* - * AKPM: should this return a partial result if some of the IOs were - * successfully submitted? - */ - for (i=0; i MAX_AIO_SUBMITS) nr = MAX_AIO_SUBMITS; - iocb64 = compat_alloc_user_space(nr * sizeof(*iocb64)); - ret = copy_iocb(nr, iocb, iocb64); - if (!ret) - ret = do_io_submit(ctx_id, nr, iocb64, 1); - return ret; + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) { + pr_debug("EINVAL: invalid context id\n"); + return -EINVAL; + } + + blk_start_plug(&plug); + for (i = 0; i < nr; i++) { + compat_uptr_t user_iocb; + + if (unlikely(get_user(user_iocb, iocbpp + i))) { + ret = -EFAULT; + break; + } + + ret = io_submit_one(ctx, compat_ptr(user_iocb), true); + if (ret) + break; + } + blk_finish_plug(&plug); + + percpu_ref_put(&ctx->users); + return i ? i : ret; } #endif -- cgit v1.2.3-59-g8ed1b From 1da92779e2e8f309d5aecbbed346e7f812b174e8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 26 May 2018 20:10:07 -0400 Subject: aio: sanitize the limit checking in io_submit(2) as it is, the logics in native io_submit(2) is "if asked for more than LONG_MAX/sizeof(pointer) iocbs to submit, don't bother with more than LONG_MAX/sizeof(pointer)" (i.e. 512M requests on 32bit and 1E requests on 64bit) while compat io_submit(2) goes with "stop after the first PAGE_SIZE/sizeof(pointer) iocbs", i.e. 1K or so. Which is * inconsistent * *way* too much in native case * possibly too little in compat one and * wrong anyway, since the natural point where we ought to stop bothering is ctx->nr_events Reviewed-by: Christoph Hellwig Signed-off-by: Al Viro --- fs/aio.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/aio.c b/fs/aio.c index da859574c953..b850e92ee0d5 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1841,15 +1841,15 @@ SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, if (unlikely(nr < 0)) return -EINVAL; - if (unlikely(nr > LONG_MAX/sizeof(*iocbpp))) - nr = LONG_MAX/sizeof(*iocbpp); - ctx = lookup_ioctx(ctx_id); if (unlikely(!ctx)) { pr_debug("EINVAL: invalid context id\n"); return -EINVAL; } + if (nr > ctx->nr_events) + nr = ctx->nr_events; + blk_start_plug(&plug); for (i = 0; i < nr; i++) { struct iocb __user *user_iocb; @@ -1870,8 +1870,6 @@ SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, } #ifdef CONFIG_COMPAT -#define MAX_AIO_SUBMITS (PAGE_SIZE/sizeof(struct iocb *)) - COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, int, nr, compat_uptr_t __user *, iocbpp) { @@ -1883,15 +1881,15 @@ COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, if (unlikely(nr < 0)) return -EINVAL; - if (nr > MAX_AIO_SUBMITS) - nr = MAX_AIO_SUBMITS; - ctx = lookup_ioctx(ctx_id); if (unlikely(!ctx)) { pr_debug("EINVAL: invalid context id\n"); return -EINVAL; } + if (nr > ctx->nr_events) + nr = ctx->nr_events; + blk_start_plug(&plug); for (i = 0; i < nr; i++) { compat_uptr_t user_iocb; -- cgit v1.2.3-59-g8ed1b