diff options
| author |   Pavel Begunkov <asml.silence@gmail.com> | 2021-06-26 21:40:45 +0100 |
|---|---|---|
| committer |   Jens Axboe <axboe@kernel.dk> | 2021-06-30 14:15:39 -0600 |
| commit | dd432ea5204eeb92a2abf246ce518e68679da772 (patch) | |
| tree | b589b2d49cc62b3fc2bed8a1d0ee84f3e7ed4739 /fs/io_uring.c | |
| parent | io_uring: refactor io_arm_poll_handler() (diff) | |
| download | linux-dev-dd432ea5204eeb92a2abf246ce518e68679da772.tar.xz linux-dev-dd432ea5204eeb92a2abf246ce518e68679da772.zip | |
io_uring: mainstream sqpoll task_work running
task_works are widely used, so place io_run_task_work() directly into
the main path of io_sq_thread(), and remove it from other places where
it's not needed anymore.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/24eb5e35d519c590d3dffbd694b4c61a5fe49029.1624739600.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 7 |
1 files changed, 3 insertions, 4 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index bfba7558ea86..80b7a6f04841 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6868,7 +6868,6 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd) cond_resched(); mutex_lock(&sqd->lock); } - io_run_task_work(); return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); } @@ -6897,7 +6896,6 @@ static int io_sq_thread(void *data) if (io_sqd_handle_event(sqd)) break; timeout = jiffies + sqd->sq_thread_idle; - continue; } cap_entries = !list_is_singular(&sqd->ctx_list); @@ -6907,9 +6905,10 @@ static int io_sq_thread(void *data) if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; } + if (io_run_task_work()) + sqt_spin = true; if (sqt_spin || !time_after(jiffies, timeout)) { - io_run_task_work(); cond_resched(); if (sqt_spin) timeout = jiffies + sqd->sq_thread_idle; @@ -6917,7 +6916,7 @@ static int io_sq_thread(void *data) } prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); - if (!io_sqd_events_pending(sqd) && !io_run_task_work()) { + if (!io_sqd_events_pending(sqd) && !current->task_works) { bool needs_sched = true; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { |