diff options
| author |   Pavel Begunkov <asml.silence@gmail.com> | 2020-11-24 17:58:12 +0000 |
|---|---|---|
| committer |   Jens Axboe <axboe@kernel.dk> | 2020-12-02 09:46:55 -0700 |
| commit | 6b6667aa4d1e0866f00b62d35a9be3875c7551f8 (patch) | |
| tree | c495f7838c57e1865a421173e4f40ac442e797d0 | |
| parent | block: stop using bdget_disk for partition 0 (diff) | |
| download | linux-dev-6b6667aa4d1e0866f00b62d35a9be3875c7551f8.tar.xz linux-dev-6b6667aa4d1e0866f00b62d35a9be3875c7551f8.zip | |
block: optimise for_each_bvec() advance
Because of how for_each_bvec() works it never advances across multiple
entries at a time, so bvec_iter_advance() is an overkill. Add
specialised bvec_iter_advance_single() that is faster. It also handles
zero-len bvecs, so can kill bvec_iter_skip_zero_bvec().
text data bss dec hex filename
before:
23977 805 0 24782 60ce lib/iov_iter.o
before, bvec_iter_advance() w/o WARN_ONCE()
22886 600 0 23486 5bbe ./lib/iov_iter.o
after:
21862 600 0 22462 57be lib/iov_iter.o
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | include/linux/bvec.h | 20 |
1 files changed, 15 insertions, 5 deletions
diff --git a/include/linux/bvec.h b/include/linux/bvec.h index 2efec10bf792..ff832e698efb 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -121,18 +121,28 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, return true; } -static inline void bvec_iter_skip_zero_bvec(struct bvec_iter *iter) +/* + * A simpler version of bvec_iter_advance(), @bytes should not span + * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len + */ +static inline void bvec_iter_advance_single(const struct bio_vec *bv, + struct bvec_iter *iter, unsigned int bytes) { - iter->bi_bvec_done = 0; - iter->bi_idx++; + unsigned int done = iter->bi_bvec_done + bytes; + + if (done == bv[iter->bi_idx].bv_len) { + done = 0; + iter->bi_idx++; + } + iter->bi_bvec_done = done; + iter->bi_size -= bytes; } #define for_each_bvec(bvl, bio_vec, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ - (bvl).bv_len ? (void)bvec_iter_advance((bio_vec), &(iter), \ - (bvl).bv_len) : bvec_iter_skip_zero_bvec(&(iter))) + bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) /* for iterating one bio from start to end */ #define BVEC_ITER_ALL_INIT (struct bvec_iter) \ |