From 1a49a97df657c63a4e8ffcd1ea9b6ed95581789b Mon Sep 17 00:00:00 2001 From: Daniel Xu Date: Thu, 12 Nov 2020 17:55:06 -0800 Subject: btrfs: tree-checker: add missing return after error in root_item There's a missing return statement after an error is found in the root_item, this can cause further problems when a crafted image triggers the error. Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=210181 Fixes: 259ee7754b67 ("btrfs: tree-checker: Add ROOT_ITEM check") CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Qu Wenruo Signed-off-by: Daniel Xu Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 8784b74f5232..6cefabd27209 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1068,6 +1068,7 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, "invalid root item size, have %u expect %zu or %u", btrfs_item_size_nr(leaf, slot), sizeof(ri), btrfs_legacy_root_item_size()); + return -EUCLEAN; } /* -- cgit v1.2.3-59-g8ed1b From 6d06b0ad94d3dd7e3503d8ad39c39c4634884611 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 16 Nov 2020 19:53:52 +0100 Subject: btrfs: tree-checker: add missing returns after data_ref alignment checks There are sectorsize alignment checks that are reported but then check_extent_data_ref continues. This was not intended, wrong alignment is not a minor problem and we should return with error. CC: stable@vger.kernel.org # 5.4+ Fixes: 0785a9aacf9d ("btrfs: tree-checker: Add EXTENT_DATA_REF check") Reviewed-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 6cefabd27209..ea2bb4cb5890 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1424,6 +1424,7 @@ static int check_extent_data_ref(struct extent_buffer *leaf, "invalid item size, have %u expect aligned to %zu for key type %u", btrfs_item_size_nr(leaf, slot), sizeof(*dref), key->type); + return -EUCLEAN; } if (!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize)) { generic_err(leaf, slot, @@ -1452,6 +1453,7 @@ static int check_extent_data_ref(struct extent_buffer *leaf, extent_err(leaf, slot, "invalid extent data backref offset, have %llu expect aligned to %u", offset, leaf->fs_info->sectorsize); + return -EUCLEAN; } } return 0; -- cgit v1.2.3-59-g8ed1b From c842268458d904c04f08259cc7ecee3870fa2746 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 15 Sep 2020 21:44:52 +0200 Subject: btrfs: add set/get accessors for root_item::drop_level The drop_level member is used directly unlike all the other int types in root_item. Add the definition and use it everywhere. The type is u8 so there's no conversion necessary and the helpers are properly inlined, this is for consistency. Reviewed-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 + fs/btrfs/disk-io.c | 2 +- fs/btrfs/extent-tree.c | 6 +++--- fs/btrfs/inode.c | 2 +- fs/btrfs/relocation.c | 10 +++++----- fs/btrfs/tree-checker.c | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index d9188ecf863d..b1b6d9a69b99 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2089,6 +2089,7 @@ BTRFS_SETGET_FUNCS(disk_root_level, struct btrfs_root_item, level, 8); BTRFS_SETGET_STACK_FUNCS(root_generation, struct btrfs_root_item, generation, 64); BTRFS_SETGET_STACK_FUNCS(root_bytenr, struct btrfs_root_item, bytenr, 64); +BTRFS_SETGET_STACK_FUNCS(root_drop_level, struct btrfs_root_item, drop_level, 8); BTRFS_SETGET_STACK_FUNCS(root_level, struct btrfs_root_item, level, 8); BTRFS_SETGET_STACK_FUNCS(root_dirid, struct btrfs_root_item, root_dirid, 64); BTRFS_SETGET_STACK_FUNCS(root_refs, struct btrfs_root_item, refs, 32); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index cf01f97a54cb..bd01e2cc165b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1179,7 +1179,7 @@ struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans, generate_random_guid(root->root_item.uuid); else export_guid(root->root_item.uuid, &guid_null); - root->root_item.drop_level = 0; + btrfs_set_root_drop_level(&root->root_item, 0); key.objectid = objectid; key.type = BTRFS_ROOT_ITEM_KEY; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 5fd60b13f4f8..bac1eb17e498 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -5496,7 +5496,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) memcpy(&wc->update_progress, &key, sizeof(wc->update_progress)); - level = root_item->drop_level; + level = btrfs_root_drop_level(root_item); BUG_ON(level == 0); path->lowest_level = level; ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); @@ -5529,7 +5529,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) } BUG_ON(wc->refs[level] == 0); - if (level == root_item->drop_level) + if (level == btrfs_root_drop_level(root_item)) break; btrfs_tree_unlock(path->nodes[level]); @@ -5574,7 +5574,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) } btrfs_cpu_key_to_disk(&root_item->drop_progress, &wc->drop_progress); - root_item->drop_level = wc->drop_level; + btrfs_set_root_drop_level(root_item, wc->drop_level); BUG_ON(wc->level == 0); if (btrfs_should_end_transaction(trans) || diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 9c0bf692a362..2c97169b6e26 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4105,7 +4105,7 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) memset(&dest->root_item.drop_progress, 0, sizeof(dest->root_item.drop_progress)); - dest->root_item.drop_level = 0; + btrfs_set_root_drop_level(&dest->root_item, 0); btrfs_set_root_refs(&dest->root_item, 0); if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 9ba92d86da0b..3d4618a01ef3 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -783,7 +783,7 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, btrfs_set_root_refs(root_item, 0); memset(&root_item->drop_progress, 0, sizeof(struct btrfs_disk_key)); - root_item->drop_level = 0; + btrfs_set_root_drop_level(root_item, 0); } btrfs_tree_unlock(eb); @@ -1575,7 +1575,7 @@ static void insert_dirty_subvol(struct btrfs_trans_handle *trans, reloc_root_item = &reloc_root->root_item; memset(&reloc_root_item->drop_progress, 0, sizeof(reloc_root_item->drop_progress)); - reloc_root_item->drop_level = 0; + btrfs_set_root_drop_level(reloc_root_item, 0); btrfs_set_root_refs(reloc_root_item, 0); btrfs_update_reloc_root(trans, root); @@ -1672,7 +1672,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, } else { btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); - level = root_item->drop_level; + level = btrfs_root_drop_level(root_item); BUG_ON(level == 0); path->lowest_level = level; ret = btrfs_search_slot(NULL, reloc_root, &key, path, 0, 0); @@ -1769,7 +1769,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, */ btrfs_node_key(path->nodes[level], &root_item->drop_progress, path->slots[level]); - root_item->drop_level = level; + btrfs_set_root_drop_level(root_item, level); btrfs_end_transaction_throttle(trans); trans = NULL; @@ -3694,7 +3694,7 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) memset(&root->root_item.drop_progress, 0, sizeof(root->root_item.drop_progress)); - root->root_item.drop_level = 0; + btrfs_set_root_drop_level(&root->root_item, 0); btrfs_set_root_refs(&root->root_item, 0); ret = btrfs_update_root(trans, fs_info->tree_root, &root->root_key, &root->root_item); diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index ea2bb4cb5890..cb9041da7d87 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1118,10 +1118,10 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, btrfs_root_level(&ri), BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } - if (ri.drop_level >= BTRFS_MAX_LEVEL) { + if (btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL) { generic_err(leaf, slot, "invalid root level, have %u expect [0, %u]", - ri.drop_level, BTRFS_MAX_LEVEL - 1); + btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } -- cgit v1.2.3-59-g8ed1b From 55fc29bed8ddb4c3848ecf8cf7133e34c946f223 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 30 Jun 2020 02:01:31 +0200 Subject: btrfs: use cached value of fs_info::csum_size everywhere btrfs_get_16 shows up in the system performance profiles (helper to read 16bit values from on-disk structures). This is partially because of the checksum size that's frequently read along with data reads/writes, other u16 uses are from item size or directory entries. Replace all calls to btrfs_super_csum_size by the cached value from fs_info. Reviewed-by: Johannes Thumshirn Reviewed-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 3 +-- fs/btrfs/check-integrity.c | 2 +- fs/btrfs/compression.c | 6 +++--- fs/btrfs/disk-io.c | 6 +++--- fs/btrfs/extent_io.c | 2 +- fs/btrfs/file-item.c | 14 +++++++------- fs/btrfs/inode.c | 6 +++--- fs/btrfs/ordered-data.c | 2 +- fs/btrfs/ordered-data.h | 2 +- fs/btrfs/scrub.c | 2 +- fs/btrfs/tree-checker.c | 2 +- 11 files changed, 23 insertions(+), 24 deletions(-) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 16c50a438f70..a31b7e32dd60 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -331,8 +331,7 @@ static inline void btrfs_print_data_csum_error(struct btrfs_inode *inode, u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) { struct btrfs_root *root = inode->root; - struct btrfs_super_block *sb = root->fs_info->super_copy; - const u16 csum_size = btrfs_super_csum_size(sb); + const u16 csum_size = root->fs_info->csum_size; /* Output minus objectid, which is more meaningful */ if (root->root_key.objectid >= BTRFS_LAST_FREE_OBJECTID) diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index 106874c959a0..288fd4bbe597 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c @@ -660,7 +660,7 @@ static int btrfsic_process_superblock(struct btrfsic_state *state, return -1; } - state->csum_size = btrfs_super_csum_size(selected_super); + state->csum_size = state->fs_info->csum_size; for (pass = 0; pass < 3; pass++) { int num_copies; diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 972fb68a85ac..00bbd859f31b 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -131,7 +131,7 @@ static int btrfs_decompress_bio(struct compressed_bio *cb); static inline int compressed_bio_size(struct btrfs_fs_info *fs_info, unsigned long disk_size) { - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; return sizeof(struct compressed_bio) + (DIV_ROUND_UP(disk_size, fs_info->sectorsize)) * csum_size; @@ -142,7 +142,7 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct bio *bio, { struct btrfs_fs_info *fs_info = inode->root->fs_info; SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; struct page *page; unsigned long i; char *kaddr; @@ -628,7 +628,7 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, struct extent_map *em; blk_status_t ret = BLK_STS_RESOURCE; int faili = 0; - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; u8 *sums; em_tree = &BTRFS_I(inode)->extent_tree; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 0e540ddd7d99..95ebe7c84dd7 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -321,7 +321,7 @@ static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); - if (memcmp(disk_sb->csum, result, btrfs_super_csum_size(disk_sb))) + if (memcmp(disk_sb->csum, result, fs_info->csum_size)) return 1; return 0; @@ -454,7 +454,7 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct page *page) u64 start = page_offset(page); u64 found_start; u8 result[BTRFS_CSUM_SIZE]; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; struct extent_buffer *eb; int ret; @@ -543,7 +543,7 @@ int btrfs_validate_metadata_buffer(struct btrfs_io_bio *io_bio, u64 phy_offset, eb = (struct extent_buffer *)page->private; fs_info = eb->fs_info; - csum_size = btrfs_super_csum_size(fs_info->super_copy); + csum_size = fs_info->csum_size; /* the pending IO might have been the only thing that kept this buffer * in memory. Make sure we have a ref for all this other checks diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 1eeba31b551d..d699f5d9b710 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2689,7 +2689,7 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, repair_bio->bi_private = failed_bio->bi_private; if (failed_io_bio->csum) { - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; repair_io_bio->csum = repair_io_bio->csum_inline; memcpy(repair_io_bio->csum, diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 5d79b1f79287..90e673847b47 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -181,7 +181,7 @@ btrfs_lookup_csum(struct btrfs_trans_handle *trans, struct btrfs_csum_item *item; struct extent_buffer *leaf; u64 csum_offset = 0; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; int csums_in_item; file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; @@ -270,7 +270,7 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u32 diff; int nblocks; int count = 0; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; if (!fs_info->csum_root || (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) return BLK_STS_OK; @@ -409,7 +409,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, int ret; size_t size; u64 csum_end; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && IS_ALIGNED(end + 1, fs_info->sectorsize)); @@ -540,7 +540,7 @@ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, int i; u64 offset; unsigned nofs_flag; - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; nofs_flag = memalloc_nofs_save(); sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), @@ -638,7 +638,7 @@ static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info, u64 bytenr, u64 len) { struct extent_buffer *leaf; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; u64 csum_end; u64 end_byte = bytenr + len; u32 blocksize_bits = fs_info->sectorsize_bits; @@ -692,7 +692,7 @@ int btrfs_del_csums(struct btrfs_trans_handle *trans, u64 csum_end; struct extent_buffer *leaf; int ret; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; u32 blocksize_bits = fs_info->sectorsize_bits; ASSERT(root == fs_info->csum_root || @@ -847,7 +847,7 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, int index = 0; int found_next; int ret; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; path = btrfs_alloc_path(); if (!path) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0b0c751dac85..358d900d0cf5 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2906,7 +2906,7 @@ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); char *kaddr; u32 len = fs_info->sectorsize; - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; u8 *csum_expected; u8 csum[BTRFS_CSUM_SIZE]; @@ -7825,7 +7825,7 @@ static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, csum_offset = file_offset - dip->logical_offset; csum_offset >>= fs_info->sectorsize_bits; - csum_offset *= btrfs_super_csum_size(fs_info->super_copy); + csum_offset *= fs_info->csum_size; btrfs_io_bio(bio)->csum = dip->csums + csum_offset; } map: @@ -7850,7 +7850,7 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, dip_size = sizeof(*dip); if (!write && csum) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; size_t nblocks; nblocks = dio_bio->bi_iter.bi_size >> fs_info->sectorsize_bits; diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 78c1f28f3bb6..6a63fb141f23 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -868,7 +868,7 @@ int btrfs_find_ordered_sum(struct btrfs_inode *inode, u64 offset, struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; unsigned long num_sectors; unsigned long i; - const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; int index = 0; ordered = btrfs_lookup_ordered_extent(inode, offset); diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index c3a2325e64a4..4662fd8ca546 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -137,7 +137,7 @@ static inline int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes) { int num_sectors = (int)DIV_ROUND_UP(bytes, fs_info->sectorsize); - int csum_size = btrfs_super_csum_size(fs_info->super_copy); + const u16 csum_size = fs_info->csum_size; return sizeof(struct btrfs_ordered_sum) + num_sectors * csum_size; } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index c307f25c4846..51222534618b 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -610,7 +610,7 @@ static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( atomic_set(&sctx->bios_in_flight, 0); atomic_set(&sctx->workers_pending, 0); atomic_set(&sctx->cancel_req, 0); - sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy); + sctx->csum_size = fs_info->csum_size; spin_lock_init(&sctx->list_lock); spin_lock_init(&sctx->stat_lock); diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index cb9041da7d87..1fe5961a9f60 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -336,7 +336,7 @@ static int check_csum_item(struct extent_buffer *leaf, struct btrfs_key *key, { struct btrfs_fs_info *fs_info = leaf->fs_info; u32 sectorsize = fs_info->sectorsize; - u32 csumsize = btrfs_super_csum_size(fs_info->super_copy); + const u16 csumsize = fs_info->csum_size; if (key->objectid != BTRFS_EXTENT_CSUM_OBJECTID) { generic_err(leaf, slot, -- cgit v1.2.3-59-g8ed1b From 223486c27b369a10ceb6180c40d7aa354e903446 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 2 Jul 2020 11:27:30 +0200 Subject: btrfs: switch cached fs_info::csum_size from u16 to u32 The fs_info value is 32bit, switch also the local u16 variables. This leads to a better assembly code generated due to movzwl. This simple change will shave some bytes on x86_64 and release config: text data bss dec hex filename 1090000 17980 14912 1122892 11224c pre/btrfs.ko 1089794 17980 14912 1122686 11217e post/btrfs.ko DELTA: -206 Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 2 +- fs/btrfs/compression.c | 7 +++---- fs/btrfs/disk-io.c | 2 +- fs/btrfs/extent_io.c | 2 +- fs/btrfs/file-item.c | 14 +++++++------- fs/btrfs/inode.c | 5 ++--- fs/btrfs/ordered-data.c | 2 +- fs/btrfs/ordered-data.h | 2 +- fs/btrfs/tree-checker.c | 2 +- 9 files changed, 18 insertions(+), 20 deletions(-) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index a31b7e32dd60..8494f62f8aa4 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -331,7 +331,7 @@ static inline void btrfs_print_data_csum_error(struct btrfs_inode *inode, u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) { struct btrfs_root *root = inode->root; - const u16 csum_size = root->fs_info->csum_size; + const u32 csum_size = root->fs_info->csum_size; /* Output minus objectid, which is more meaningful */ if (root->root_key.objectid >= BTRFS_LAST_FREE_OBJECTID) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 00bbd859f31b..b2bb4681975e 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -131,8 +131,7 @@ static int btrfs_decompress_bio(struct compressed_bio *cb); static inline int compressed_bio_size(struct btrfs_fs_info *fs_info, unsigned long disk_size) { - const u16 csum_size = fs_info->csum_size; - + const u32 csum_size = fs_info->csum_size; return sizeof(struct compressed_bio) + (DIV_ROUND_UP(disk_size, fs_info->sectorsize)) * csum_size; } @@ -142,7 +141,7 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct bio *bio, { struct btrfs_fs_info *fs_info = inode->root->fs_info; SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; struct page *page; unsigned long i; char *kaddr; @@ -628,7 +627,7 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, struct extent_map *em; blk_status_t ret = BLK_STS_RESOURCE; int faili = 0; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; u8 *sums; em_tree = &BTRFS_I(inode)->extent_tree; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 95ebe7c84dd7..02b36908327a 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -454,7 +454,7 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct page *page) u64 start = page_offset(page); u64 found_start; u8 result[BTRFS_CSUM_SIZE]; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; struct extent_buffer *eb; int ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index d699f5d9b710..f3515d3c1321 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2689,7 +2689,7 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, repair_bio->bi_private = failed_bio->bi_private; if (failed_io_bio->csum) { - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; repair_io_bio->csum = repair_io_bio->csum_inline; memcpy(repair_io_bio->csum, diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 90e673847b47..73ec6c5eadcb 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -181,7 +181,7 @@ btrfs_lookup_csum(struct btrfs_trans_handle *trans, struct btrfs_csum_item *item; struct extent_buffer *leaf; u64 csum_offset = 0; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; int csums_in_item; file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; @@ -270,7 +270,7 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u32 diff; int nblocks; int count = 0; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; if (!fs_info->csum_root || (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) return BLK_STS_OK; @@ -409,7 +409,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, int ret; size_t size; u64 csum_end; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && IS_ALIGNED(end + 1, fs_info->sectorsize)); @@ -540,7 +540,7 @@ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, int i; u64 offset; unsigned nofs_flag; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; nofs_flag = memalloc_nofs_save(); sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), @@ -638,7 +638,7 @@ static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info, u64 bytenr, u64 len) { struct extent_buffer *leaf; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; u64 csum_end; u64 end_byte = bytenr + len; u32 blocksize_bits = fs_info->sectorsize_bits; @@ -692,7 +692,7 @@ int btrfs_del_csums(struct btrfs_trans_handle *trans, u64 csum_end; struct extent_buffer *leaf; int ret; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; u32 blocksize_bits = fs_info->sectorsize_bits; ASSERT(root == fs_info->csum_root || @@ -847,7 +847,7 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, int index = 0; int found_next; int ret; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; path = btrfs_alloc_path(); if (!path) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 358d900d0cf5..d1915049d7e7 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2906,7 +2906,7 @@ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); char *kaddr; u32 len = fs_info->sectorsize; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; u8 *csum_expected; u8 csum[BTRFS_CSUM_SIZE]; @@ -7850,11 +7850,10 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, dip_size = sizeof(*dip); if (!write && csum) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - const u16 csum_size = fs_info->csum_size; size_t nblocks; nblocks = dio_bio->bi_iter.bi_size >> fs_info->sectorsize_bits; - dip_size += csum_size * nblocks; + dip_size += fs_info->csum_size * nblocks; } dip = kzalloc(dip_size, GFP_NOFS); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 6a63fb141f23..0d61f9fefc02 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -868,7 +868,7 @@ int btrfs_find_ordered_sum(struct btrfs_inode *inode, u64 offset, struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; unsigned long num_sectors; unsigned long i; - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; int index = 0; ordered = btrfs_lookup_ordered_extent(inode, offset); diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 4662fd8ca546..e48810104e4a 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -137,7 +137,7 @@ static inline int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes) { int num_sectors = (int)DIV_ROUND_UP(bytes, fs_info->sectorsize); - const u16 csum_size = fs_info->csum_size; + const u32 csum_size = fs_info->csum_size; return sizeof(struct btrfs_ordered_sum) + num_sectors * csum_size; } diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 1fe5961a9f60..f3f666b343ef 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -336,7 +336,7 @@ static int check_csum_item(struct extent_buffer *leaf, struct btrfs_key *key, { struct btrfs_fs_info *fs_info = leaf->fs_info; u32 sectorsize = fs_info->sectorsize; - const u16 csumsize = fs_info->csum_size; + const u32 csumsize = fs_info->csum_size; if (key->objectid != BTRFS_EXTENT_CSUM_OBJECTID) { generic_err(leaf, slot, -- cgit v1.2.3-59-g8ed1b From c7c01a4a2524b3f130c1821fbaf1677fe8394165 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 4 Nov 2020 16:12:45 +0100 Subject: btrfs: tree-checker: annotate all error branches as unlikely The tree checker is called many times as it verifies metadata at read/write time. The checks follow a simple pattern: if (error_condition) { report_error(); return -EUCLEAN; } All the error reporting functions are annotated as __cold that is supposed to hint the compiler to move the statement block out of the hot path. This does not seem to happen that often. As the error condition is expected to be false almost always, we can annotate it with 'unlikely' as this satisfies one of the few use cases for the annotation. The expected outcome is a stronger hint to compiler to reorder the checks test jump to exit test jump to exit ... which can be observed in asm of eg. check_dir_item, btrfs_check_chunk_valid, check_root_item or check_leaf. There's a measurable run time improvement reported by Josef, the testing workload went from 655 MiB/s to 677 MiB/s, which is about +3%. There should be no functional changes but some of the conditions have been rewritten to produce more readable result, some lines are longer than 80, for the sake of readability. Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 333 +++++++++++++++++++++++++----------------------- 1 file changed, 173 insertions(+), 160 deletions(-) (limited to 'fs/btrfs/tree-checker.c') diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index f3f666b343ef..028e733e42f3 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -100,7 +100,8 @@ static void file_extent_err(const struct extent_buffer *eb, int slot, */ #define CHECK_FE_ALIGNED(leaf, slot, fi, name, alignment) \ ({ \ - if (!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)), (alignment))) \ + if (unlikely(!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)), \ + (alignment)))) \ file_extent_err((leaf), (slot), \ "invalid %s for file extent, have %llu, should be aligned to %u", \ (#name), btrfs_file_extent_##name((leaf), (fi)), \ @@ -203,7 +204,7 @@ static int check_extent_data_item(struct extent_buffer *leaf, u32 item_size = btrfs_item_size_nr(leaf, slot); u64 extent_end; - if (!IS_ALIGNED(key->offset, sectorsize)) { + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { file_extent_err(leaf, slot, "unaligned file_offset for file extent, have %llu should be aligned to %u", key->offset, sectorsize); @@ -216,7 +217,7 @@ static int check_extent_data_item(struct extent_buffer *leaf, * But if objectids mismatch, it means we have a missing * INODE_ITEM. */ - if (!check_prev_ino(leaf, key, slot, prev_key)) + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) return -EUCLEAN; fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); @@ -225,14 +226,15 @@ static int check_extent_data_item(struct extent_buffer *leaf, * Make sure the item contains at least inline header, so the file * extent type is not some garbage. */ - if (item_size < BTRFS_FILE_EXTENT_INLINE_DATA_START) { + if (unlikely(item_size < BTRFS_FILE_EXTENT_INLINE_DATA_START)) { file_extent_err(leaf, slot, "invalid item size, have %u expect [%zu, %u)", item_size, BTRFS_FILE_EXTENT_INLINE_DATA_START, SZ_4K); return -EUCLEAN; } - if (btrfs_file_extent_type(leaf, fi) >= BTRFS_NR_FILE_EXTENT_TYPES) { + if (unlikely(btrfs_file_extent_type(leaf, fi) >= + BTRFS_NR_FILE_EXTENT_TYPES)) { file_extent_err(leaf, slot, "invalid type for file extent, have %u expect range [0, %u]", btrfs_file_extent_type(leaf, fi), @@ -244,14 +246,15 @@ static int check_extent_data_item(struct extent_buffer *leaf, * Support for new compression/encryption must introduce incompat flag, * and must be caught in open_ctree(). */ - if (btrfs_file_extent_compression(leaf, fi) >= BTRFS_NR_COMPRESS_TYPES) { + if (unlikely(btrfs_file_extent_compression(leaf, fi) >= + BTRFS_NR_COMPRESS_TYPES)) { file_extent_err(leaf, slot, "invalid compression for file extent, have %u expect range [0, %u]", btrfs_file_extent_compression(leaf, fi), BTRFS_NR_COMPRESS_TYPES - 1); return -EUCLEAN; } - if (btrfs_file_extent_encryption(leaf, fi)) { + if (unlikely(btrfs_file_extent_encryption(leaf, fi))) { file_extent_err(leaf, slot, "invalid encryption for file extent, have %u expect 0", btrfs_file_extent_encryption(leaf, fi)); @@ -259,7 +262,7 @@ static int check_extent_data_item(struct extent_buffer *leaf, } if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) { /* Inline extent must have 0 as key offset */ - if (key->offset) { + if (unlikely(key->offset)) { file_extent_err(leaf, slot, "invalid file_offset for inline file extent, have %llu expect 0", key->offset); @@ -272,8 +275,8 @@ static int check_extent_data_item(struct extent_buffer *leaf, return 0; /* Uncompressed inline extent size must match item size */ - if (item_size != BTRFS_FILE_EXTENT_INLINE_DATA_START + - btrfs_file_extent_ram_bytes(leaf, fi)) { + if (unlikely(item_size != BTRFS_FILE_EXTENT_INLINE_DATA_START + + btrfs_file_extent_ram_bytes(leaf, fi))) { file_extent_err(leaf, slot, "invalid ram_bytes for uncompressed inline extent, have %u expect %llu", item_size, BTRFS_FILE_EXTENT_INLINE_DATA_START + @@ -284,22 +287,22 @@ static int check_extent_data_item(struct extent_buffer *leaf, } /* Regular or preallocated extent has fixed item size */ - if (item_size != sizeof(*fi)) { + if (unlikely(item_size != sizeof(*fi))) { file_extent_err(leaf, slot, "invalid item size for reg/prealloc file extent, have %u expect %zu", item_size, sizeof(*fi)); return -EUCLEAN; } - if (CHECK_FE_ALIGNED(leaf, slot, fi, ram_bytes, sectorsize) || - CHECK_FE_ALIGNED(leaf, slot, fi, disk_bytenr, sectorsize) || - CHECK_FE_ALIGNED(leaf, slot, fi, disk_num_bytes, sectorsize) || - CHECK_FE_ALIGNED(leaf, slot, fi, offset, sectorsize) || - CHECK_FE_ALIGNED(leaf, slot, fi, num_bytes, sectorsize)) + if (unlikely(CHECK_FE_ALIGNED(leaf, slot, fi, ram_bytes, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, disk_bytenr, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, disk_num_bytes, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, offset, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, num_bytes, sectorsize))) return -EUCLEAN; /* Catch extent end overflow */ - if (check_add_overflow(btrfs_file_extent_num_bytes(leaf, fi), - key->offset, &extent_end)) { + if (unlikely(check_add_overflow(btrfs_file_extent_num_bytes(leaf, fi), + key->offset, &extent_end))) { file_extent_err(leaf, slot, "extent end overflow, have file offset %llu extent num bytes %llu", key->offset, @@ -320,7 +323,7 @@ static int check_extent_data_item(struct extent_buffer *leaf, prev_fi = btrfs_item_ptr(leaf, slot - 1, struct btrfs_file_extent_item); prev_end = file_extent_end(leaf, prev_key, prev_fi); - if (prev_end > key->offset) { + if (unlikely(prev_end > key->offset)) { file_extent_err(leaf, slot - 1, "file extent end range (%llu) goes beyond start offset (%llu) of the next file extent", prev_end, key->offset); @@ -338,19 +341,19 @@ static int check_csum_item(struct extent_buffer *leaf, struct btrfs_key *key, u32 sectorsize = fs_info->sectorsize; const u32 csumsize = fs_info->csum_size; - if (key->objectid != BTRFS_EXTENT_CSUM_OBJECTID) { + if (unlikely(key->objectid != BTRFS_EXTENT_CSUM_OBJECTID)) { generic_err(leaf, slot, "invalid key objectid for csum item, have %llu expect %llu", key->objectid, BTRFS_EXTENT_CSUM_OBJECTID); return -EUCLEAN; } - if (!IS_ALIGNED(key->offset, sectorsize)) { + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { generic_err(leaf, slot, "unaligned key offset for csum item, have %llu should be aligned to %u", key->offset, sectorsize); return -EUCLEAN; } - if (!IS_ALIGNED(btrfs_item_size_nr(leaf, slot), csumsize)) { + if (unlikely(!IS_ALIGNED(btrfs_item_size_nr(leaf, slot), csumsize))) { generic_err(leaf, slot, "unaligned item size for csum item, have %u should be aligned to %u", btrfs_item_size_nr(leaf, slot), csumsize); @@ -363,7 +366,7 @@ static int check_csum_item(struct extent_buffer *leaf, struct btrfs_key *key, prev_item_size = btrfs_item_size_nr(leaf, slot - 1); prev_csum_end = (prev_item_size / csumsize) * sectorsize; prev_csum_end += prev_key->offset; - if (prev_csum_end > key->offset) { + if (unlikely(prev_csum_end > key->offset)) { generic_err(leaf, slot - 1, "csum end range (%llu) goes beyond the start range (%llu) of the next csum item", prev_csum_end, key->offset); @@ -388,15 +391,16 @@ static int check_inode_key(struct extent_buffer *leaf, struct btrfs_key *key, /* For XATTR_ITEM, location key should be all 0 */ if (item_key.type == BTRFS_XATTR_ITEM_KEY) { - if (key->type != 0 || key->objectid != 0 || key->offset != 0) + if (unlikely(key->objectid != 0 || key->type != 0 || + key->offset != 0)) return -EUCLEAN; return 0; } - if ((key->objectid < BTRFS_FIRST_FREE_OBJECTID || - key->objectid > BTRFS_LAST_FREE_OBJECTID) && - key->objectid != BTRFS_ROOT_TREE_DIR_OBJECTID && - key->objectid != BTRFS_FREE_INO_OBJECTID) { + if (unlikely((key->objectid < BTRFS_FIRST_FREE_OBJECTID || + key->objectid > BTRFS_LAST_FREE_OBJECTID) && + key->objectid != BTRFS_ROOT_TREE_DIR_OBJECTID && + key->objectid != BTRFS_FREE_INO_OBJECTID)) { if (is_inode_item) { generic_err(leaf, slot, "invalid key objectid: has %llu expect %llu or [%llu, %llu] or %llu", @@ -414,7 +418,7 @@ static int check_inode_key(struct extent_buffer *leaf, struct btrfs_key *key, } return -EUCLEAN; } - if (key->offset != 0) { + if (unlikely(key->offset != 0)) { if (is_inode_item) inode_item_err(leaf, slot, "invalid key offset: has %llu expect 0", @@ -438,7 +442,7 @@ static int check_root_key(struct extent_buffer *leaf, struct btrfs_key *key, is_root_item = (item_key.type == BTRFS_ROOT_ITEM_KEY); /* No such tree id */ - if (key->objectid == 0) { + if (unlikely(key->objectid == 0)) { if (is_root_item) generic_err(leaf, slot, "invalid root id 0"); else @@ -448,7 +452,7 @@ static int check_root_key(struct extent_buffer *leaf, struct btrfs_key *key, } /* DIR_ITEM/INDEX/INODE_REF is not allowed to point to non-fs trees */ - if (!is_fstree(key->objectid) && !is_root_item) { + if (unlikely(!is_fstree(key->objectid) && !is_root_item)) { dir_item_err(leaf, slot, "invalid location key objectid, have %llu expect [%llu, %llu]", key->objectid, BTRFS_FIRST_FREE_OBJECTID, @@ -464,7 +468,8 @@ static int check_root_key(struct extent_buffer *leaf, struct btrfs_key *key, * So here we only check offset for reloc tree whose key->offset must * be a valid tree. */ - if (key->objectid == BTRFS_TREE_RELOC_OBJECTID && key->offset == 0) { + if (unlikely(key->objectid == BTRFS_TREE_RELOC_OBJECTID && + key->offset == 0)) { generic_err(leaf, slot, "invalid root id 0 for reloc tree"); return -EUCLEAN; } @@ -480,8 +485,9 @@ static int check_dir_item(struct extent_buffer *leaf, u32 item_size = btrfs_item_size_nr(leaf, slot); u32 cur = 0; - if (!check_prev_ino(leaf, key, slot, prev_key)) + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) return -EUCLEAN; + di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); while (cur < item_size) { struct btrfs_key location_key; @@ -494,7 +500,7 @@ static int check_dir_item(struct extent_buffer *leaf, int ret; /* header itself should not cross item boundary */ - if (cur + sizeof(*di) > item_size) { + if (unlikely(cur + sizeof(*di) > item_size)) { dir_item_err(leaf, slot, "dir item header crosses item boundary, have %zu boundary %u", cur + sizeof(*di), item_size); @@ -505,12 +511,12 @@ static int check_dir_item(struct extent_buffer *leaf, btrfs_dir_item_key_to_cpu(leaf, di, &location_key); if (location_key.type == BTRFS_ROOT_ITEM_KEY) { ret = check_root_key(leaf, &location_key, slot); - if (ret < 0) + if (unlikely(ret < 0)) return ret; } else if (location_key.type == BTRFS_INODE_ITEM_KEY || location_key.type == 0) { ret = check_inode_key(leaf, &location_key, slot); - if (ret < 0) + if (unlikely(ret < 0)) return ret; } else { dir_item_err(leaf, slot, @@ -522,22 +528,22 @@ static int check_dir_item(struct extent_buffer *leaf, /* dir type check */ dir_type = btrfs_dir_type(leaf, di); - if (dir_type >= BTRFS_FT_MAX) { + if (unlikely(dir_type >= BTRFS_FT_MAX)) { dir_item_err(leaf, slot, "invalid dir item type, have %u expect [0, %u)", dir_type, BTRFS_FT_MAX); return -EUCLEAN; } - if (key->type == BTRFS_XATTR_ITEM_KEY && - dir_type != BTRFS_FT_XATTR) { + if (unlikely(key->type == BTRFS_XATTR_ITEM_KEY && + dir_type != BTRFS_FT_XATTR)) { dir_item_err(leaf, slot, "invalid dir item type for XATTR key, have %u expect %u", dir_type, BTRFS_FT_XATTR); return -EUCLEAN; } - if (dir_type == BTRFS_FT_XATTR && - key->type != BTRFS_XATTR_ITEM_KEY) { + if (unlikely(dir_type == BTRFS_FT_XATTR && + key->type != BTRFS_XATTR_ITEM_KEY)) { dir_item_err(leaf, slot, "xattr dir type found for non-XATTR key"); return -EUCLEAN; @@ -550,13 +556,13 @@ static int check_dir_item(struct extent_buffer *leaf, /* Name/data length check */ name_len = btrfs_dir_name_len(leaf, di); data_len = btrfs_dir_data_len(leaf, di); - if (name_len > max_name_len) { + if (unlikely(name_len > max_name_len)) { dir_item_err(leaf, slot, "dir item name len too long, have %u max %u", name_len, max_name_len); return -EUCLEAN; } - if (name_len + data_len > BTRFS_MAX_XATTR_SIZE(fs_info)) { + if (unlikely(name_len + data_len > BTRFS_MAX_XATTR_SIZE(fs_info))) { dir_item_err(leaf, slot, "dir item name and data len too long, have %u max %u", name_len + data_len, @@ -564,7 +570,7 @@ static int check_dir_item(struct extent_buffer *leaf, return -EUCLEAN; } - if (data_len && dir_type != BTRFS_FT_XATTR) { + if (unlikely(data_len && dir_type != BTRFS_FT_XATTR)) { dir_item_err(leaf, slot, "dir item with invalid data len, have %u expect 0", data_len); @@ -574,7 +580,7 @@ static int check_dir_item(struct extent_buffer *leaf, total_size = sizeof(*di) + name_len + data_len; /* header and name/data should not cross item boundary */ - if (cur + total_size > item_size) { + if (unlikely(cur + total_size > item_size)) { dir_item_err(leaf, slot, "dir item data crosses item boundary, have %u boundary %u", cur + total_size, item_size); @@ -592,7 +598,7 @@ static int check_dir_item(struct extent_buffer *leaf, read_extent_buffer(leaf, namebuf, (unsigned long)(di + 1), name_len); name_hash = btrfs_name_hash(namebuf, name_len); - if (key->offset != name_hash) { + if (unlikely(key->offset != name_hash)) { dir_item_err(leaf, slot, "name hash mismatch with key, have 0x%016x expect 0x%016llx", name_hash, key->offset); @@ -641,13 +647,13 @@ static int check_block_group_item(struct extent_buffer *leaf, * Here we don't really care about alignment since extent allocator can * handle it. We care more about the size. */ - if (key->offset == 0) { + if (unlikely(key->offset == 0)) { block_group_err(leaf, slot, "invalid block group size 0"); return -EUCLEAN; } - if (item_size != sizeof(bgi)) { + if (unlikely(item_size != sizeof(bgi))) { block_group_err(leaf, slot, "invalid item size, have %u expect %zu", item_size, sizeof(bgi)); @@ -656,8 +662,8 @@ static int check_block_group_item(struct extent_buffer *leaf, read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), sizeof(bgi)); - if (btrfs_stack_block_group_chunk_objectid(&bgi) != - BTRFS_FIRST_CHUNK_TREE_OBJECTID) { + if (unlikely(btrfs_stack_block_group_chunk_objectid(&bgi) != + BTRFS_FIRST_CHUNK_TREE_OBJECTID)) { block_group_err(leaf, slot, "invalid block group chunk objectid, have %llu expect %llu", btrfs_stack_block_group_chunk_objectid(&bgi), @@ -665,7 +671,7 @@ static int check_block_group_item(struct extent_buffer *leaf, return -EUCLEAN; } - if (btrfs_stack_block_group_used(&bgi) > key->offset) { + if (unlikely(btrfs_stack_block_group_used(&bgi) > key->offset)) { block_group_err(leaf, slot, "invalid block group used, have %llu expect [0, %llu)", btrfs_stack_block_group_used(&bgi), key->offset); @@ -673,7 +679,7 @@ static int check_block_group_item(struct extent_buffer *leaf, } flags = btrfs_stack_block_group_flags(&bgi); - if (hweight64(flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) > 1) { + if (unlikely(hweight64(flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) > 1)) { block_group_err(leaf, slot, "invalid profile flags, have 0x%llx (%lu bits set) expect no more than 1 bit set", flags & BTRFS_BLOCK_GROUP_PROFILE_MASK, @@ -682,11 +688,11 @@ static int check_block_group_item(struct extent_buffer *leaf, } type = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; - if (type != BTRFS_BLOCK_GROUP_DATA && - type != BTRFS_BLOCK_GROUP_METADATA && - type != BTRFS_BLOCK_GROUP_SYSTEM && - type != (BTRFS_BLOCK_GROUP_METADATA | - BTRFS_BLOCK_GROUP_DATA)) { + if (unlikely(type != BTRFS_BLOCK_GROUP_DATA && + type != BTRFS_BLOCK_GROUP_METADATA && + type != BTRFS_BLOCK_GROUP_SYSTEM && + type != (BTRFS_BLOCK_GROUP_METADATA | + BTRFS_BLOCK_GROUP_DATA))) { block_group_err(leaf, slot, "invalid type, have 0x%llx (%lu bits set) expect either 0x%llx, 0x%llx, 0x%llx or 0x%llx", type, hweight64(type), @@ -773,49 +779,49 @@ int btrfs_check_chunk_valid(struct extent_buffer *leaf, ncopies = btrfs_raid_array[raid_index].ncopies; nparity = btrfs_raid_array[raid_index].nparity; - if (!num_stripes) { + if (unlikely(!num_stripes)) { chunk_err(leaf, chunk, logical, "invalid chunk num_stripes, have %u", num_stripes); return -EUCLEAN; } - if (num_stripes < ncopies) { + if (unlikely(num_stripes < ncopies)) { chunk_err(leaf, chunk, logical, "invalid chunk num_stripes < ncopies, have %u < %d", num_stripes, ncopies); return -EUCLEAN; } - if (nparity && num_stripes == nparity) { + if (unlikely(nparity && num_stripes == nparity)) { chunk_err(leaf, chunk, logical, "invalid chunk num_stripes == nparity, have %u == %d", num_stripes, nparity); return -EUCLEAN; } - if (!IS_ALIGNED(logical, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(logical, fs_info->sectorsize))) { chunk_err(leaf, chunk, logical, "invalid chunk logical, have %llu should aligned to %u", logical, fs_info->sectorsize); return -EUCLEAN; } - if (btrfs_chunk_sector_size(leaf, chunk) != fs_info->sectorsize) { + if (unlikely(btrfs_chunk_sector_size(leaf, chunk) != fs_info->sectorsize)) { chunk_err(leaf, chunk, logical, "invalid chunk sectorsize, have %u expect %u", btrfs_chunk_sector_size(leaf, chunk), fs_info->sectorsize); return -EUCLEAN; } - if (!length || !IS_ALIGNED(length, fs_info->sectorsize)) { + if (unlikely(!length || !IS_ALIGNED(length, fs_info->sectorsize))) { chunk_err(leaf, chunk, logical, "invalid chunk length, have %llu", length); return -EUCLEAN; } - if (!is_power_of_2(stripe_len) || stripe_len != BTRFS_STRIPE_LEN) { + if (unlikely(!is_power_of_2(stripe_len) || stripe_len != BTRFS_STRIPE_LEN)) { chunk_err(leaf, chunk, logical, "invalid chunk stripe length: %llu", stripe_len); return -EUCLEAN; } - if (~(BTRFS_BLOCK_GROUP_TYPE_MASK | BTRFS_BLOCK_GROUP_PROFILE_MASK) & - type) { + if (unlikely(type & ~(BTRFS_BLOCK_GROUP_TYPE_MASK | + BTRFS_BLOCK_GROUP_PROFILE_MASK))) { chunk_err(leaf, chunk, logical, "unrecognized chunk type: 0x%llx", ~(BTRFS_BLOCK_GROUP_TYPE_MASK | @@ -824,22 +830,23 @@ int btrfs_check_chunk_valid(struct extent_buffer *leaf, return -EUCLEAN; } - if (!has_single_bit_set(type & BTRFS_BLOCK_GROUP_PROFILE_MASK) && - (type & BTRFS_BLOCK_GROUP_PROFILE_MASK) != 0) { + if (unlikely(!has_single_bit_set(type & BTRFS_BLOCK_GROUP_PROFILE_MASK) && + (type & BTRFS_BLOCK_GROUP_PROFILE_MASK) != 0)) { chunk_err(leaf, chunk, logical, "invalid chunk profile flag: 0x%llx, expect 0 or 1 bit set", type & BTRFS_BLOCK_GROUP_PROFILE_MASK); return -EUCLEAN; } - if ((type & BTRFS_BLOCK_GROUP_TYPE_MASK) == 0) { + if (unlikely((type & BTRFS_BLOCK_GROUP_TYPE_MASK) == 0)) { chunk_err(leaf, chunk, logical, "missing chunk type flag, have 0x%llx one bit must be set in 0x%llx", type, BTRFS_BLOCK_GROUP_TYPE_MASK); return -EUCLEAN; } - if ((type & BTRFS_BLOCK_GROUP_SYSTEM) && - (type & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA))) { + if (unlikely((type & BTRFS_BLOCK_GROUP_SYSTEM) && + (type & (BTRFS_BLOCK_GROUP_METADATA | + BTRFS_BLOCK_GROUP_DATA)))) { chunk_err(leaf, chunk, logical, "system chunk with data or metadata type: 0x%llx", type); @@ -851,20 +858,21 @@ int btrfs_check_chunk_valid(struct extent_buffer *leaf, mixed = true; if (!mixed) { - if ((type & BTRFS_BLOCK_GROUP_METADATA) && - (type & BTRFS_BLOCK_GROUP_DATA)) { + if (unlikely((type & BTRFS_BLOCK_GROUP_METADATA) && + (type & BTRFS_BLOCK_GROUP_DATA))) { chunk_err(leaf, chunk, logical, "mixed chunk type in non-mixed mode: 0x%llx", type); return -EUCLEAN; } } - if ((type & BTRFS_BLOCK_GROUP_RAID10 && sub_stripes != 2) || - (type & BTRFS_BLOCK_GROUP_RAID1 && num_stripes != 2) || - (type & BTRFS_BLOCK_GROUP_RAID5 && num_stripes < 2) || - (type & BTRFS_BLOCK_GROUP_RAID6 && num_stripes < 3) || - (type & BTRFS_BLOCK_GROUP_DUP && num_stripes != 2) || - ((type & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 && num_stripes != 1)) { + if (unlikely((type & BTRFS_BLOCK_GROUP_RAID10 && sub_stripes != 2) || + (type & BTRFS_BLOCK_GROUP_RAID1 && num_stripes != 2) || + (type & BTRFS_BLOCK_GROUP_RAID5 && num_stripes < 2) || + (type & BTRFS_BLOCK_GROUP_RAID6 && num_stripes < 3) || + (type & BTRFS_BLOCK_GROUP_DUP && num_stripes != 2) || + ((type & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 && + num_stripes != 1))) { chunk_err(leaf, chunk, logical, "invalid num_stripes:sub_stripes %u:%u for profile %llu", num_stripes, sub_stripes, @@ -887,7 +895,7 @@ static int check_leaf_chunk_item(struct extent_buffer *leaf, { int num_stripes; - if (btrfs_item_size_nr(leaf, slot) < sizeof(struct btrfs_chunk)) { + if (unlikely(btrfs_item_size_nr(leaf, slot) < sizeof(struct btrfs_chunk))) { chunk_err(leaf, chunk, key->offset, "invalid chunk item size: have %u expect [%zu, %u)", btrfs_item_size_nr(leaf, slot), @@ -901,8 +909,8 @@ static int check_leaf_chunk_item(struct extent_buffer *leaf, if (num_stripes == 0) goto out; - if (btrfs_chunk_item_size(num_stripes) != - btrfs_item_size_nr(leaf, slot)) { + if (unlikely(btrfs_chunk_item_size(num_stripes) != + btrfs_item_size_nr(leaf, slot))) { chunk_err(leaf, chunk, key->offset, "invalid chunk item size: have %u expect %lu", btrfs_item_size_nr(leaf, slot), @@ -941,14 +949,14 @@ static int check_dev_item(struct extent_buffer *leaf, { struct btrfs_dev_item *ditem; - if (key->objectid != BTRFS_DEV_ITEMS_OBJECTID) { + if (unlikely(key->objectid != BTRFS_DEV_ITEMS_OBJECTID)) { dev_item_err(leaf, slot, "invalid objectid: has=%llu expect=%llu", key->objectid, BTRFS_DEV_ITEMS_OBJECTID); return -EUCLEAN; } ditem = btrfs_item_ptr(leaf, slot, struct btrfs_dev_item); - if (btrfs_device_id(leaf, ditem) != key->offset) { + if (unlikely(btrfs_device_id(leaf, ditem) != key->offset)) { dev_item_err(leaf, slot, "devid mismatch: key has=%llu item has=%llu", key->offset, btrfs_device_id(leaf, ditem)); @@ -960,8 +968,8 @@ static int check_dev_item(struct extent_buffer *leaf, * it can be 0 for device removal. Device size check can only be done * by dev extents check. */ - if (btrfs_device_bytes_used(leaf, ditem) > - btrfs_device_total_bytes(leaf, ditem)) { + if (unlikely(btrfs_device_bytes_used(leaf, ditem) > + btrfs_device_total_bytes(leaf, ditem))) { dev_item_err(leaf, slot, "invalid bytes used: have %llu expect [0, %llu]", btrfs_device_bytes_used(leaf, ditem), @@ -986,13 +994,13 @@ static int check_inode_item(struct extent_buffer *leaf, int ret; ret = check_inode_key(leaf, key, slot); - if (ret < 0) + if (unlikely(ret < 0)) return ret; iitem = btrfs_item_ptr(leaf, slot, struct btrfs_inode_item); /* Here we use super block generation + 1 to handle log tree */ - if (btrfs_inode_generation(leaf, iitem) > super_gen + 1) { + if (unlikely(btrfs_inode_generation(leaf, iitem) > super_gen + 1)) { inode_item_err(leaf, slot, "invalid inode generation: has %llu expect (0, %llu]", btrfs_inode_generation(leaf, iitem), @@ -1000,7 +1008,7 @@ static int check_inode_item(struct extent_buffer *leaf, return -EUCLEAN; } /* Note for ROOT_TREE_DIR_ITEM, mkfs could set its transid 0 */ - if (btrfs_inode_transid(leaf, iitem) > super_gen + 1) { + if (unlikely(btrfs_inode_transid(leaf, iitem) > super_gen + 1)) { inode_item_err(leaf, slot, "invalid inode transid: has %llu expect [0, %llu]", btrfs_inode_transid(leaf, iitem), super_gen + 1); @@ -1013,7 +1021,7 @@ static int check_inode_item(struct extent_buffer *leaf, * anything in the fs. So here we skip the check. */ mode = btrfs_inode_mode(leaf, iitem); - if (mode & ~valid_mask) { + if (unlikely(mode & ~valid_mask)) { inode_item_err(leaf, slot, "unknown mode bit detected: 0x%x", mode & ~valid_mask); @@ -1026,20 +1034,20 @@ static int check_inode_item(struct extent_buffer *leaf, * FIFO/CHR/DIR/REG. Only needs to check BLK, LNK and SOCKS */ if (!has_single_bit_set(mode & S_IFMT)) { - if (!S_ISLNK(mode) && !S_ISBLK(mode) && !S_ISSOCK(mode)) { + if (unlikely(!S_ISLNK(mode) && !S_ISBLK(mode) && !S_ISSOCK(mode))) { inode_item_err(leaf, slot, "invalid mode: has 0%o expect valid S_IF* bit(s)", mode & S_IFMT); return -EUCLEAN; } } - if (S_ISDIR(mode) && btrfs_inode_nlink(leaf, iitem) > 1) { + if (unlikely(S_ISDIR(mode) && btrfs_inode_nlink(leaf, iitem) > 1)) { inode_item_err(leaf, slot, "invalid nlink: has %u expect no more than 1 for dir", btrfs_inode_nlink(leaf, iitem)); return -EUCLEAN; } - if (btrfs_inode_flags(leaf, iitem) & ~BTRFS_INODE_FLAG_MASK) { + if (unlikely(btrfs_inode_flags(leaf, iitem) & ~BTRFS_INODE_FLAG_MASK)) { inode_item_err(leaf, slot, "unknown flags detected: 0x%llx", btrfs_inode_flags(leaf, iitem) & @@ -1059,11 +1067,12 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, int ret; ret = check_root_key(leaf, key, slot); - if (ret < 0) + if (unlikely(ret < 0)) return ret; - if (btrfs_item_size_nr(leaf, slot) != sizeof(ri) && - btrfs_item_size_nr(leaf, slot) != btrfs_legacy_root_item_size()) { + if (unlikely(btrfs_item_size_nr(leaf, slot) != sizeof(ri) && + btrfs_item_size_nr(leaf, slot) != + btrfs_legacy_root_item_size())) { generic_err(leaf, slot, "invalid root item size, have %u expect %zu or %u", btrfs_item_size_nr(leaf, slot), sizeof(ri), @@ -1080,24 +1089,24 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, btrfs_item_size_nr(leaf, slot)); /* Generation related */ - if (btrfs_root_generation(&ri) > - btrfs_super_generation(fs_info->super_copy) + 1) { + if (unlikely(btrfs_root_generation(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { generic_err(leaf, slot, "invalid root generation, have %llu expect (0, %llu]", btrfs_root_generation(&ri), btrfs_super_generation(fs_info->super_copy) + 1); return -EUCLEAN; } - if (btrfs_root_generation_v2(&ri) > - btrfs_super_generation(fs_info->super_copy) + 1) { + if (unlikely(btrfs_root_generation_v2(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { generic_err(leaf, slot, "invalid root v2 generation, have %llu expect (0, %llu]", btrfs_root_generation_v2(&ri), btrfs_super_generation(fs_info->super_copy) + 1); return -EUCLEAN; } - if (btrfs_root_last_snapshot(&ri) > - btrfs_super_generation(fs_info->super_copy) + 1) { + if (unlikely(btrfs_root_last_snapshot(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { generic_err(leaf, slot, "invalid root last_snapshot, have %llu expect (0, %llu]", btrfs_root_last_snapshot(&ri), @@ -1106,19 +1115,19 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, } /* Alignment and level check */ - if (!IS_ALIGNED(btrfs_root_bytenr(&ri), fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(btrfs_root_bytenr(&ri), fs_info->sectorsize))) { generic_err(leaf, slot, "invalid root bytenr, have %llu expect to be aligned to %u", btrfs_root_bytenr(&ri), fs_info->sectorsize); return -EUCLEAN; } - if (btrfs_root_level(&ri) >= BTRFS_MAX_LEVEL) { + if (unlikely(btrfs_root_level(&ri) >= BTRFS_MAX_LEVEL)) { generic_err(leaf, slot, "invalid root level, have %u expect [0, %u]", btrfs_root_level(&ri), BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } - if (btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL) { + if (unlikely(btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL)) { generic_err(leaf, slot, "invalid root level, have %u expect [0, %u]", btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1); @@ -1126,7 +1135,7 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, } /* Flags check */ - if (btrfs_root_flags(&ri) & ~valid_root_flags) { + if (unlikely(btrfs_root_flags(&ri) & ~valid_root_flags)) { generic_err(leaf, slot, "invalid root flags, have 0x%llx expect mask 0x%llx", btrfs_root_flags(&ri), valid_root_flags); @@ -1180,14 +1189,14 @@ static int check_extent_item(struct extent_buffer *leaf, u64 total_refs; /* Total refs in btrfs_extent_item */ u64 inline_refs = 0; /* found total inline refs */ - if (key->type == BTRFS_METADATA_ITEM_KEY && - !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) { + if (unlikely(key->type == BTRFS_METADATA_ITEM_KEY && + !btrfs_fs_incompat(fs_info, SKINNY_METADATA))) { generic_err(leaf, slot, "invalid key type, METADATA_ITEM type invalid when SKINNY_METADATA feature disabled"); return -EUCLEAN; } /* key->objectid is the bytenr for both key types */ - if (!IS_ALIGNED(key->objectid, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(key->objectid, fs_info->sectorsize))) { generic_err(leaf, slot, "invalid key objectid, have %llu expect to be aligned to %u", key->objectid, fs_info->sectorsize); @@ -1195,8 +1204,8 @@ static int check_extent_item(struct extent_buffer *leaf, } /* key->offset is tree level for METADATA_ITEM_KEY */ - if (key->type == BTRFS_METADATA_ITEM_KEY && - key->offset >= BTRFS_MAX_LEVEL) { + if (unlikely(key->type == BTRFS_METADATA_ITEM_KEY && + key->offset >= BTRFS_MAX_LEVEL)) { extent_err(leaf, slot, "invalid tree level, have %llu expect [0, %u]", key->offset, BTRFS_MAX_LEVEL - 1); @@ -1222,7 +1231,7 @@ static int check_extent_item(struct extent_buffer *leaf, * Either using btrfs_extent_inline_ref::offset, or specific * data structure. */ - if (item_size < sizeof(*ei)) { + if (unlikely(item_size < sizeof(*ei))) { extent_err(leaf, slot, "invalid item size, have %u expect [%zu, %u)", item_size, sizeof(*ei), @@ -1236,15 +1245,16 @@ static int check_extent_item(struct extent_buffer *leaf, flags = btrfs_extent_flags(leaf, ei); total_refs = btrfs_extent_refs(leaf, ei); generation = btrfs_extent_generation(leaf, ei); - if (generation > btrfs_super_generation(fs_info->super_copy) + 1) { + if (unlikely(generation > + btrfs_super_generation(fs_info->super_copy) + 1)) { extent_err(leaf, slot, "invalid generation, have %llu expect (0, %llu]", generation, btrfs_super_generation(fs_info->super_copy) + 1); return -EUCLEAN; } - if (!has_single_bit_set(flags & (BTRFS_EXTENT_FLAG_DATA | - BTRFS_EXTENT_FLAG_TREE_BLOCK))) { + if (unlikely(!has_single_bit_set(flags & (BTRFS_EXTENT_FLAG_DATA | + BTRFS_EXTENT_FLAG_TREE_BLOCK)))) { extent_err(leaf, slot, "invalid extent flag, have 0x%llx expect 1 bit set in 0x%llx", flags, BTRFS_EXTENT_FLAG_DATA | @@ -1253,21 +1263,21 @@ static int check_extent_item(struct extent_buffer *leaf, } is_tree_block = !!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK); if (is_tree_block) { - if (key->type == BTRFS_EXTENT_ITEM_KEY && - key->offset != fs_info->nodesize) { + if (unlikely(key->type == BTRFS_EXTENT_ITEM_KEY && + key->offset != fs_info->nodesize)) { extent_err(leaf, slot, "invalid extent length, have %llu expect %u", key->offset, fs_info->nodesize); return -EUCLEAN; } } else { - if (key->type != BTRFS_EXTENT_ITEM_KEY) { + if (unlikely(key->type != BTRFS_EXTENT_ITEM_KEY)) { extent_err(leaf, slot, "invalid key type, have %u expect %u for data backref", key->type, BTRFS_EXTENT_ITEM_KEY); return -EUCLEAN; } - if (!IS_ALIGNED(key->offset, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(key->offset, fs_info->sectorsize))) { extent_err(leaf, slot, "invalid extent length, have %llu expect aligned to %u", key->offset, fs_info->sectorsize); @@ -1281,7 +1291,7 @@ static int check_extent_item(struct extent_buffer *leaf, struct btrfs_tree_block_info *info; info = (struct btrfs_tree_block_info *)ptr; - if (btrfs_tree_block_level(leaf, info) >= BTRFS_MAX_LEVEL) { + if (unlikely(btrfs_tree_block_level(leaf, info) >= BTRFS_MAX_LEVEL)) { extent_err(leaf, slot, "invalid tree block info level, have %u expect [0, %u]", btrfs_tree_block_level(leaf, info), @@ -1300,7 +1310,7 @@ static int check_extent_item(struct extent_buffer *leaf, u64 inline_offset; u8 inline_type; - if (ptr + sizeof(*iref) > end) { + if (unlikely(ptr + sizeof(*iref) > end)) { extent_err(leaf, slot, "inline ref item overflows extent item, ptr %lu iref size %zu end %lu", ptr, sizeof(*iref), end); @@ -1309,7 +1319,7 @@ static int check_extent_item(struct extent_buffer *leaf, iref = (struct btrfs_extent_inline_ref *)ptr; inline_type = btrfs_extent_inline_ref_type(leaf, iref); inline_offset = btrfs_extent_inline_ref_offset(leaf, iref); - if (ptr + btrfs_extent_inline_ref_size(inline_type) > end) { + if (unlikely(ptr + btrfs_extent_inline_ref_size(inline_type) > end)) { extent_err(leaf, slot, "inline ref item overflows extent item, ptr %lu iref size %u end %lu", ptr, inline_type, end); @@ -1323,7 +1333,8 @@ static int check_extent_item(struct extent_buffer *leaf, break; /* Contains parent bytenr */ case BTRFS_SHARED_BLOCK_REF_KEY: - if (!IS_ALIGNED(inline_offset, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(inline_offset, + fs_info->sectorsize))) { extent_err(leaf, slot, "invalid tree parent bytenr, have %llu expect aligned to %u", inline_offset, fs_info->sectorsize); @@ -1338,7 +1349,8 @@ static int check_extent_item(struct extent_buffer *leaf, case BTRFS_EXTENT_DATA_REF_KEY: dref = (struct btrfs_extent_data_ref *)(&iref->offset); dref_offset = btrfs_extent_data_ref_offset(leaf, dref); - if (!IS_ALIGNED(dref_offset, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(dref_offset, + fs_info->sectorsize))) { extent_err(leaf, slot, "invalid data ref offset, have %llu expect aligned to %u", dref_offset, fs_info->sectorsize); @@ -1349,7 +1361,8 @@ static int check_extent_item(struct extent_buffer *leaf, /* Contains parent bytenr and ref count */ case BTRFS_SHARED_DATA_REF_KEY: sref = (struct btrfs_shared_data_ref *)(iref + 1); - if (!IS_ALIGNED(inline_offset, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(inline_offset, + fs_info->sectorsize))) { extent_err(leaf, slot, "invalid data parent bytenr, have %llu expect aligned to %u", inline_offset, fs_info->sectorsize); @@ -1365,14 +1378,14 @@ static int check_extent_item(struct extent_buffer *leaf, ptr += btrfs_extent_inline_ref_size(inline_type); } /* No padding is allowed */ - if (ptr != end) { + if (unlikely(ptr != end)) { extent_err(leaf, slot, "invalid extent item size, padding bytes found"); return -EUCLEAN; } /* Finally, check the inline refs against total refs */ - if (inline_refs > total_refs) { + if (unlikely(inline_refs > total_refs)) { extent_err(leaf, slot, "invalid extent refs, have %llu expect >= inline %llu", total_refs, inline_refs); @@ -1389,21 +1402,21 @@ static int check_simple_keyed_refs(struct extent_buffer *leaf, if (key->type == BTRFS_SHARED_DATA_REF_KEY) expect_item_size = sizeof(struct btrfs_shared_data_ref); - if (btrfs_item_size_nr(leaf, slot) != expect_item_size) { + if (unlikely(btrfs_item_size_nr(leaf, slot) != expect_item_size)) { generic_err(leaf, slot, "invalid item size, have %u expect %u for key type %u", btrfs_item_size_nr(leaf, slot), expect_item_size, key->type); return -EUCLEAN; } - if (!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize))) { generic_err(leaf, slot, "invalid key objectid for shared block ref, have %llu expect aligned to %u", key->objectid, leaf->fs_info->sectorsize); return -EUCLEAN; } - if (key->type != BTRFS_TREE_BLOCK_REF_KEY && - !IS_ALIGNED(key->offset, leaf->fs_info->sectorsize)) { + if (unlikely(key->type != BTRFS_TREE_BLOCK_REF_KEY && + !IS_ALIGNED(key->offset, leaf->fs_info->sectorsize))) { extent_err(leaf, slot, "invalid tree parent bytenr, have %llu expect aligned to %u", key->offset, leaf->fs_info->sectorsize); @@ -1419,14 +1432,14 @@ static int check_extent_data_ref(struct extent_buffer *leaf, unsigned long ptr = btrfs_item_ptr_offset(leaf, slot); const unsigned long end = ptr + btrfs_item_size_nr(leaf, slot); - if (btrfs_item_size_nr(leaf, slot) % sizeof(*dref) != 0) { + if (unlikely(btrfs_item_size_nr(leaf, slot) % sizeof(*dref) != 0)) { generic_err(leaf, slot, "invalid item size, have %u expect aligned to %zu for key type %u", btrfs_item_size_nr(leaf, slot), sizeof(*dref), key->type); return -EUCLEAN; } - if (!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize))) { generic_err(leaf, slot, "invalid key objectid for shared block ref, have %llu expect aligned to %u", key->objectid, leaf->fs_info->sectorsize); @@ -1443,13 +1456,13 @@ static int check_extent_data_ref(struct extent_buffer *leaf, owner = btrfs_extent_data_ref_objectid(leaf, dref); offset = btrfs_extent_data_ref_offset(leaf, dref); hash = hash_extent_data_ref(root_objectid, owner, offset); - if (hash != key->offset) { + if (unlikely(hash != key->offset)) { extent_err(leaf, slot, "invalid extent data ref hash, item has 0x%016llx key has 0x%016llx", hash, key->offset); return -EUCLEAN; } - if (!IS_ALIGNED(offset, leaf->fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { extent_err(leaf, slot, "invalid extent data backref offset, have %llu expect aligned to %u", offset, leaf->fs_info->sectorsize); @@ -1469,10 +1482,10 @@ static int check_inode_ref(struct extent_buffer *leaf, unsigned long ptr; unsigned long end; - if (!check_prev_ino(leaf, key, slot, prev_key)) + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) return -EUCLEAN; /* namelen can't be 0, so item_size == sizeof() is also invalid */ - if (btrfs_item_size_nr(leaf, slot) <= sizeof(*iref)) { + if (unlikely(btrfs_item_size_nr(leaf, slot) <= sizeof(*iref))) { inode_ref_err(leaf, slot, "invalid item size, have %u expect (%zu, %u)", btrfs_item_size_nr(leaf, slot), @@ -1485,7 +1498,7 @@ static int check_inode_ref(struct extent_buffer *leaf, while (ptr < end) { u16 namelen; - if (ptr + sizeof(iref) > end) { + if (unlikely(ptr + sizeof(iref) > end)) { inode_ref_err(leaf, slot, "inode ref overflow, ptr %lu end %lu inode_ref_size %zu", ptr, end, sizeof(iref)); @@ -1494,7 +1507,7 @@ static int check_inode_ref(struct extent_buffer *leaf, iref = (struct btrfs_inode_ref *)ptr; namelen = btrfs_inode_ref_name_len(leaf, iref); - if (ptr + sizeof(*iref) + namelen > end) { + if (unlikely(ptr + sizeof(*iref) + namelen > end)) { inode_ref_err(leaf, slot, "inode ref overflow, ptr %lu end %lu namelen %u", ptr, end, namelen); @@ -1577,7 +1590,7 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) u32 nritems = btrfs_header_nritems(leaf); int slot; - if (btrfs_header_level(leaf) != 0) { + if (unlikely(btrfs_header_level(leaf) != 0)) { generic_err(leaf, 0, "invalid level for leaf, have %d expect 0", btrfs_header_level(leaf)); @@ -1596,19 +1609,19 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) u64 owner = btrfs_header_owner(leaf); /* These trees must never be empty */ - if (owner == BTRFS_ROOT_TREE_OBJECTID || - owner == BTRFS_CHUNK_TREE_OBJECTID || - owner == BTRFS_EXTENT_TREE_OBJECTID || - owner == BTRFS_DEV_TREE_OBJECTID || - owner == BTRFS_FS_TREE_OBJECTID || - owner == BTRFS_DATA_RELOC_TREE_OBJECTID) { + if (unlikely(owner == BTRFS_ROOT_TREE_OBJECTID || + owner == BTRFS_CHUNK_TREE_OBJECTID || + owner == BTRFS_EXTENT_TREE_OBJECTID || + owner == BTRFS_DEV_TREE_OBJECTID || + owner == BTRFS_FS_TREE_OBJECTID || + owner == BTRFS_DATA_RELOC_TREE_OBJECTID)) { generic_err(leaf, 0, "invalid root, root %llu must never be empty", owner); return -EUCLEAN; } /* Unknown tree */ - if (owner == 0) { + if (unlikely(owner == 0)) { generic_err(leaf, 0, "invalid owner, root 0 is not defined"); return -EUCLEAN; @@ -1616,7 +1629,7 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) return 0; } - if (nritems == 0) + if (unlikely(nritems == 0)) return 0; /* @@ -1637,7 +1650,7 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) btrfs_item_key_to_cpu(leaf, &key, slot); /* Make sure the keys are in the right order */ - if (btrfs_comp_cpu_keys(&prev_key, &key) >= 0) { + if (unlikely(btrfs_comp_cpu_keys(&prev_key, &key) >= 0)) { generic_err(leaf, slot, "bad key order, prev (%llu %u %llu) current (%llu %u %llu)", prev_key.objectid, prev_key.type, @@ -1656,7 +1669,7 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) else item_end_expected = btrfs_item_offset_nr(leaf, slot - 1); - if (btrfs_item_end_nr(leaf, slot) != item_end_expected) { + if (unlikely(btrfs_item_end_nr(leaf, slot) != item_end_expected)) { generic_err(leaf, slot, "unexpected item end, have %u expect %u", btrfs_item_end_nr(leaf, slot), @@ -1669,8 +1682,8 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) * just in case all the items are consistent to each other, but * all point outside of the leaf. */ - if (btrfs_item_end_nr(leaf, slot) > - BTRFS_LEAF_DATA_SIZE(fs_info)) { + if (unlikely(btrfs_item_end_nr(leaf, slot) > + BTRFS_LEAF_DATA_SIZE(fs_info))) { generic_err(leaf, slot, "slot end outside of leaf, have %u expect range [0, %u]", btrfs_item_end_nr(leaf, slot), @@ -1679,8 +1692,8 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) } /* Also check if the item pointer overlaps with btrfs item. */ - if (btrfs_item_nr_offset(slot) + sizeof(struct btrfs_item) > - btrfs_item_ptr_offset(leaf, slot)) { + if (unlikely(btrfs_item_ptr_offset(leaf, slot) < + btrfs_item_nr_offset(slot) + sizeof(struct btrfs_item))) { generic_err(leaf, slot, "slot overlaps with its data, item end %lu data start %lu", btrfs_item_nr_offset(slot) + @@ -1695,7 +1708,7 @@ static int check_leaf(struct extent_buffer *leaf, bool check_item_data) * criteria */ ret = check_leaf_item(leaf, &key, slot, &prev_key); - if (ret < 0) + if (unlikely(ret < 0)) return ret; } @@ -1728,13 +1741,13 @@ int btrfs_check_node(struct extent_buffer *node) u64 bytenr; int ret = 0; - if (level <= 0 || level >= BTRFS_MAX_LEVEL) { + if (unlikely(level <= 0 || level >= BTRFS_MAX_LEVEL)) { generic_err(node, 0, "invalid level for node, have %d expect [1, %d]", level, BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } - if (nr == 0 || nr > BTRFS_NODEPTRS_PER_BLOCK(fs_info)) { + if (unlikely(nr == 0 || nr > BTRFS_NODEPTRS_PER_BLOCK(fs_info))) { btrfs_crit(fs_info, "corrupt node: root=%llu block=%llu, nritems too %s, have %lu expect range [1,%u]", btrfs_header_owner(node), node->start, @@ -1748,13 +1761,13 @@ int btrfs_check_node(struct extent_buffer *node) btrfs_node_key_to_cpu(node, &key, slot); btrfs_node_key_to_cpu(node, &next_key, slot + 1); - if (!bytenr) { + if (unlikely(!bytenr)) { generic_err(node, slot, "invalid NULL node pointer"); ret = -EUCLEAN; goto out; } - if (!IS_ALIGNED(bytenr, fs_info->sectorsize)) { + if (unlikely(!IS_ALIGNED(bytenr, fs_info->sectorsize))) { generic_err(node, slot, "unaligned pointer, have %llu should be aligned to %u", bytenr, fs_info->sectorsize); @@ -1762,7 +1775,7 @@ int btrfs_check_node(struct extent_buffer *node) goto out; } - if (btrfs_comp_cpu_keys(&key, &next_key) >= 0) { + if (unlikely(btrfs_comp_cpu_keys(&key, &next_key) >= 0)) { generic_err(node, slot, "bad key order, current (%llu %u %llu) next (%llu %u %llu)", key.objectid, key.type, key.offset, -- cgit v1.2.3-59-g8ed1b