diff options
Diffstat (limited to 'drivers/staging/erofs/unzip_vle.c')
-rw-r--r-- | drivers/staging/erofs/unzip_vle.c | 168 |
1 files changed, 95 insertions, 73 deletions
diff --git a/drivers/staging/erofs/unzip_vle.c b/drivers/staging/erofs/unzip_vle.c index 4ac1099a39c6..8715bc50e09c 100644 --- a/drivers/staging/erofs/unzip_vle.c +++ b/drivers/staging/erofs/unzip_vle.c @@ -107,15 +107,30 @@ enum z_erofs_vle_work_role { Z_EROFS_VLE_WORK_SECONDARY, Z_EROFS_VLE_WORK_PRIMARY, /* - * The current work has at least been linked with the following - * processed chained works, which means if the processing page - * is the tail partial page of the work, the current work can - * safely use the whole page, as illustrated below: - * +--------------+-------------------------------------------+ - * | tail page | head page (of the previous work) | - * +--------------+-------------------------------------------+ - * /\ which belongs to the current work - * [ (*) this page can be used for the current work itself. ] + * The current work was the tail of an exist chain, and the previous + * processed chained works are all decided to be hooked up to it. + * A new chain should be created for the remaining unprocessed works, + * therefore different from Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, + * the next work cannot reuse the whole page in the following scenario: + * ________________________________________________________________ + * | tail (partial) page | head (partial) page | + * | (belongs to the next work) | (belongs to the current work) | + * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| + */ + Z_EROFS_VLE_WORK_PRIMARY_HOOKED, + /* + * The current work has been linked with the processed chained works, + * and could be also linked with the potential remaining works, which + * means if the processing page is the tail partial page of the work, + * the current work can safely use the whole page (since the next work + * is under control) for in-place decompression, as illustrated below: + * ________________________________________________________________ + * | tail (partial) page | head (partial) page | + * | (of the current work) | (of the previous work) | + * | PRIMARY_FOLLOWED or | | + * |_____PRIMARY_HOOKED____|____________PRIMARY_FOLLOWED____________| + * + * [ (*) the above page can be used for the current work itself. ] */ Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, Z_EROFS_VLE_WORK_MAX @@ -238,14 +253,9 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, { struct erofs_sb_info *const sbi = EROFS_SB(mapping->host->i_sb); const unsigned int clusterpages = erofs_clusterpages(sbi); - - struct z_erofs_vle_workgroup *grp; + struct z_erofs_vle_workgroup *const grp = (void *)page_private(page); int ret = 0; /* 0 - busy */ - /* prevent the workgroup from being freed */ - rcu_read_lock(); - grp = (void *)page_private(page); - if (erofs_workgroup_try_to_freeze(&grp->obj, 1)) { unsigned int i; @@ -257,12 +267,11 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, } } erofs_workgroup_unfreeze(&grp->obj, 1); - } - rcu_read_unlock(); - if (ret) { - ClearPagePrivate(page); - put_page(page); + if (ret) { + ClearPagePrivate(page); + put_page(page); + } } return ret; } @@ -315,10 +324,10 @@ static int z_erofs_vle_work_add_page( return ret ? 0 : -EAGAIN; } -static inline bool try_to_claim_workgroup( - struct z_erofs_vle_workgroup *grp, - z_erofs_vle_owned_workgrp_t *owned_head, - bool *hosted) +static enum z_erofs_vle_work_role +try_to_claim_workgroup(struct z_erofs_vle_workgroup *grp, + z_erofs_vle_owned_workgrp_t *owned_head, + bool *hosted) { DBG_BUGON(*hosted == true); @@ -332,6 +341,9 @@ retry: *owned_head = &grp->next; *hosted = true; + /* lucky, I am the followee :) */ + return Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED; + } else if (grp->next == Z_EROFS_VLE_WORKGRP_TAIL) { /* * type 2, link to the end of a existing open chain, @@ -341,12 +353,11 @@ retry: if (cmpxchg(&grp->next, Z_EROFS_VLE_WORKGRP_TAIL, *owned_head) != Z_EROFS_VLE_WORKGRP_TAIL) goto retry; - *owned_head = Z_EROFS_VLE_WORKGRP_TAIL; - } else - return false; /* :( better luck next time */ + return Z_EROFS_VLE_WORK_PRIMARY_HOOKED; + } - return true; /* lucky, I am the followee :) */ + return Z_EROFS_VLE_WORK_PRIMARY; /* :( better luck next time */ } struct z_erofs_vle_work_finder { @@ -424,12 +435,9 @@ z_erofs_vle_work_lookup(const struct z_erofs_vle_work_finder *f) *f->hosted = false; if (!primary) *f->role = Z_EROFS_VLE_WORK_SECONDARY; - /* claim the workgroup if possible */ - else if (try_to_claim_workgroup(grp, f->owned_head, f->hosted)) - *f->role = Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED; - else - *f->role = Z_EROFS_VLE_WORK_PRIMARY; - + else /* claim the workgroup if possible */ + *f->role = try_to_claim_workgroup(grp, f->owned_head, + f->hosted); return work; } @@ -493,6 +501,9 @@ z_erofs_vle_work_register(const struct z_erofs_vle_work_finder *f, return work; } +#define builder_is_hooked(builder) \ + ((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_HOOKED) + #define builder_is_followed(builder) \ ((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED) @@ -539,7 +550,7 @@ repeat: if (unlikely(work == ERR_PTR(-EAGAIN))) goto repeat; - if (unlikely(IS_ERR(work))) + if (IS_ERR(work)) return PTR_ERR(work); got_it: z_erofs_pagevec_ctor_init(&builder->vector, @@ -589,7 +600,7 @@ static void __z_erofs_vle_work_release(struct z_erofs_vle_workgroup *grp, erofs_workgroup_put(&grp->obj); } -void z_erofs_vle_work_release(struct z_erofs_vle_work *work) +static void z_erofs_vle_work_release(struct z_erofs_vle_work *work) { struct z_erofs_vle_workgroup *grp = z_erofs_vle_work_workgroup(work, true); @@ -636,7 +647,7 @@ struct z_erofs_vle_frontend { struct inode *const inode; struct z_erofs_vle_work_builder builder; - struct erofs_map_blocks_iter m_iter; + struct erofs_map_blocks map; z_erofs_vle_owned_workgrp_t owned_head; @@ -647,8 +658,9 @@ struct z_erofs_vle_frontend { #define VLE_FRONTEND_INIT(__i) { \ .inode = __i, \ - .m_iter = { \ - { .m_llen = 0, .m_plen = 0 }, \ + .map = { \ + .m_llen = 0, \ + .m_plen = 0, \ .mpage = NULL \ }, \ .builder = VLE_WORK_BUILDER_INIT(), \ @@ -681,12 +693,11 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, { struct super_block *const sb = fe->inode->i_sb; struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb); - struct erofs_map_blocks_iter *const m = &fe->m_iter; - struct erofs_map_blocks *const map = &m->map; + struct erofs_map_blocks *const map = &fe->map; struct z_erofs_vle_work_builder *const builder = &fe->builder; const loff_t offset = page_offset(page); - bool tight = builder_is_followed(builder); + bool tight = builder_is_hooked(builder); struct z_erofs_vle_work *work = builder->work; enum z_erofs_cache_alloctype cache_strategy; @@ -704,8 +715,12 @@ repeat: /* lucky, within the range of the current map_blocks */ if (offset + cur >= map->m_la && - offset + cur < map->m_la + map->m_llen) + offset + cur < map->m_la + map->m_llen) { + /* didn't get a valid unzip work previously (very rare) */ + if (!builder->work) + goto restart_now; goto hitted; + } /* go ahead the next map_blocks */ debugln("%s: [out-of-range] pos %llu", __func__, offset + cur); @@ -715,10 +730,11 @@ repeat: map->m_la = offset + cur; map->m_llen = 0; - err = erofs_map_blocks_iter(fe->inode, map, &m->mpage, 0); + err = z_erofs_map_blocks_iter(fe->inode, map, 0); if (unlikely(err)) goto err_out; +restart_now: if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) goto hitted; @@ -740,7 +756,7 @@ repeat: map->m_plen / PAGE_SIZE, cache_strategy, page_pool, GFP_KERNEL); - tight &= builder_is_followed(builder); + tight &= builder_is_hooked(builder); work = builder->work; hitted: cur = end - min_t(unsigned int, offset + end - map->m_la, end); @@ -755,6 +771,9 @@ hitted: (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED)); + if (cur) + tight &= builder_is_followed(builder); + retry: err = z_erofs_vle_work_add_page(builder, page, page_type); /* should allocate an additional staging page for pagevec */ @@ -830,8 +849,9 @@ static inline void z_erofs_vle_read_endio(struct bio *bio) #ifdef EROFS_FS_HAS_MANAGED_CACHE struct address_space *mc = NULL; #endif + struct bvec_iter_all iter_all; - bio_for_each_segment_all(bvec, bio, i) { + bio_for_each_segment_all(bvec, bio, i, iter_all) { struct page *page = bvec->bv_page; bool cachemngd = false; @@ -992,11 +1012,10 @@ repeat: if (llen > grp->llen) llen = grp->llen; - err = z_erofs_vle_unzip_fast_percpu(compressed_pages, - clusterpages, pages, llen, work->pageofs, - z_erofs_onlinepage_endio); + err = z_erofs_vle_unzip_fast_percpu(compressed_pages, clusterpages, + pages, llen, work->pageofs); if (err != -ENOTSUPP) - goto out_percpu; + goto out; if (sparsemem_pages >= nr_pages) goto skip_allocpage; @@ -1017,8 +1036,25 @@ skip_allocpage: erofs_vunmap(vout, nr_pages); out: + /* must handle all compressed pages before endding pages */ + for (i = 0; i < clusterpages; ++i) { + page = compressed_pages[i]; + +#ifdef EROFS_FS_HAS_MANAGED_CACHE + if (page->mapping == MNGD_MAPPING(sbi)) + continue; +#endif + /* recycle all individual staging pages */ + (void)z_erofs_gather_if_stagingpage(page_pool, page); + + WRITE_ONCE(compressed_pages[i], NULL); + } + for (i = 0; i < nr_pages; ++i) { page = pages[i]; + if (!page) + continue; + DBG_BUGON(!page->mapping); /* recycle all individual staging pages */ @@ -1031,20 +1067,6 @@ out: z_erofs_onlinepage_endio(page); } -out_percpu: - for (i = 0; i < clusterpages; ++i) { - page = compressed_pages[i]; - -#ifdef EROFS_FS_HAS_MANAGED_CACHE - if (page->mapping == MNGD_MAPPING(sbi)) - continue; -#endif - /* recycle all individual staging pages */ - (void)z_erofs_gather_if_stagingpage(page_pool, page); - - WRITE_ONCE(compressed_pages[i], NULL); - } - if (pages == z_pagemap_global) mutex_unlock(&z_pagemap_global_lock); else if (unlikely(pages != pages_onstack)) @@ -1484,8 +1506,8 @@ static int z_erofs_vle_normalaccess_readpage(struct file *file, z_erofs_submit_and_unzip(&f, &pagepool, true); out: - if (f.m_iter.mpage) - put_page(f.m_iter.mpage); + if (f.map.mpage) + put_page(f.map.mpage); /* clean up the remaining free pages */ put_pages_list(&pagepool); @@ -1555,8 +1577,8 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp, z_erofs_submit_and_unzip(&f, &pagepool, sync); - if (f.m_iter.mpage) - put_page(f.m_iter.mpage); + if (f.map.mpage) + put_page(f.map.mpage); /* clean up the remaining free pages */ put_pages_list(&pagepool); @@ -1701,14 +1723,14 @@ vle_get_logical_extent_head(const struct vle_map_blocks_iter_ctx *ctx, int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, - struct page **mpage_ret, int flags) + int flags) { void *kaddr; const struct vle_map_blocks_iter_ctx ctx = { .inode = inode, .sb = inode->i_sb, .clusterbits = EROFS_I_SB(inode)->clusterbits, - .mpage_ret = mpage_ret, + .mpage_ret = &map->mpage, .kaddr_ret = &kaddr }; const unsigned int clustersize = 1 << ctx.clusterbits; @@ -1722,7 +1744,7 @@ int z_erofs_map_blocks_iter(struct inode *inode, /* initialize `pblk' to keep gcc from printing foolish warnings */ erofs_blk_t mblk, pblk = 0; - struct page *mpage = *mpage_ret; + struct page *mpage = map->mpage; struct z_erofs_vle_decompressed_index *di; unsigned int cluster_type, logical_cluster_ofs; int err = 0; @@ -1758,7 +1780,7 @@ int z_erofs_map_blocks_iter(struct inode *inode, err = PTR_ERR(mpage); goto out; } - *mpage_ret = mpage; + map->mpage = mpage; } else { lock_page(mpage); DBG_BUGON(!PageUptodate(mpage)); @@ -1818,7 +1840,7 @@ int z_erofs_map_blocks_iter(struct inode *inode, /* get the correspoinding first chunk */ err = vle_get_logical_extent_head(&ctx, lcn, &ofs, &pblk, &map->m_flags); - mpage = *mpage_ret; + mpage = map->mpage; if (unlikely(err)) { if (mpage) |