diff options
Diffstat (limited to 'fs')
75 files changed, 5100 insertions, 9207 deletions
diff --git a/fs/9p/cache.c b/fs/9p/cache.c index f2ba131cede1..55e108e5e133 100644 --- a/fs/9p/cache.c +++ b/fs/9p/cache.c @@ -16,186 +16,61 @@ #include "v9fs.h" #include "cache.h" -#define CACHETAG_LEN 11 - -struct fscache_netfs v9fs_cache_netfs = { - .name = "9p", - .version = 0, -}; - -/* - * v9fs_random_cachetag - Generate a random tag to be associated - * with a new cache session. - * - * The value of jiffies is used for a fairly randomly cache tag. - */ - -static -int v9fs_random_cachetag(struct v9fs_session_info *v9ses) +int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, + const char *dev_name) { - v9ses->cachetag = kmalloc(CACHETAG_LEN, GFP_KERNEL); - if (!v9ses->cachetag) - return -ENOMEM; + struct fscache_volume *vcookie; + char *name, *p; - return scnprintf(v9ses->cachetag, CACHETAG_LEN, "%lu", jiffies); -} - -const struct fscache_cookie_def v9fs_cache_session_index_def = { - .name = "9P.session", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; + name = kasprintf(GFP_KERNEL, "9p,%s,%s", + dev_name, v9ses->cachetag ?: v9ses->aname); + if (!name) + return -ENOMEM; -void v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses) -{ - /* If no cache session tag was specified, we generate a random one. */ - if (!v9ses->cachetag) { - if (v9fs_random_cachetag(v9ses) < 0) { - v9ses->fscache = NULL; - kfree(v9ses->cachetag); - v9ses->cachetag = NULL; - return; + for (p = name; *p; p++) + if (*p == '/') + *p = ';'; + + vcookie = fscache_acquire_volume(name, NULL, NULL, 0); + p9_debug(P9_DEBUG_FSC, "session %p get volume %p (%s)\n", + v9ses, vcookie, name); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(name); + return PTR_ERR(vcookie); } + pr_err("Cache volume key already in use (%s)\n", name); + vcookie = NULL; } - - v9ses->fscache = fscache_acquire_cookie(v9fs_cache_netfs.primary_index, - &v9fs_cache_session_index_def, - v9ses->cachetag, - strlen(v9ses->cachetag), - NULL, 0, - v9ses, 0, true); - p9_debug(P9_DEBUG_FSC, "session %p get cookie %p\n", - v9ses, v9ses->fscache); -} - -void v9fs_cache_session_put_cookie(struct v9fs_session_info *v9ses) -{ - p9_debug(P9_DEBUG_FSC, "session %p put cookie %p\n", - v9ses, v9ses->fscache); - fscache_relinquish_cookie(v9ses->fscache, NULL, false); - v9ses->fscache = NULL; -} - -static enum -fscache_checkaux v9fs_cache_inode_check_aux(void *cookie_netfs_data, - const void *buffer, - uint16_t buflen, - loff_t object_size) -{ - const struct v9fs_inode *v9inode = cookie_netfs_data; - - if (buflen != sizeof(v9inode->qid.version)) - return FSCACHE_CHECKAUX_OBSOLETE; - - if (memcmp(buffer, &v9inode->qid.version, - sizeof(v9inode->qid.version))) - return FSCACHE_CHECKAUX_OBSOLETE; - - return FSCACHE_CHECKAUX_OKAY; + v9ses->fscache = vcookie; + kfree(name); + return 0; } -const struct fscache_cookie_def v9fs_cache_inode_index_def = { - .name = "9p.inode", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .check_aux = v9fs_cache_inode_check_aux, -}; - void v9fs_cache_inode_get_cookie(struct inode *inode) { struct v9fs_inode *v9inode; struct v9fs_session_info *v9ses; + __le32 version; + __le64 path; if (!S_ISREG(inode->i_mode)) return; v9inode = V9FS_I(inode); - if (v9inode->fscache) + if (WARN_ON(v9inode->fscache)) return; + version = cpu_to_le32(v9inode->qid.version); + path = cpu_to_le64(v9inode->qid.path); v9ses = v9fs_inode2v9ses(inode); - v9inode->fscache = fscache_acquire_cookie(v9ses->fscache, - &v9fs_cache_inode_index_def, - &v9inode->qid.path, - sizeof(v9inode->qid.path), - &v9inode->qid.version, - sizeof(v9inode->qid.version), - v9inode, - i_size_read(&v9inode->vfs_inode), - true); + v9inode->fscache = + fscache_acquire_cookie(v9fs_session_cache(v9ses), + 0, + &path, sizeof(path), + &version, sizeof(version), + i_size_read(&v9inode->vfs_inode)); p9_debug(P9_DEBUG_FSC, "inode %p get cookie %p\n", inode, v9inode->fscache); } - -void v9fs_cache_inode_put_cookie(struct inode *inode) -{ - struct v9fs_inode *v9inode = V9FS_I(inode); - - if (!v9inode->fscache) - return; - p9_debug(P9_DEBUG_FSC, "inode %p put cookie %p\n", - inode, v9inode->fscache); - - fscache_relinquish_cookie(v9inode->fscache, &v9inode->qid.version, - false); - v9inode->fscache = NULL; -} - -void v9fs_cache_inode_flush_cookie(struct inode *inode) -{ - struct v9fs_inode *v9inode = V9FS_I(inode); - - if (!v9inode->fscache) - return; - p9_debug(P9_DEBUG_FSC, "inode %p flush cookie %p\n", - inode, v9inode->fscache); - - fscache_relinquish_cookie(v9inode->fscache, NULL, true); - v9inode->fscache = NULL; -} - -void v9fs_cache_inode_set_cookie(struct inode *inode, struct file *filp) -{ - struct v9fs_inode *v9inode = V9FS_I(inode); - - if (!v9inode->fscache) - return; - - mutex_lock(&v9inode->fscache_lock); - - if ((filp->f_flags & O_ACCMODE) != O_RDONLY) - v9fs_cache_inode_flush_cookie(inode); - else - v9fs_cache_inode_get_cookie(inode); - - mutex_unlock(&v9inode->fscache_lock); -} - -void v9fs_cache_inode_reset_cookie(struct inode *inode) -{ - struct v9fs_inode *v9inode = V9FS_I(inode); - struct v9fs_session_info *v9ses; - struct fscache_cookie *old; - - if (!v9inode->fscache) - return; - - old = v9inode->fscache; - - mutex_lock(&v9inode->fscache_lock); - fscache_relinquish_cookie(v9inode->fscache, NULL, true); - - v9ses = v9fs_inode2v9ses(inode); - v9inode->fscache = fscache_acquire_cookie(v9ses->fscache, - &v9fs_cache_inode_index_def, - &v9inode->qid.path, - sizeof(v9inode->qid.path), - &v9inode->qid.version, - sizeof(v9inode->qid.version), - v9inode, - i_size_read(&v9inode->vfs_inode), - true); - p9_debug(P9_DEBUG_FSC, "inode %p revalidating cookie old %p new %p\n", - inode, old, v9inode->fscache); - - mutex_unlock(&v9inode->fscache_lock); -} diff --git a/fs/9p/cache.h b/fs/9p/cache.h index 7480b4b49fea..1923affcdc62 100644 --- a/fs/9p/cache.h +++ b/fs/9p/cache.h @@ -7,26 +7,15 @@ #ifndef _9P_CACHE_H #define _9P_CACHE_H -#define FSCACHE_USE_NEW_IO_API + #include <linux/fscache.h> #ifdef CONFIG_9P_FSCACHE -extern struct fscache_netfs v9fs_cache_netfs; -extern const struct fscache_cookie_def v9fs_cache_session_index_def; -extern const struct fscache_cookie_def v9fs_cache_inode_index_def; - -extern void v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses); -extern void v9fs_cache_session_put_cookie(struct v9fs_session_info *v9ses); +extern int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, + const char *dev_name); extern void v9fs_cache_inode_get_cookie(struct inode *inode); -extern void v9fs_cache_inode_put_cookie(struct inode *inode); -extern void v9fs_cache_inode_flush_cookie(struct inode *inode); -extern void v9fs_cache_inode_set_cookie(struct inode *inode, struct file *filp); -extern void v9fs_cache_inode_reset_cookie(struct inode *inode); - -extern int __v9fs_cache_register(void); -extern void __v9fs_cache_unregister(void); #else /* CONFIG_9P_FSCACHE */ @@ -34,13 +23,5 @@ static inline void v9fs_cache_inode_get_cookie(struct inode *inode) { } -static inline void v9fs_cache_inode_put_cookie(struct inode *inode) -{ -} - -static inline void v9fs_cache_inode_set_cookie(struct inode *inode, struct file *file) -{ -} - #endif /* CONFIG_9P_FSCACHE */ #endif /* _9P_CACHE_H */ diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index e32dd5f7721b..08f65c40af4f 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -469,7 +469,11 @@ struct p9_fid *v9fs_session_init(struct v9fs_session_info *v9ses, #ifdef CONFIG_9P_FSCACHE /* register the session for caching */ - v9fs_cache_session_get_cookie(v9ses); + if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) { + rc = v9fs_cache_session_get_cookie(v9ses, dev_name); + if (rc < 0) + goto err_clnt; + } #endif spin_lock(&v9fs_sessionlist_lock); list_add(&v9ses->slist, &v9fs_sessionlist); @@ -502,8 +506,7 @@ void v9fs_session_close(struct v9fs_session_info *v9ses) } #ifdef CONFIG_9P_FSCACHE - if (v9ses->fscache) - v9fs_cache_session_put_cookie(v9ses); + fscache_relinquish_volume(v9fs_session_cache(v9ses), NULL, false); kfree(v9ses->cachetag); #endif kfree(v9ses->uname); @@ -665,20 +668,12 @@ static int v9fs_cache_register(void) ret = v9fs_init_inode_cache(); if (ret < 0) return ret; -#ifdef CONFIG_9P_FSCACHE - ret = fscache_register_netfs(&v9fs_cache_netfs); - if (ret < 0) - v9fs_destroy_inode_cache(); -#endif return ret; } static void v9fs_cache_unregister(void) { v9fs_destroy_inode_cache(); -#ifdef CONFIG_9P_FSCACHE - fscache_unregister_netfs(&v9fs_cache_netfs); -#endif } /** diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h index 1647a8e63671..bc8b30205d36 100644 --- a/fs/9p/v9fs.h +++ b/fs/9p/v9fs.h @@ -89,7 +89,7 @@ struct v9fs_session_info { unsigned int cache; #ifdef CONFIG_9P_FSCACHE char *cachetag; - struct fscache_cookie *fscache; + struct fscache_volume *fscache; #endif char *uname; /* user name to mount as */ @@ -109,7 +109,6 @@ struct v9fs_session_info { struct v9fs_inode { #ifdef CONFIG_9P_FSCACHE - struct mutex fscache_lock; struct fscache_cookie *fscache; #endif struct p9_qid qid; @@ -133,6 +132,16 @@ static inline struct fscache_cookie *v9fs_inode_cookie(struct v9fs_inode *v9inod #endif } +static inline struct fscache_volume *v9fs_session_cache(struct v9fs_session_info *v9ses) +{ +#ifdef CONFIG_9P_FSCACHE + return v9ses->fscache; +#else + return NULL; +#endif +} + + extern int v9fs_show_options(struct seq_file *m, struct dentry *root); struct p9_fid *v9fs_session_init(struct v9fs_session_info *v9ses, diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index fac918ccb305..c72e9f8f5f32 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -16,6 +16,7 @@ #include <linux/pagemap.h> #include <linux/idr.h> #include <linux/sched.h> +#include <linux/swap.h> #include <linux/uio.h> #include <linux/netfs.h> #include <net/9p/9p.h> @@ -78,7 +79,7 @@ static bool v9fs_is_cache_enabled(struct inode *inode) { struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(inode)); - return fscache_cookie_enabled(cookie) && !hlist_empty(&cookie->backing_objects); + return fscache_cookie_enabled(cookie) && cookie->cache_priv; } /** @@ -87,9 +88,13 @@ static bool v9fs_is_cache_enabled(struct inode *inode) */ static int v9fs_begin_cache_operation(struct netfs_read_request *rreq) { +#ifdef CONFIG_9P_FSCACHE struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(rreq->inode)); - return fscache_begin_read_operation(rreq, cookie); + return fscache_begin_read_operation(&rreq->cache_resources, cookie); +#else + return -ENOBUFS; +#endif } static const struct netfs_read_request_ops v9fs_req_ops = { @@ -133,16 +138,18 @@ static void v9fs_vfs_readahead(struct readahead_control *ractl) static int v9fs_release_page(struct page *page, gfp_t gfp) { struct folio *folio = page_folio(page); + struct inode *inode = folio_inode(folio); if (folio_test_private(folio)) return 0; #ifdef CONFIG_9P_FSCACHE if (folio_test_fscache(folio)) { - if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) + if (current_is_kswapd() || !(gfp & __GFP_FS)) return 0; folio_wait_fscache(folio); } #endif + fscache_note_page_release(v9fs_inode_cookie(V9FS_I(inode))); return 1; } @@ -161,10 +168,25 @@ static void v9fs_invalidate_page(struct page *page, unsigned int offset, folio_wait_fscache(folio); } +static void v9fs_write_to_cache_done(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct v9fs_inode *v9inode = priv; + __le32 version; + + if (IS_ERR_VALUE(transferred_or_error) && + transferred_or_error != -ENOBUFS) { + version = cpu_to_le32(v9inode->qid.version); + fscache_invalidate(v9fs_inode_cookie(v9inode), &version, + i_size_read(&v9inode->vfs_inode), 0); + } +} + static int v9fs_vfs_write_folio_locked(struct folio *folio) { struct inode *inode = folio_inode(folio); struct v9fs_inode *v9inode = V9FS_I(inode); + struct fscache_cookie *cookie = v9fs_inode_cookie(v9inode); loff_t start = folio_pos(folio); loff_t i_size = i_size_read(inode); struct iov_iter from; @@ -181,10 +203,21 @@ static int v9fs_vfs_write_folio_locked(struct folio *folio) /* We should have writeback_fid always set */ BUG_ON(!v9inode->writeback_fid); + folio_wait_fscache(folio); folio_start_writeback(folio); p9_client_write(v9inode->writeback_fid, start, &from, &err); + if (err == 0 && + fscache_cookie_enabled(cookie) && + test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) { + folio_start_fscache(folio); + fscache_write_to_cache(v9fs_inode_cookie(v9inode), + folio_mapping(folio), start, len, i_size, + v9fs_write_to_cache_done, v9inode, + true); + } + folio_end_writeback(folio); return err; } @@ -303,6 +336,7 @@ static int v9fs_write_end(struct file *filp, struct address_space *mapping, loff_t last_pos = pos + copied; struct folio *folio = page_folio(subpage); struct inode *inode = mapping->host; + struct v9fs_inode *v9inode = V9FS_I(inode); p9_debug(P9_DEBUG_VFS, "filp %p, mapping %p\n", filp, mapping); @@ -322,6 +356,7 @@ static int v9fs_write_end(struct file *filp, struct address_space *mapping, if (last_pos > inode->i_size) { inode_add_bytes(inode, last_pos - inode->i_size); i_size_write(inode, last_pos); + fscache_update_cookie(v9fs_inode_cookie(v9inode), NULL, &last_pos); } folio_mark_dirty(folio); out: @@ -331,11 +366,25 @@ out: return copied; } +#ifdef CONFIG_9P_FSCACHE +/* + * Mark a page as having been made dirty and thus needing writeback. We also + * need to pin the cache object to write back to. + */ +static int v9fs_set_page_dirty(struct page *page) +{ + struct v9fs_inode *v9inode = V9FS_I(page->mapping->host); + + return fscache_set_page_dirty(page, v9fs_inode_cookie(v9inode)); +} +#else +#define v9fs_set_page_dirty __set_page_dirty_nobuffers +#endif const struct address_space_operations v9fs_addr_operations = { .readpage = v9fs_vfs_readpage, .readahead = v9fs_vfs_readahead, - .set_page_dirty = __set_page_dirty_nobuffers, + .set_page_dirty = v9fs_set_page_dirty, .writepage = v9fs_vfs_writepage, .write_begin = v9fs_write_begin, .write_end = v9fs_write_end, diff --git a/fs/9p/vfs_dir.c b/fs/9p/vfs_dir.c index 8c854d8cb0cd..958680f7f23e 100644 --- a/fs/9p/vfs_dir.c +++ b/fs/9p/vfs_dir.c @@ -17,6 +17,7 @@ #include <linux/idr.h> #include <linux/slab.h> #include <linux/uio.h> +#include <linux/fscache.h> #include <net/9p/9p.h> #include <net/9p/client.h> @@ -205,7 +206,10 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) int v9fs_dir_release(struct inode *inode, struct file *filp) { + struct v9fs_inode *v9inode = V9FS_I(inode); struct p9_fid *fid; + __le32 version; + loff_t i_size; fid = filp->private_data; p9_debug(P9_DEBUG_VFS, "inode: %p filp: %p fid: %d\n", @@ -216,6 +220,15 @@ int v9fs_dir_release(struct inode *inode, struct file *filp) spin_unlock(&inode->i_lock); p9_client_clunk(fid); } + + if ((filp->f_mode & FMODE_WRITE)) { + version = cpu_to_le32(v9inode->qid.version); + i_size = i_size_read(inode); + fscache_unuse_cookie(v9fs_inode_cookie(v9inode), + &version, &i_size); + } else { + fscache_unuse_cookie(v9fs_inode_cookie(v9inode), NULL, NULL); + } return 0; } diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c index 612e297f3763..be72ad9edb3e 100644 --- a/fs/9p/vfs_file.c +++ b/fs/9p/vfs_file.c @@ -93,7 +93,8 @@ int v9fs_file_open(struct inode *inode, struct file *file) } mutex_unlock(&v9inode->v_mutex); if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) - v9fs_cache_inode_set_cookie(inode, file); + fscache_use_cookie(v9fs_inode_cookie(v9inode), + file->f_mode & FMODE_WRITE); v9fs_open_fid_add(inode, fid); return 0; out_error: diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 328c338ff304..2a10242c79c7 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -233,7 +233,6 @@ struct inode *v9fs_alloc_inode(struct super_block *sb) return NULL; #ifdef CONFIG_9P_FSCACHE v9inode->fscache = NULL; - mutex_init(&v9inode->fscache_lock); #endif v9inode->writeback_fid = NULL; v9inode->cache_validity = 0; @@ -381,12 +380,16 @@ struct inode *v9fs_get_inode(struct super_block *sb, umode_t mode, dev_t rdev) void v9fs_evict_inode(struct inode *inode) { struct v9fs_inode *v9inode = V9FS_I(inode); + __le32 version; truncate_inode_pages_final(&inode->i_data); + version = cpu_to_le32(v9inode->qid.version); + fscache_clear_inode_writeback(v9fs_inode_cookie(v9inode), inode, + &version); clear_inode(inode); filemap_fdatawrite(&inode->i_data); - v9fs_cache_inode_put_cookie(inode); + fscache_relinquish_cookie(v9fs_inode_cookie(v9inode), false); /* clunk the fid stashed in writeback_fid */ if (v9inode->writeback_fid) { p9_client_clunk(v9inode->writeback_fid); @@ -869,7 +872,8 @@ v9fs_vfs_atomic_open(struct inode *dir, struct dentry *dentry, file->private_data = fid; if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) - v9fs_cache_inode_set_cookie(d_inode(dentry), file); + fscache_use_cookie(v9fs_inode_cookie(v9inode), + file->f_mode & FMODE_WRITE); v9fs_open_fid_add(inode, fid); file->f_mode |= FMODE_CREATED; @@ -1072,6 +1076,8 @@ static int v9fs_vfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *iattr) { int retval, use_dentry = 0; + struct inode *inode = d_inode(dentry); + struct v9fs_inode *v9inode = V9FS_I(inode); struct v9fs_session_info *v9ses; struct p9_fid *fid = NULL; struct p9_wstat wstat; @@ -1117,7 +1123,7 @@ static int v9fs_vfs_setattr(struct user_namespace *mnt_userns, /* Write all dirty data */ if (d_is_reg(dentry)) - filemap_write_and_wait(d_inode(dentry)->i_mapping); + filemap_write_and_wait(inode->i_mapping); retval = p9_client_wstat(fid, &wstat); @@ -1128,13 +1134,15 @@ static int v9fs_vfs_setattr(struct user_namespace *mnt_userns, return retval; if ((iattr->ia_valid & ATTR_SIZE) && - iattr->ia_size != i_size_read(d_inode(dentry))) - truncate_setsize(d_inode(dentry), iattr->ia_size); + iattr->ia_size != i_size_read(inode)) { + truncate_setsize(inode, iattr->ia_size); + fscache_resize_cookie(v9fs_inode_cookie(v9inode), iattr->ia_size); + } - v9fs_invalidate_inode_attr(d_inode(dentry)); + v9fs_invalidate_inode_attr(inode); - setattr_copy(&init_user_ns, d_inode(dentry), iattr); - mark_inode_dirty(d_inode(dentry)); + setattr_copy(&init_user_ns, inode, iattr); + mark_inode_dirty(inode); return 0; } diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 7dee89ba32e7..cae301d09cd3 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -344,7 +344,8 @@ v9fs_vfs_atomic_open_dotl(struct inode *dir, struct dentry *dentry, goto err_clunk_old_fid; file->private_data = ofid; if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) - v9fs_cache_inode_set_cookie(inode, file); + fscache_use_cookie(v9fs_inode_cookie(v9inode), + file->f_mode & FMODE_WRITE); v9fs_open_fid_add(inode, ofid); file->f_mode |= FMODE_CREATED; out: diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index b739e02f5ef7..97e23b4e6982 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -20,6 +20,7 @@ #include <linux/slab.h> #include <linux/statfs.h> #include <linux/magic.h> +#include <linux/fscache.h> #include <net/9p/9p.h> #include <net/9p/client.h> @@ -309,6 +310,7 @@ static int v9fs_write_inode(struct inode *inode, __mark_inode_dirty(inode, I_DIRTY_DATASYNC); return ret; } + fscache_unpin_writeback(wbc, v9fs_inode_cookie(v9inode)); return 0; } @@ -332,6 +334,7 @@ static int v9fs_write_inode_dotl(struct inode *inode, __mark_inode_dirty(inode, I_DIRTY_DATASYNC); return ret; } + fscache_unpin_writeback(wbc, v9fs_inode_cookie(v9inode)); return 0; } diff --git a/fs/afs/Makefile b/fs/afs/Makefile index 75c4e4043d1d..e8956b65d7ff 100644 --- a/fs/afs/Makefile +++ b/fs/afs/Makefile @@ -3,10 +3,7 @@ # Makefile for Red Hat Linux AFS client. # -afs-cache-$(CONFIG_AFS_FSCACHE) := cache.o - kafs-y := \ - $(afs-cache-y) \ addr_list.o \ callback.o \ cell.o \ diff --git a/fs/afs/cache.c b/fs/afs/cache.c deleted file mode 100644 index 037af93e3aba..000000000000 --- a/fs/afs/cache.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* AFS caching stuff - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include <linux/sched.h> -#include "internal.h" - -static enum fscache_checkaux afs_vnode_cache_check_aux(void *cookie_netfs_data, - const void *buffer, - uint16_t buflen, - loff_t object_size); - -struct fscache_netfs afs_cache_netfs = { - .name = "afs", - .version = 2, -}; - -struct fscache_cookie_def afs_cell_cache_index_def = { - .name = "AFS.cell", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -struct fscache_cookie_def afs_volume_cache_index_def = { - .name = "AFS.volume", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -struct fscache_cookie_def afs_vnode_cache_index_def = { - .name = "AFS.vnode", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .check_aux = afs_vnode_cache_check_aux, -}; - -/* - * check that the auxiliary data indicates that the entry is still valid - */ -static enum fscache_checkaux afs_vnode_cache_check_aux(void *cookie_netfs_data, - const void *buffer, - uint16_t buflen, - loff_t object_size) -{ - struct afs_vnode *vnode = cookie_netfs_data; - struct afs_vnode_cache_aux aux; - - _enter("{%llx,%x,%llx},%p,%u", - vnode->fid.vnode, vnode->fid.unique, vnode->status.data_version, - buffer, buflen); - - memcpy(&aux, buffer, sizeof(aux)); - - /* check the size of the data is what we're expecting */ - if (buflen != sizeof(aux)) { - _leave(" = OBSOLETE [len %hx != %zx]", buflen, sizeof(aux)); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - if (vnode->status.data_version != aux.data_version) { - _leave(" = OBSOLETE [vers %llx != %llx]", - aux.data_version, vnode->status.data_version); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - _leave(" = SUCCESS"); - return FSCACHE_CHECKAUX_OKAY; -} diff --git a/fs/afs/cell.c b/fs/afs/cell.c index d88407fb9bc0..07ad744eef77 100644 --- a/fs/afs/cell.c +++ b/fs/afs/cell.c @@ -680,13 +680,6 @@ static int afs_activate_cell(struct afs_net *net, struct afs_cell *cell) return ret; } -#ifdef CONFIG_AFS_FSCACHE - cell->cache = fscache_acquire_cookie(afs_cache_netfs.primary_index, - &afs_cell_cache_index_def, - cell->name, strlen(cell->name), - NULL, 0, - cell, 0, true); -#endif ret = afs_proc_cell_setup(cell); if (ret < 0) return ret; @@ -723,11 +716,6 @@ static void afs_deactivate_cell(struct afs_net *net, struct afs_cell *cell) afs_dynroot_rmdir(net, cell); mutex_unlock(&net->proc_cells_lock); -#ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(cell->cache, NULL, false); - cell->cache = NULL; -#endif - _leave(""); } diff --git a/fs/afs/file.c b/fs/afs/file.c index afe4b803f84b..720818a7c166 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -14,6 +14,7 @@ #include <linux/gfp.h> #include <linux/task_io_accounting_ops.h> #include <linux/mm.h> +#include <linux/swap.h> #include <linux/netfs.h> #include "internal.h" @@ -158,7 +159,9 @@ int afs_open(struct inode *inode, struct file *file) if (file->f_flags & O_TRUNC) set_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); - + + fscache_use_cookie(afs_vnode_cache(vnode), file->f_mode & FMODE_WRITE); + file->private_data = af; _leave(" = 0"); return 0; @@ -177,8 +180,10 @@ error: */ int afs_release(struct inode *inode, struct file *file) { + struct afs_vnode_cache_aux aux; struct afs_vnode *vnode = AFS_FS_I(inode); struct afs_file *af = file->private_data; + loff_t i_size; int ret = 0; _enter("{%llx:%llu},", vnode->fid.vid, vnode->fid.vnode); @@ -189,6 +194,15 @@ int afs_release(struct inode *inode, struct file *file) file->private_data = NULL; if (af->wb) afs_put_wb_key(af->wb); + + if ((file->f_mode & FMODE_WRITE)) { + i_size = i_size_read(&vnode->vfs_inode); + afs_set_cache_aux(vnode, &aux); + fscache_unuse_cookie(afs_vnode_cache(vnode), &aux, &i_size); + } else { + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); + } + key_put(af->key); kfree(af); afs_prune_wb_keys(vnode); @@ -354,14 +368,19 @@ static bool afs_is_cache_enabled(struct inode *inode) { struct fscache_cookie *cookie = afs_vnode_cache(AFS_FS_I(inode)); - return fscache_cookie_enabled(cookie) && !hlist_empty(&cookie->backing_objects); + return fscache_cookie_enabled(cookie) && cookie->cache_priv; } static int afs_begin_cache_operation(struct netfs_read_request *rreq) { +#ifdef CONFIG_AFS_FSCACHE struct afs_vnode *vnode = AFS_FS_I(rreq->inode); - return fscache_begin_read_operation(rreq, afs_vnode_cache(vnode)); + return fscache_begin_read_operation(&rreq->cache_resources, + afs_vnode_cache(vnode)); +#else + return -ENOBUFS; +#endif } static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len, @@ -398,6 +417,12 @@ static void afs_readahead(struct readahead_control *ractl) netfs_readahead(ractl, &afs_req_ops, NULL); } +int afs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + fscache_unpin_writeback(wbc, afs_vnode_cache(AFS_FS_I(inode))); + return 0; +} + /* * Adjust the dirty region of the page on truncation or full invalidation, * getting rid of the markers altogether if the region is entirely invalidated. @@ -480,23 +505,24 @@ static void afs_invalidatepage(struct page *page, unsigned int offset, * release a page and clean up its private state if it's not busy * - return true if the page can now be released, false if not */ -static int afs_releasepage(struct page *page, gfp_t gfp_flags) +static int afs_releasepage(struct page *page, gfp_t gfp) { struct folio *folio = page_folio(page); struct afs_vnode *vnode = AFS_FS_I(folio_inode(folio)); _enter("{{%llx:%llu}[%lu],%lx},%x", vnode->fid.vid, vnode->fid.vnode, folio_index(folio), folio->flags, - gfp_flags); + gfp); /* deny if page is being written to the cache and the caller hasn't * elected to wait */ #ifdef CONFIG_AFS_FSCACHE if (folio_test_fscache(folio)) { - if (!(gfp_flags & __GFP_DIRECT_RECLAIM) || !(gfp_flags & __GFP_FS)) + if (current_is_kswapd() || !(gfp & __GFP_FS)) return false; folio_wait_fscache(folio); } + fscache_note_page_release(afs_vnode_cache(vnode)); #endif if (folio_test_private(folio)) { diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 16906eb592d9..5964f8aee090 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -413,9 +413,9 @@ static void afs_get_inode_cache(struct afs_vnode *vnode) { #ifdef CONFIG_AFS_FSCACHE struct { - u32 vnode_id; - u32 unique; - u32 vnode_id_ext[2]; /* Allow for a 96-bit key */ + __be32 vnode_id; + __be32 unique; + __be32 vnode_id_ext[2]; /* Allow for a 96-bit key */ } __packed key; struct afs_vnode_cache_aux aux; @@ -424,17 +424,18 @@ static void afs_get_inode_cache(struct afs_vnode *vnode) return; } - key.vnode_id = vnode->fid.vnode; - key.unique = vnode->fid.unique; - key.vnode_id_ext[0] = vnode->fid.vnode >> 32; - key.vnode_id_ext[1] = vnode->fid.vnode_hi; - aux.data_version = vnode->status.data_version; - - vnode->cache = fscache_acquire_cookie(vnode->volume->cache, - &afs_vnode_cache_index_def, - &key, sizeof(key), - &aux, sizeof(aux), - vnode, vnode->status.size, true); + key.vnode_id = htonl(vnode->fid.vnode); + key.unique = htonl(vnode->fid.unique); + key.vnode_id_ext[0] = htonl(vnode->fid.vnode >> 32); + key.vnode_id_ext[1] = htonl(vnode->fid.vnode_hi); + afs_set_cache_aux(vnode, &aux); + + vnode->cache = fscache_acquire_cookie( + vnode->volume->cache, + vnode->status.type == AFS_FTYPE_FILE ? 0 : FSCACHE_ADV_SINGLE_CHUNK, + &key, sizeof(key), + &aux, sizeof(aux), + vnode->status.size); #endif } @@ -563,9 +564,7 @@ static void afs_zap_data(struct afs_vnode *vnode) { _enter("{%llx:%llu}", vnode->fid.vid, vnode->fid.vnode); -#ifdef CONFIG_AFS_FSCACHE - fscache_invalidate(vnode->cache); -#endif + afs_invalidate_cache(vnode, 0); /* nuke all the non-dirty pages that aren't locked, mapped or being * written back in a regular file and completely discard the pages in a @@ -762,9 +761,8 @@ int afs_drop_inode(struct inode *inode) */ void afs_evict_inode(struct inode *inode) { - struct afs_vnode *vnode; - - vnode = AFS_FS_I(inode); + struct afs_vnode_cache_aux aux; + struct afs_vnode *vnode = AFS_FS_I(inode); _enter("{%llx:%llu.%d}", vnode->fid.vid, @@ -776,6 +774,9 @@ void afs_evict_inode(struct inode *inode) ASSERTCMP(inode->i_ino, ==, vnode->fid.vnode); truncate_inode_pages_final(&inode->i_data); + + afs_set_cache_aux(vnode, &aux); + fscache_clear_inode_writeback(afs_vnode_cache(vnode), inode, &aux); clear_inode(inode); while (!list_empty(&vnode->wb_keys)) { @@ -786,14 +787,9 @@ void afs_evict_inode(struct inode *inode) } #ifdef CONFIG_AFS_FSCACHE - { - struct afs_vnode_cache_aux aux; - - aux.data_version = vnode->status.data_version; - fscache_relinquish_cookie(vnode->cache, &aux, - test_bit(AFS_VNODE_DELETED, &vnode->flags)); - vnode->cache = NULL; - } + fscache_relinquish_cookie(vnode->cache, + test_bit(AFS_VNODE_DELETED, &vnode->flags)); + vnode->cache = NULL; #endif afs_prune_wb_keys(vnode); @@ -833,6 +829,9 @@ static void afs_setattr_edit_file(struct afs_operation *op) if (size < i_size) truncate_pagecache(inode, size); + if (size != i_size) + fscache_resize_cookie(afs_vnode_cache(vp->vnode), + vp->scb.status.size); } } @@ -849,40 +848,67 @@ static const struct afs_operation_ops afs_setattr_operation = { int afs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *attr) { + const unsigned int supported = + ATTR_SIZE | ATTR_MODE | ATTR_UID | ATTR_GID | + ATTR_MTIME | ATTR_MTIME_SET | ATTR_TIMES_SET | ATTR_TOUCH; struct afs_operation *op; struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); + struct inode *inode = &vnode->vfs_inode; + loff_t i_size; int ret; _enter("{%llx:%llu},{n=%pd},%x", vnode->fid.vid, vnode->fid.vnode, dentry, attr->ia_valid); - if (!(attr->ia_valid & (ATTR_SIZE | ATTR_MODE | ATTR_UID | ATTR_GID | - ATTR_MTIME | ATTR_MTIME_SET | ATTR_TIMES_SET | - ATTR_TOUCH))) { + if (!(attr->ia_valid & supported)) { _leave(" = 0 [unsupported]"); return 0; } + i_size = i_size_read(inode); if (attr->ia_valid & ATTR_SIZE) { - if (!S_ISREG(vnode->vfs_inode.i_mode)) + if (!S_ISREG(inode->i_mode)) return -EISDIR; - ret = inode_newsize_ok(&vnode->vfs_inode, attr->ia_size); + ret = inode_newsize_ok(inode, attr->ia_size); if (ret) return ret; - if (attr->ia_size == i_size_read(&vnode->vfs_inode)) + if (attr->ia_size == i_size) attr->ia_valid &= ~ATTR_SIZE; } - /* flush any dirty data outstanding on a regular file */ - if (S_ISREG(vnode->vfs_inode.i_mode)) - filemap_write_and_wait(vnode->vfs_inode.i_mapping); + fscache_use_cookie(afs_vnode_cache(vnode), true); /* Prevent any new writebacks from starting whilst we do this. */ down_write(&vnode->validate_lock); + if ((attr->ia_valid & ATTR_SIZE) && S_ISREG(inode->i_mode)) { + loff_t size = attr->ia_size; + + /* Wait for any outstanding writes to the server to complete */ + loff_t from = min(size, i_size); + loff_t to = max(size, i_size); + ret = filemap_fdatawait_range(inode->i_mapping, from, to); + if (ret < 0) + goto out_unlock; + + /* Don't talk to the server if we're just shortening in-memory + * writes that haven't gone to the server yet. + */ + if (!(attr->ia_valid & (supported & ~ATTR_SIZE & ~ATTR_MTIME)) && + attr->ia_size < i_size && + attr->ia_size > vnode->status.size) { + truncate_pagecache(inode, attr->ia_size); + fscache_resize_cookie(afs_vnode_cache(vnode), + attr->ia_size); + i_size_write(inode, attr->ia_size); + ret = 0; + goto out_unlock; + } + } + op = afs_alloc_operation(((attr->ia_valid & ATTR_FILE) ? afs_file_key(attr->ia_file) : NULL), vnode->volume); @@ -907,6 +933,7 @@ int afs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, out_unlock: up_write(&vnode->validate_lock); + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); _leave(" = %d", ret); return ret; } diff --git a/fs/afs/internal.h b/fs/afs/internal.h index aa4c0d6c9780..b6f02321fc09 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -14,7 +14,6 @@ #include <linux/key.h> #include <linux/workqueue.h> #include <linux/sched.h> -#define FSCACHE_USE_NEW_IO_API #include <linux/fscache.h> #include <linux/backing-dev.h> #include <linux/uuid.h> @@ -364,9 +363,6 @@ struct afs_cell { struct key *anonymous_key; /* anonymous user key for this cell */ struct work_struct manager; /* Manager for init/deinit/dns */ struct hlist_node proc_link; /* /proc cell list link */ -#ifdef CONFIG_AFS_FSCACHE - struct fscache_cookie *cache; /* caching cookie */ -#endif time64_t dns_expiry; /* Time AFSDB/SRV record expires */ time64_t last_inactive; /* Time of last drop of usage count */ atomic_t ref; /* Struct refcount */ @@ -590,7 +586,7 @@ struct afs_volume { #define AFS_VOLUME_BUSY 5 /* - T if volume busy notice given */ #define AFS_VOLUME_MAYBE_NO_IBULK 6 /* - T if some servers don't have InlineBulkStatus */ #ifdef CONFIG_AFS_FSCACHE - struct fscache_cookie *cache; /* caching cookie */ + struct fscache_volume *cache; /* Caching cookie */ #endif struct afs_server_list __rcu *servers; /* List of servers on which volume resides */ rwlock_t servers_lock; /* Lock for ->servers */ @@ -872,9 +868,24 @@ struct afs_operation { * Cache auxiliary data. */ struct afs_vnode_cache_aux { - u64 data_version; + __be64 data_version; } __packed; +static inline void afs_set_cache_aux(struct afs_vnode *vnode, + struct afs_vnode_cache_aux *aux) +{ + aux->data_version = cpu_to_be64(vnode->status.data_version); +} + +static inline void afs_invalidate_cache(struct afs_vnode *vnode, unsigned int flags) +{ + struct afs_vnode_cache_aux aux; + + afs_set_cache_aux(vnode, &aux); + fscache_invalidate(afs_vnode_cache(vnode), &aux, + i_size_read(&vnode->vfs_inode), flags); +} + /* * We use folio->private to hold the amount of the folio that we've written to, * splitting the field into two parts. However, we need to represent a range @@ -962,13 +973,6 @@ extern void afs_merge_fs_addr6(struct afs_addr_list *, __be32 *, u16); */ #ifdef CONFIG_AFS_FSCACHE extern struct fscache_netfs afs_cache_netfs; -extern struct fscache_cookie_def afs_cell_cache_index_def; -extern struct fscache_cookie_def afs_volume_cache_index_def; -extern struct fscache_cookie_def afs_vnode_cache_index_def; -#else -#define afs_cell_cache_index_def (*(struct fscache_cookie_def *) NULL) -#define afs_volume_cache_index_def (*(struct fscache_cookie_def *) NULL) -#define afs_vnode_cache_index_def (*(struct fscache_cookie_def *) NULL) #endif /* @@ -1068,6 +1072,7 @@ extern int afs_release(struct inode *, struct file *); extern int afs_fetch_data(struct afs_vnode *, struct afs_read *); extern struct afs_read *afs_alloc_read(gfp_t); extern void afs_put_read(struct afs_read *); +extern int afs_write_inode(struct inode *, struct writeback_control *); static inline struct afs_read *afs_get_read(struct afs_read *req) { @@ -1506,7 +1511,7 @@ extern struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *, * volume.c */ extern struct afs_volume *afs_create_volume(struct afs_fs_context *); -extern void afs_activate_volume(struct afs_volume *); +extern int afs_activate_volume(struct afs_volume *); extern void afs_deactivate_volume(struct afs_volume *); extern struct afs_volume *afs_get_volume(struct afs_volume *, enum afs_volume_trace); extern void afs_put_volume(struct afs_net *, struct afs_volume *, enum afs_volume_trace); @@ -1515,7 +1520,11 @@ extern int afs_check_volume_status(struct afs_volume *, struct afs_operation *); /* * write.c */ +#ifdef CONFIG_AFS_FSCACHE extern int afs_set_page_dirty(struct page *); +#else +#define afs_set_page_dirty __set_page_dirty_nobuffers +#endif extern int afs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); diff --git a/fs/afs/main.c b/fs/afs/main.c index 179004b15566..eae288c8d40a 100644 --- a/fs/afs/main.c +++ b/fs/afs/main.c @@ -186,13 +186,6 @@ static int __init afs_init(void) if (!afs_lock_manager) goto error_lockmgr; -#ifdef CONFIG_AFS_FSCACHE - /* we want to be able to cache */ - ret = fscache_register_netfs(&afs_cache_netfs); - if (ret < 0) - goto error_cache; -#endif - ret = register_pernet_device(&afs_net_ops); if (ret < 0) goto error_net; @@ -215,10 +208,6 @@ error_proc: error_fs: unregister_pernet_device(&afs_net_ops); error_net: -#ifdef CONFIG_AFS_FSCACHE - fscache_unregister_netfs(&afs_cache_netfs); -error_cache: -#endif destroy_workqueue(afs_lock_manager); error_lockmgr: destroy_workqueue(afs_async_calls); @@ -245,9 +234,6 @@ static void __exit afs_exit(void) proc_remove(afs_proc_symlink); afs_fs_exit(); unregister_pernet_device(&afs_net_ops); -#ifdef CONFIG_AFS_FSCACHE - fscache_unregister_netfs(&afs_cache_netfs); -#endif destroy_workqueue(afs_lock_manager); destroy_workqueue(afs_async_calls); destroy_workqueue(afs_wq); diff --git a/fs/afs/super.c b/fs/afs/super.c index 34c68724c98b..5ec9fd97eccc 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -55,6 +55,7 @@ int afs_net_id; static const struct super_operations afs_super_ops = { .statfs = afs_statfs, .alloc_inode = afs_alloc_inode, + .write_inode = afs_write_inode, .drop_inode = afs_drop_inode, .destroy_inode = afs_destroy_inode, .free_inode = afs_free_inode, diff --git a/fs/afs/volume.c b/fs/afs/volume.c index f84194b791d3..94a3d247924b 100644 --- a/fs/afs/volume.c +++ b/fs/afs/volume.c @@ -268,15 +268,30 @@ void afs_put_volume(struct afs_net *net, struct afs_volume *volume, /* * Activate a volume. */ -void afs_activate_volume(struct afs_volume *volume) +int afs_activate_volume(struct afs_volume *volume) { #ifdef CONFIG_AFS_FSCACHE - volume->cache = fscache_acquire_cookie(volume->cell->cache, - &afs_volume_cache_index_def, - &volume->vid, sizeof(volume->vid), - NULL, 0, - volume, 0, true); + struct fscache_volume *vcookie; + char *name; + + name = kasprintf(GFP_KERNEL, "afs,%s,%llx", + volume->cell->name, volume->vid); + if (!name) + return -ENOMEM; + + vcookie = fscache_acquire_volume(name, NULL, NULL, 0); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(name); + return PTR_ERR(vcookie); + } + pr_err("AFS: Cache volume key already in use (%s)\n", name); + vcookie = NULL; + } + volume->cache = vcookie; + kfree(name); #endif + return 0; } /* @@ -287,7 +302,7 @@ void afs_deactivate_volume(struct afs_volume *volume) _enter("%s", volume->name); #ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(volume->cache, NULL, + fscache_relinquish_volume(volume->cache, NULL, test_bit(AFS_VOLUME_DELETED, &volume->flags)); volume->cache = NULL; #endif diff --git a/fs/afs/write.c b/fs/afs/write.c index ca4909baf5e6..5e9157d0da29 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -12,17 +12,30 @@ #include <linux/writeback.h> #include <linux/pagevec.h> #include <linux/netfs.h> -#include <linux/fscache.h> #include "internal.h" +static void afs_write_to_cache(struct afs_vnode *vnode, loff_t start, size_t len, + loff_t i_size, bool caching); + +#ifdef CONFIG_AFS_FSCACHE /* - * mark a page as having been made dirty and thus needing writeback + * Mark a page as having been made dirty and thus needing writeback. We also + * need to pin the cache object to write back to. */ int afs_set_page_dirty(struct page *page) { - _enter(""); - return __set_page_dirty_nobuffers(page); + return fscache_set_page_dirty(page, afs_vnode_cache(AFS_FS_I(page->mapping->host))); +} +static void afs_folio_start_fscache(bool caching, struct folio *folio) +{ + if (caching) + folio_start_fscache(folio); +} +#else +static void afs_folio_start_fscache(bool caching, struct folio *folio) +{ } +#endif /* * prepare to perform part of a write to a page @@ -114,7 +127,7 @@ int afs_write_end(struct file *file, struct address_space *mapping, unsigned long priv; unsigned int f, from = offset_in_folio(folio, pos); unsigned int t, to = from + copied; - loff_t i_size, maybe_i_size; + loff_t i_size, write_end_pos; _enter("{%llx:%llu},{%lx}", vnode->fid.vid, vnode->fid.vnode, folio_index(folio)); @@ -131,15 +144,16 @@ int afs_write_end(struct file *file, struct address_space *mapping, if (copied == 0) goto out; - maybe_i_size = pos + copied; + write_end_pos = pos + copied; i_size = i_size_read(&vnode->vfs_inode); - if (maybe_i_size > i_size) { + if (write_end_pos > i_size) { write_seqlock(&vnode->cb_lock); i_size = i_size_read(&vnode->vfs_inode); - if (maybe_i_size > i_size) - afs_set_i_size(vnode, maybe_i_size); + if (write_end_pos > i_size) + afs_set_i_size(vnode, write_end_pos); write_sequnlock(&vnode->cb_lock); + fscache_update_cookie(afs_vnode_cache(vnode), NULL, &write_end_pos); } if (folio_test_private(folio)) { @@ -418,6 +432,7 @@ static void afs_extend_writeback(struct address_space *mapping, loff_t start, loff_t max_len, bool new_content, + bool caching, unsigned int *_len) { struct pagevec pvec; @@ -464,7 +479,9 @@ static void afs_extend_writeback(struct address_space *mapping, folio_put(folio); break; } - if (!folio_test_dirty(folio) || folio_test_writeback(folio)) { + if (!folio_test_dirty(folio) || + folio_test_writeback(folio) || + folio_test_fscache(folio)) { folio_unlock(folio); folio_put(folio); break; @@ -512,6 +529,7 @@ static void afs_extend_writeback(struct address_space *mapping, BUG(); if (folio_start_writeback(folio)) BUG(); + afs_folio_start_fscache(caching, folio); *_count -= folio_nr_pages(folio); folio_unlock(folio); @@ -539,6 +557,7 @@ static ssize_t afs_write_back_from_locked_folio(struct address_space *mapping, unsigned int offset, to, len, max_len; loff_t i_size = i_size_read(&vnode->vfs_inode); bool new_content = test_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); + bool caching = fscache_cookie_enabled(afs_vnode_cache(vnode)); long count = wbc->nr_to_write; int ret; @@ -546,6 +565,7 @@ static ssize_t afs_write_back_from_locked_folio(struct address_space *mapping, if (folio_start_writeback(folio)) BUG(); + afs_folio_start_fscache(caching, folio); count -= folio_nr_pages(folio); @@ -572,7 +592,8 @@ static ssize_t afs_write_back_from_locked_folio(struct address_space *mapping, if (len < max_len && (to == folio_size(folio) || new_content)) afs_extend_writeback(mapping, vnode, &count, - start, max_len, new_content, &len); + start, max_len, new_content, + caching, &len); len = min_t(loff_t, len, max_len); } @@ -585,12 +606,19 @@ static ssize_t afs_write_back_from_locked_folio(struct address_space *mapping, if (start < i_size) { _debug("write back %x @%llx [%llx]", len, start, i_size); + /* Speculatively write to the cache. We have to fix this up + * later if the store fails. + */ + afs_write_to_cache(vnode, start, len, i_size, caching); + iov_iter_xarray(&iter, WRITE, &mapping->i_pages, start, len); ret = afs_store_data(vnode, &iter, start, false); } else { _debug("write discard %x @%llx [%llx]", len, start, i_size); /* The dirty region was entirely beyond the EOF. */ + fscache_clear_page_bits(afs_vnode_cache(vnode), + mapping, start, len, caching); afs_pages_written_back(vnode, start, len); ret = 0; } @@ -649,6 +677,10 @@ int afs_writepage(struct page *subpage, struct writeback_control *wbc) _enter("{%lx},", folio_index(folio)); +#ifdef CONFIG_AFS_FSCACHE + folio_wait_fscache(folio); +#endif + start = folio_index(folio) * PAGE_SIZE; ret = afs_write_back_from_locked_folio(folio_mapping(folio), wbc, folio, start, LLONG_MAX - start); @@ -714,10 +746,15 @@ static int afs_writepages_region(struct address_space *mapping, continue; } - if (folio_test_writeback(folio)) { + if (folio_test_writeback(folio) || + folio_test_fscache(folio)) { folio_unlock(folio); - if (wbc->sync_mode != WB_SYNC_NONE) + if (wbc->sync_mode != WB_SYNC_NONE) { folio_wait_writeback(folio); +#ifdef CONFIG_AFS_FSCACHE + folio_wait_fscache(folio); +#endif + } folio_put(folio); continue; } @@ -970,3 +1007,28 @@ int afs_launder_page(struct page *subpage) folio_wait_fscache(folio); return ret; } + +/* + * Deal with the completion of writing the data to the cache. + */ +static void afs_write_to_cache_done(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct afs_vnode *vnode = priv; + + if (IS_ERR_VALUE(transferred_or_error) && + transferred_or_error != -ENOBUFS) + afs_invalidate_cache(vnode, 0); +} + +/* + * Save the write to the cache also. + */ +static void afs_write_to_cache(struct afs_vnode *vnode, + loff_t start, size_t len, loff_t i_size, + bool caching) +{ + fscache_write_to_cache(afs_vnode_cache(vnode), + vnode->vfs_inode.i_mapping, start, len, i_size, + afs_write_to_cache_done, vnode, caching); +} diff --git a/fs/cachefiles/Kconfig b/fs/cachefiles/Kconfig index 6827b40f7ddc..719faeeda168 100644 --- a/fs/cachefiles/Kconfig +++ b/fs/cachefiles/Kconfig @@ -19,3 +19,10 @@ config CACHEFILES_DEBUG caching on files module. If this is set, the debugging output may be enabled by setting bits in /sys/modules/cachefiles/parameter/debug or by including a debugging specifier in /etc/cachefilesd.conf. + +config CACHEFILES_ERROR_INJECTION + bool "Provide error injection for cachefiles" + depends on CACHEFILES && SYSCTL + help + This permits error injection to be enabled in cachefiles whilst a + cache is in service. diff --git a/fs/cachefiles/Makefile b/fs/cachefiles/Makefile index 02fd17731769..16d811f1a2fa 100644 --- a/fs/cachefiles/Makefile +++ b/fs/cachefiles/Makefile @@ -4,15 +4,17 @@ # cachefiles-y := \ - bind.o \ + cache.o \ daemon.o \ interface.o \ io.o \ key.o \ main.o \ namei.o \ - rdwr.o \ security.o \ + volume.o \ xattr.o +cachefiles-$(CONFIG_CACHEFILES_ERROR_INJECTION) += error_inject.o + obj-$(CONFIG_CACHEFILES) := cachefiles.o diff --git a/fs/cachefiles/bind.c b/fs/cachefiles/bind.c deleted file mode 100644 index 146291be6263..000000000000 --- a/fs/cachefiles/bind.c +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Bind and unbind a cache from the filesystem backing it - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include <linux/module.h> -#include <linux/init.h> -#include <linux/sched.h> -#include <linux/completion.h> -#include <linux/slab.h> -#include <linux/fs.h> -#include <linux/file.h> -#include <linux/namei.h> -#include <linux/mount.h> -#include <linux/statfs.h> -#include <linux/ctype.h> -#include <linux/xattr.h> -#include "internal.h" - -static int cachefiles_daemon_add_cache(struct cachefiles_cache *caches); - -/* - * bind a directory as a cache - */ -int cachefiles_daemon_bind(struct cachefiles_cache *cache, char *args) -{ - _enter("{%u,%u,%u,%u,%u,%u},%s", - cache->frun_percent, - cache->fcull_percent, - cache->fstop_percent, - cache->brun_percent, - cache->bcull_percent, - cache->bstop_percent, - args); - - /* start by checking things over */ - ASSERT(cache->fstop_percent >= 0 && - cache->fstop_percent < cache->fcull_percent && - cache->fcull_percent < cache->frun_percent && - cache->frun_percent < 100); - - ASSERT(cache->bstop_percent >= 0 && - cache->bstop_percent < cache->bcull_percent && - cache->bcull_percent < cache->brun_percent && - cache->brun_percent < 100); - - if (*args) { - pr_err("'bind' command doesn't take an argument\n"); - return -EINVAL; - } - - if (!cache->rootdirname) { - pr_err("No cache directory specified\n"); - return -EINVAL; - } - - /* don't permit already bound caches to be re-bound */ - if (test_bit(CACHEFILES_READY, &cache->flags)) { - pr_err("Cache already bound\n"); - return -EBUSY; - } - - /* make sure we have copies of the tag and dirname strings */ - if (!cache->tag) { - /* the tag string is released by the fops->release() - * function, so we don't release it on error here */ - cache->tag = kstrdup("CacheFiles", GFP_KERNEL); - if (!cache->tag) - return -ENOMEM; - } - - /* add the cache */ - return cachefiles_daemon_add_cache(cache); -} - -/* - * add a cache - */ -static int cachefiles_daemon_add_cache(struct cachefiles_cache *cache) -{ - struct cachefiles_object *fsdef; - struct path path; - struct kstatfs stats; - struct dentry *graveyard, *cachedir, *root; - const struct cred *saved_cred; - int ret; - - _enter(""); - - /* we want to work under the module's security ID */ - ret = cachefiles_get_security_ID(cache); - if (ret < 0) - return ret; - - cachefiles_begin_secure(cache, &saved_cred); - - /* allocate the root index object */ - ret = -ENOMEM; - - fsdef = kmem_cache_alloc(cachefiles_object_jar, GFP_KERNEL); - if (!fsdef) - goto error_root_object; - - ASSERTCMP(fsdef->backer, ==, NULL); - - atomic_set(&fsdef->usage, 1); - fsdef->type = FSCACHE_COOKIE_TYPE_INDEX; - - /* look up the directory at the root of the cache */ - ret = kern_path(cache->rootdirname, LOOKUP_DIRECTORY, &path); - if (ret < 0) - goto error_open_root; - - cache->mnt = path.mnt; - root = path.dentry; - - ret = -EINVAL; - if (is_idmapped_mnt(path.mnt)) { - pr_warn("File cache on idmapped mounts not supported"); - goto error_unsupported; - } - - /* check parameters */ - ret = -EOPNOTSUPP; - if (d_is_negative(root) || - !d_backing_inode(root)->i_op->lookup || - !d_backing_inode(root)->i_op->mkdir || - !(d_backing_inode(root)->i_opflags & IOP_XATTR) || - !root->d_sb->s_op->statfs || - !root->d_sb->s_op->sync_fs) - goto error_unsupported; - - ret = -EROFS; - if (sb_rdonly(root->d_sb)) - goto error_unsupported; - - /* determine the security of the on-disk cache as this governs - * security ID of files we create */ - ret = cachefiles_determine_cache_security(cache, root, &saved_cred); - if (ret < 0) - goto error_unsupported; - - /* get the cache size and blocksize */ - ret = vfs_statfs(&path, &stats); - if (ret < 0) - goto error_unsupported; - - ret = -ERANGE; - if (stats.f_bsize <= 0) - goto error_unsupported; - - ret = -EOPNOTSUPP; - if (stats.f_bsize > PAGE_SIZE) - goto error_unsupported; - - cache->bsize = stats.f_bsize; - cache->bshift = 0; - if (stats.f_bsize < PAGE_SIZE) - cache->bshift = PAGE_SHIFT - ilog2(stats.f_bsize); - - _debug("blksize %u (shift %u)", - cache->bsize, cache->bshift); - - _debug("size %llu, avail %llu", - (unsigned long long) stats.f_blocks, - (unsigned long long) stats.f_bavail); - - /* set up caching limits */ - do_div(stats.f_files, 100); - cache->fstop = stats.f_files * cache->fstop_percent; - cache->fcull = stats.f_files * cache->fcull_percent; - cache->frun = stats.f_files * cache->frun_percent; - - _debug("limits {%llu,%llu,%llu} files", - (unsigned long long) cache->frun, - (unsigned long long) cache->fcull, - (unsigned long long) cache->fstop); - - stats.f_blocks >>= cache->bshift; - do_div(stats.f_blocks, 100); - cache->bstop = stats.f_blocks * cache->bstop_percent; - cache->bcull = stats.f_blocks * cache->bcull_percent; - cache->brun = stats.f_blocks * cache->brun_percent; - - _debug("limits {%llu,%llu,%llu} blocks", - (unsigned long long) cache->brun, - (unsigned long long) cache->bcull, - (unsigned long long) cache->bstop); - - /* get the cache directory and check its type */ - cachedir = cachefiles_get_directory(cache, root, "cache"); - if (IS_ERR(cachedir)) { - ret = PTR_ERR(cachedir); - goto error_unsupported; - } - - fsdef->dentry = cachedir; - fsdef->fscache.cookie = NULL; - - ret = cachefiles_check_object_type(fsdef); - if (ret < 0) - goto error_unsupported; - - /* get the graveyard directory */ - graveyard = cachefiles_get_directory(cache, root, "graveyard"); - if (IS_ERR(graveyard)) { - ret = PTR_ERR(graveyard); - goto error_unsupported; - } - - cache->graveyard = graveyard; - - /* publish the cache */ - fscache_init_cache(&cache->cache, - &cachefiles_cache_ops, - "%s", - fsdef->dentry->d_sb->s_id); - - fscache_object_init(&fsdef->fscache, &fscache_fsdef_index, - &cache->cache); - - ret = fscache_add_cache(&cache->cache, &fsdef->fscache, cache->tag); - if (ret < 0) - goto error_add_cache; - - /* done */ - set_bit(CACHEFILES_READY, &cache->flags); - dput(root); - - pr_info("File cache on %s registered\n", cache->cache.identifier); - - /* check how much space the cache has */ - cachefiles_has_space(cache, 0, 0); - cachefiles_end_secure(cache, saved_cred); - return 0; - -error_add_cache: - dput(cache->graveyard); - cache->graveyard = NULL; -error_unsupported: - mntput(cache->mnt); - cache->mnt = NULL; - dput(fsdef->dentry); - fsdef->dentry = NULL; - dput(root); -error_open_root: - kmem_cache_free(cachefiles_object_jar, fsdef); -error_root_object: - cachefiles_end_secure(cache, saved_cred); - pr_err("Failed to register: %d\n", ret); - return ret; -} - -/* - * unbind a cache on fd release - */ -void cachefiles_daemon_unbind(struct cachefiles_cache *cache) -{ - _enter(""); - - if (test_bit(CACHEFILES_READY, &cache->flags)) { - pr_info("File cache on %s unregistering\n", - cache->cache.identifier); - - fscache_withdraw_cache(&cache->cache); - } - - dput(cache->graveyard); - mntput(cache->mnt); - - kfree(cache->rootdirname); - kfree(cache->secctx); - kfree(cache->tag); - - _leave(""); -} diff --git a/fs/cachefiles/cache.c b/fs/cachefiles/cache.c new file mode 100644 index 000000000000..ce4d4785003c --- /dev/null +++ b/fs/cachefiles/cache.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Manage high-level VFS aspects of a cache. + * + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/slab.h> +#include <linux/statfs.h> +#include <linux/namei.h> +#include "internal.h" + +/* + * Bring a cache online. + */ +int cachefiles_add_cache(struct cachefiles_cache *cache) +{ + struct fscache_cache *cache_cookie; + struct path path; + struct kstatfs stats; + struct dentry *graveyard, *cachedir, *root; + const struct cred *saved_cred; + int ret; + + _enter(""); + + cache_cookie = fscache_acquire_cache(cache->tag); + if (IS_ERR(cache_cookie)) + return PTR_ERR(cache_cookie); + + /* we want to work under the module's security ID */ + ret = cachefiles_get_security_ID(cache); + if (ret < 0) + goto error_getsec; + + cachefiles_begin_secure(cache, &saved_cred); + + /* look up the directory at the root of the cache */ + ret = kern_path(cache->rootdirname, LOOKUP_DIRECTORY, &path); + if (ret < 0) + goto error_open_root; + + cache->mnt = path.mnt; + root = path.dentry; + + ret = -EINVAL; + if (is_idmapped_mnt(path.mnt)) { + pr_warn("File cache on idmapped mounts not supported"); + goto error_unsupported; + } + + /* check parameters */ + ret = -EOPNOTSUPP; + if (d_is_negative(root) || + !d_backing_inode(root)->i_op->lookup || + !d_backing_inode(root)->i_op->mkdir || + !(d_backing_inode(root)->i_opflags & IOP_XATTR) || + !root->d_sb->s_op->statfs || + !root->d_sb->s_op->sync_fs || + root->d_sb->s_blocksize > PAGE_SIZE) + goto error_unsupported; + + ret = -EROFS; + if (sb_rdonly(root->d_sb)) + goto error_unsupported; + + /* determine the security of the on-disk cache as this governs + * security ID of files we create */ + ret = cachefiles_determine_cache_security(cache, root, &saved_cred); + if (ret < 0) + goto error_unsupported; + + /* get the cache size and blocksize */ + ret = vfs_statfs(&path, &stats); + if (ret < 0) + goto error_unsupported; + + ret = -ERANGE; + if (stats.f_bsize <= 0) + goto error_unsupported; + + ret = -EOPNOTSUPP; + if (stats.f_bsize > PAGE_SIZE) + goto error_unsupported; + + cache->bsize = stats.f_bsize; + cache->bshift = 0; + if (stats.f_bsize < PAGE_SIZE) + cache->bshift = PAGE_SHIFT - ilog2(stats.f_bsize); + + _debug("blksize %u (shift %u)", + cache->bsize, cache->bshift); + + _debug("size %llu, avail %llu", + (unsigned long long) stats.f_blocks, + (unsigned long long) stats.f_bavail); + + /* set up caching limits */ + do_div(stats.f_files, 100); + cache->fstop = stats.f_files * cache->fstop_percent; + cache->fcull = stats.f_files * cache->fcull_percent; + cache->frun = stats.f_files * cache->frun_percent; + + _debug("limits {%llu,%llu,%llu} files", + (unsigned long long) cache->frun, + (unsigned long long) cache->fcull, + (unsigned long long) cache->fstop); + + stats.f_blocks >>= cache->bshift; + do_div(stats.f_blocks, 100); + cache->bstop = stats.f_blocks * cache->bstop_percent; + cache->bcull = stats.f_blocks * cache->bcull_percent; + cache->brun = stats.f_blocks * cache->brun_percent; + + _debug("limits {%llu,%llu,%llu} blocks", + (unsigned long long) cache->brun, + (unsigned long long) cache->bcull, + (unsigned long long) cache->bstop); + + /* get the cache directory and check its type */ + cachedir = cachefiles_get_directory(cache, root, "cache", NULL); + if (IS_ERR(cachedir)) { + ret = PTR_ERR(cachedir); + goto error_unsupported; + } + + cache->store = cachedir; + + /* get the graveyard directory */ + graveyard = cachefiles_get_directory(cache, root, "graveyard", NULL); + if (IS_ERR(graveyard)) { + ret = PTR_ERR(graveyard); + goto error_unsupported; + } + + cache->graveyard = graveyard; + cache->cache = cache_cookie; + + ret = fscache_add_cache(cache_cookie, &cachefiles_cache_ops, cache); + if (ret < 0) + goto error_add_cache; + + /* done */ + set_bit(CACHEFILES_READY, &cache->flags); + dput(root); + + pr_info("File cache on %s registered\n", cache_cookie->name); + + /* check how much space the cache has */ + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); + cachefiles_end_secure(cache, saved_cred); + _leave(" = 0 [%px]", cache->cache); + return 0; + +error_add_cache: + cachefiles_put_directory(cache->graveyard); + cache->graveyard = NULL; +error_unsupported: + cachefiles_put_directory(cache->store); + cache->store = NULL; + mntput(cache->mnt); + cache->mnt = NULL; + dput(root); +error_open_root: + cachefiles_end_secure(cache, saved_cred); +error_getsec: + fscache_relinquish_cache(cache_cookie); + cache->cache = NULL; + pr_err("Failed to register: %d\n", ret); + return ret; +} + +/* + * See if we have space for a number of pages and/or a number of files in the + * cache + */ +int cachefiles_has_space(struct cachefiles_cache *cache, + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason) +{ + struct kstatfs stats; + u64 b_avail, b_writing; + int ret; + + struct path path = { + .mnt = cache->mnt, + .dentry = cache->mnt->mnt_root, + }; + + //_enter("{%llu,%llu,%llu,%llu,%llu,%llu},%u,%u", + // (unsigned long long) cache->frun, + // (unsigned long long) cache->fcull, + // (unsigned long long) cache->fstop, + // (unsigned long long) cache->brun, + // (unsigned long long) cache->bcull, + // (unsigned long long) cache->bstop, + // fnr, bnr); + + /* find out how many pages of blockdev are available */ + memset(&stats, 0, sizeof(stats)); + + ret = vfs_statfs(&path, &stats); + if (ret < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(path.dentry), ret, + cachefiles_trace_statfs_error); + if (ret == -EIO) + cachefiles_io_error(cache, "statfs failed"); + _leave(" = %d", ret); + return ret; + } + + b_avail = stats.f_bavail >> cache->bshift; + b_writing = atomic_long_read(&cache->b_writing); + if (b_avail > b_writing) + b_avail -= b_writing; + else + b_avail = 0; + + //_debug("avail %llu,%llu", + // (unsigned long long)stats.f_ffree, + // (unsigned long long)b_avail); + + /* see if there is sufficient space */ + if (stats.f_ffree > fnr) + stats.f_ffree -= fnr; + else + stats.f_ffree = 0; + + if (b_avail > bnr) + b_avail -= bnr; + else + b_avail = 0; + + ret = -ENOBUFS; + if (stats.f_ffree < cache->fstop || + b_avail < cache->bstop) + goto stop_and_begin_cull; + + ret = 0; + if (stats.f_ffree < cache->fcull || + b_avail < cache->bcull) + goto begin_cull; + + if (test_bit(CACHEFILES_CULLING, &cache->flags) && + stats.f_ffree >= cache->frun && + b_avail >= cache->brun && + test_and_clear_bit(CACHEFILES_CULLING, &cache->flags) + ) { + _debug("cease culling"); + cachefiles_state_changed(cache); + } + + //_leave(" = 0"); + return 0; + +stop_and_begin_cull: + switch (reason) { + case cachefiles_has_space_for_write: + fscache_count_no_write_space(); + break; + case cachefiles_has_space_for_create: + fscache_count_no_create_space(); + break; + default: + break; + } +begin_cull: + if (!test_and_set_bit(CACHEFILES_CULLING, &cache->flags)) { + _debug("### CULL CACHE ###"); + cachefiles_state_changed(cache); + } + + _leave(" = %d", ret); + return ret; +} + +/* + * Mark all the objects as being out of service and queue them all for cleanup. + */ +static void cachefiles_withdraw_objects(struct cachefiles_cache *cache) +{ + struct cachefiles_object *object; + unsigned int count = 0; + + _enter(""); + + spin_lock(&cache->object_list_lock); + + while (!list_empty(&cache->object_list)) { + object = list_first_entry(&cache->object_list, + struct cachefiles_object, cache_link); + cachefiles_see_object(object, cachefiles_obj_see_withdrawal); + list_del_init(&object->cache_link); + fscache_withdraw_cookie(object->cookie); + count++; + if ((count & 63) == 0) { + spin_unlock(&cache->object_list_lock); + cond_resched(); + spin_lock(&cache->object_list_lock); + } + } + + spin_unlock(&cache->object_list_lock); + _leave(" [%u objs]", count); +} + +/* + * Withdraw volumes. + */ +static void cachefiles_withdraw_volumes(struct cachefiles_cache *cache) +{ + _enter(""); + + for (;;) { + struct cachefiles_volume *volume = NULL; + + spin_lock(&cache->object_list_lock); + if (!list_empty(&cache->volumes)) { + volume = list_first_entry(&cache->volumes, + struct cachefiles_volume, cache_link); + list_del_init(&volume->cache_link); + } + spin_unlock(&cache->object_list_lock); + if (!volume) + break; + + cachefiles_withdraw_volume(volume); + } + + _leave(""); +} + +/* + * Sync a cache to backing disk. + */ +static void cachefiles_sync_cache(struct cachefiles_cache *cache) +{ + const struct cred *saved_cred; + int ret; + + _enter("%s", cache->cache->name); + + /* make sure all pages pinned by operations on behalf of the netfs are + * written to disc */ + cachefiles_begin_secure(cache, &saved_cred); + down_read(&cache->mnt->mnt_sb->s_umount); + ret = sync_filesystem(cache->mnt->mnt_sb); + up_read(&cache->mnt->mnt_sb->s_umount); + cachefiles_end_secure(cache, saved_cred); + + if (ret == -EIO) + cachefiles_io_error(cache, + "Attempt to sync backing fs superblock returned error %d", + ret); +} + +/* + * Withdraw cache objects. + */ +void cachefiles_withdraw_cache(struct cachefiles_cache *cache) +{ + struct fscache_cache *fscache = cache->cache; + + pr_info("File cache on %s unregistering\n", fscache->name); + + fscache_withdraw_cache(fscache); + + /* we now have to destroy all the active objects pertaining to this + * cache - which we do by passing them off to thread pool to be + * disposed of */ + cachefiles_withdraw_objects(cache); + fscache_wait_for_objects(fscache); + + cachefiles_withdraw_volumes(cache); + cachefiles_sync_cache(cache); + cache->cache = NULL; + fscache_relinquish_cache(fscache); +} diff --git a/fs/cachefiles/daemon.c b/fs/cachefiles/daemon.c index 752c1e43416f..40a792421fc1 100644 --- a/fs/cachefiles/daemon.c +++ b/fs/cachefiles/daemon.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* Daemon interface * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -41,6 +41,8 @@ static int cachefiles_daemon_dir(struct cachefiles_cache *, char *); static int cachefiles_daemon_inuse(struct cachefiles_cache *, char *); static int cachefiles_daemon_secctx(struct cachefiles_cache *, char *); static int cachefiles_daemon_tag(struct cachefiles_cache *, char *); +static int cachefiles_daemon_bind(struct cachefiles_cache *, char *); +static void cachefiles_daemon_unbind(struct cachefiles_cache *); static unsigned long cachefiles_open; @@ -78,7 +80,7 @@ static const struct cachefiles_daemon_cmd cachefiles_daemon_cmds[] = { /* - * do various checks + * Prepare a cache for caching. */ static int cachefiles_daemon_open(struct inode *inode, struct file *file) { @@ -102,9 +104,10 @@ static int cachefiles_daemon_open(struct inode *inode, struct file *file) } mutex_init(&cache->daemon_mutex); - cache->active_nodes = RB_ROOT; - rwlock_init(&cache->active_lock); init_waitqueue_head(&cache->daemon_pollwq); + INIT_LIST_HEAD(&cache->volumes); + INIT_LIST_HEAD(&cache->object_list); + spin_lock_init(&cache->object_list_lock); /* set default caching limits * - limit at 1% free space and/or free files @@ -124,7 +127,7 @@ static int cachefiles_daemon_open(struct inode *inode, struct file *file) } /* - * release a cache + * Release a cache. */ static int cachefiles_daemon_release(struct inode *inode, struct file *file) { @@ -138,8 +141,6 @@ static int cachefiles_daemon_release(struct inode *inode, struct file *file) cachefiles_daemon_unbind(cache); - ASSERT(!cache->active_nodes.rb_node); - /* clean up the control file interface */ cache->cachefilesd = NULL; file->private_data = NULL; @@ -152,7 +153,7 @@ static int cachefiles_daemon_release(struct inode *inode, struct file *file) } /* - * read the cache state + * Read the cache state. */ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, size_t buflen, loff_t *pos) @@ -169,7 +170,7 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, return 0; /* check how much space the cache has */ - cachefiles_has_space(cache, 0, 0); + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); /* summarise */ f_released = atomic_xchg(&cache->f_released, 0); @@ -206,7 +207,7 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, } /* - * command the cache + * Take a command from cachefilesd, parse it and act on it. */ static ssize_t cachefiles_daemon_write(struct file *file, const char __user *_data, @@ -225,7 +226,7 @@ static ssize_t cachefiles_daemon_write(struct file *file, if (test_bit(CACHEFILES_DEAD, &cache->flags)) return -EIO; - if (datalen < 0 || datalen > PAGE_SIZE - 1) + if (datalen > PAGE_SIZE - 1) return -EOPNOTSUPP; /* drag the command string into the kernel so we can parse it */ @@ -284,7 +285,7 @@ found_command: } /* - * poll for culling state + * Poll for culling state * - use EPOLLOUT to indicate culling state */ static __poll_t cachefiles_daemon_poll(struct file *file, @@ -306,7 +307,7 @@ static __poll_t cachefiles_daemon_poll(struct file *file, } /* - * give a range error for cache space constraints + * Give a range error for cache space constraints * - can be tail-called */ static int cachefiles_daemon_range_error(struct cachefiles_cache *cache, @@ -318,7 +319,7 @@ static int cachefiles_daemon_range_error(struct cachefiles_cache *cache, } /* - * set the percentage of files at which to stop culling + * Set the percentage of files at which to stop culling * - command: "frun <N>%" */ static int cachefiles_daemon_frun(struct cachefiles_cache *cache, char *args) @@ -342,7 +343,7 @@ static int cachefiles_daemon_frun(struct cachefiles_cache *cache, char *args) } /* - * set the percentage of files at which to start culling + * Set the percentage of files at which to start culling * - command: "fcull <N>%" */ static int cachefiles_daemon_fcull(struct cachefiles_cache *cache, char *args) @@ -366,7 +367,7 @@ static int cachefiles_daemon_fcull(struct cachefiles_cache *cache, char *args) } /* - * set the percentage of files at which to stop allocating + * Set the percentage of files at which to stop allocating * - command: "fstop <N>%" */ static int cachefiles_daemon_fstop(struct cachefiles_cache *cache, char *args) @@ -382,7 +383,7 @@ static int cachefiles_daemon_fstop(struct cachefiles_cache *cache, char *args) if (args[0] != '%' || args[1] != '\0') return -EINVAL; - if (fstop < 0 || fstop >= cache->fcull_percent) + if (fstop >= cache->fcull_percent) return cachefiles_daemon_range_error(cache, args); cache->fstop_percent = fstop; @@ -390,7 +391,7 @@ static int cachefiles_daemon_fstop(struct cachefiles_cache *cache, char *args) } /* - * set the percentage of blocks at which to stop culling + * Set the percentage of blocks at which to stop culling * - command: "brun <N>%" */ static int cachefiles_daemon_brun(struct cachefiles_cache *cache, char *args) @@ -414,7 +415,7 @@ static int cachefiles_daemon_brun(struct cachefiles_cache *cache, char *args) } /* - * set the percentage of blocks at which to start culling + * Set the percentage of blocks at which to start culling * - command: "bcull <N>%" */ static int cachefiles_daemon_bcull(struct cachefiles_cache *cache, char *args) @@ -438,7 +439,7 @@ static int cachefiles_daemon_bcull(struct cachefiles_cache *cache, char *args) } /* - * set the percentage of blocks at which to stop allocating + * Set the percentage of blocks at which to stop allocating * - command: "bstop <N>%" */ static int cachefiles_daemon_bstop(struct cachefiles_cache *cache, char *args) @@ -454,7 +455,7 @@ static int cachefiles_daemon_bstop(struct cachefiles_cache *cache, char *args) if (args[0] != '%' || args[1] != '\0') return -EINVAL; - if (bstop < 0 || bstop >= cache->bcull_percent) + if (bstop >= cache->bcull_percent) return cachefiles_daemon_range_error(cache, args); cache->bstop_percent = bstop; @@ -462,7 +463,7 @@ static int cachefiles_daemon_bstop(struct cachefiles_cache *cache, char *args) } /* - * set the cache directory + * Set the cache directory * - command: "dir <name>" */ static int cachefiles_daemon_dir(struct cachefiles_cache *cache, char *args) @@ -490,7 +491,7 @@ static int cachefiles_daemon_dir(struct cachefiles_cache *cache, char *args) } /* - * set the cache security context + * Set the cache security context * - command: "secctx <ctx>" */ static int cachefiles_daemon_secctx(struct cachefiles_cache *cache, char *args) @@ -518,7 +519,7 @@ static int cachefiles_daemon_secctx(struct cachefiles_cache *cache, char *args) } /* - * set the cache tag + * Set the cache tag * - command: "tag <name>" */ static int cachefiles_daemon_tag(struct cachefiles_cache *cache, char *args) @@ -544,7 +545,7 @@ static int cachefiles_daemon_tag(struct cachefiles_cache *cache, char *args) } /* - * request a node in the cache be culled from the current working directory + * Request a node in the cache be culled from the current working directory * - command: "cull <name>" */ static int cachefiles_daemon_cull(struct cachefiles_cache *cache, char *args) @@ -568,7 +569,6 @@ static int cachefiles_daemon_cull(struct cachefiles_cache *cache, char *args) return -EIO; } - /* extract the directory dentry from the cwd */ get_fs_pwd(current->fs, &path); if (!d_can_lookup(path.dentry)) @@ -593,7 +593,7 @@ inval: } /* - * set debugging mode + * Set debugging mode * - command: "debug <mask>" */ static int cachefiles_daemon_debug(struct cachefiles_cache *cache, char *args) @@ -616,7 +616,7 @@ inval: } /* - * find out whether an object in the current working directory is in use or not + * Find out whether an object in the current working directory is in use or not * - command: "inuse <name>" */ static int cachefiles_daemon_inuse(struct cachefiles_cache *cache, char *args) @@ -640,7 +640,6 @@ static int cachefiles_daemon_inuse(struct cachefiles_cache *cache, char *args) return -EIO; } - /* extract the directory dentry from the cwd */ get_fs_pwd(current->fs, &path); if (!d_can_lookup(path.dentry)) @@ -665,84 +664,65 @@ inval: } /* - * see if we have space for a number of pages and/or a number of files in the - * cache + * Bind a directory as a cache */ -int cachefiles_has_space(struct cachefiles_cache *cache, - unsigned fnr, unsigned bnr) +static int cachefiles_daemon_bind(struct cachefiles_cache *cache, char *args) { - struct kstatfs stats; - struct path path = { - .mnt = cache->mnt, - .dentry = cache->mnt->mnt_root, - }; - int ret; + _enter("{%u,%u,%u,%u,%u,%u},%s", + cache->frun_percent, + cache->fcull_percent, + cache->fstop_percent, + cache->brun_percent, + cache->bcull_percent, + cache->bstop_percent, + args); + + if (cache->fstop_percent >= cache->fcull_percent || + cache->fcull_percent >= cache->frun_percent || + cache->frun_percent >= 100) + return -ERANGE; + + if (cache->bstop_percent >= cache->bcull_percent || + cache->bcull_percent >= cache->brun_percent || + cache->brun_percent >= 100) + return -ERANGE; - //_enter("{%llu,%llu,%llu,%llu,%llu,%llu},%u,%u", - // (unsigned long long) cache->frun, - // (unsigned long long) cache->fcull, - // (unsigned long long) cache->fstop, - // (unsigned long long) cache->brun, - // (unsigned long long) cache->bcull, - // (unsigned long long) cache->bstop, - // fnr, bnr); - - /* find out how many pages of blockdev are available */ - memset(&stats, 0, sizeof(stats)); - - ret = vfs_statfs(&path, &stats); - if (ret < 0) { - if (ret == -EIO) - cachefiles_io_error(cache, "statfs failed"); - _leave(" = %d", ret); - return ret; + if (*args) { + pr_err("'bind' command doesn't take an argument\n"); + return -EINVAL; } - stats.f_bavail >>= cache->bshift; - - //_debug("avail %llu,%llu", - // (unsigned long long) stats.f_ffree, - // (unsigned long long) stats.f_bavail); - - /* see if there is sufficient space */ - if (stats.f_ffree > fnr) - stats.f_ffree -= fnr; - else - stats.f_ffree = 0; - - if (stats.f_bavail > bnr) - stats.f_bavail -= bnr; - else - stats.f_bavail = 0; - - ret = -ENOBUFS; - if (stats.f_ffree < cache->fstop || - stats.f_bavail < cache->bstop) - goto begin_cull; - - ret = 0; - if (stats.f_ffree < cache->fcull || - stats.f_bavail < cache->bcull) - goto begin_cull; - - if (test_bit(CACHEFILES_CULLING, &cache->flags) && - stats.f_ffree >= cache->frun && - stats.f_bavail >= cache->brun && - test_and_clear_bit(CACHEFILES_CULLING, &cache->flags) - ) { - _debug("cease culling"); - cachefiles_state_changed(cache); + if (!cache->rootdirname) { + pr_err("No cache directory specified\n"); + return -EINVAL; } - //_leave(" = 0"); - return 0; - -begin_cull: - if (!test_and_set_bit(CACHEFILES_CULLING, &cache->flags)) { - _debug("### CULL CACHE ###"); - cachefiles_state_changed(cache); + /* Don't permit already bound caches to be re-bound */ + if (test_bit(CACHEFILES_READY, &cache->flags)) { + pr_err("Cache already bound\n"); + return -EBUSY; } - _leave(" = %d", ret); - return ret; + return cachefiles_add_cache(cache); +} + +/* + * Unbind a cache. + */ +static void cachefiles_daemon_unbind(struct cachefiles_cache *cache) +{ + _enter(""); + + if (test_bit(CACHEFILES_READY, &cache->flags)) + cachefiles_withdraw_cache(cache); + + cachefiles_put_directory(cache->graveyard); + cachefiles_put_directory(cache->store); + mntput(cache->mnt); + + kfree(cache->rootdirname); + kfree(cache->secctx); + kfree(cache->tag); + + _leave(""); } diff --git a/fs/cachefiles/error_inject.c b/fs/cachefiles/error_inject.c new file mode 100644 index 000000000000..58f8aec964e4 --- /dev/null +++ b/fs/cachefiles/error_inject.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Error injection handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/sysctl.h> +#include "internal.h" + +unsigned int cachefiles_error_injection_state; + +static struct ctl_table_header *cachefiles_sysctl; +static struct ctl_table cachefiles_sysctls[] = { + { + .procname = "error_injection", + .data = &cachefiles_error_injection_state, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec, + }, + {} +}; + +static struct ctl_table cachefiles_sysctls_root[] = { + { + .procname = "cachefiles", + .mode = 0555, + .child = cachefiles_sysctls, + }, + {} +}; + +int __init cachefiles_register_error_injection(void) +{ + cachefiles_sysctl = register_sysctl_table(cachefiles_sysctls_root); + if (!cachefiles_sysctl) + return -ENOMEM; + return 0; + +} + +void cachefiles_unregister_error_injection(void) +{ + unregister_sysctl_table(cachefiles_sysctl); +} diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c index da28ac1fa225..51c968cd00a6 100644 --- a/fs/cachefiles/interface.c +++ b/fs/cachefiles/interface.c @@ -1,572 +1,445 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* FS-Cache interface to CacheFiles * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ #include <linux/slab.h> #include <linux/mount.h> +#include <linux/xattr.h> +#include <linux/file.h> +#include <linux/falloc.h> +#include <trace/events/fscache.h> #include "internal.h" -struct cachefiles_lookup_data { - struct cachefiles_xattr *auxdata; /* auxiliary data */ - char *key; /* key path */ -}; - -static int cachefiles_attr_changed(struct fscache_object *_object); +static atomic_t cachefiles_object_debug_id; /* - * allocate an object record for a cookie lookup and prepare the lookup data + * Allocate a cache object record. */ -static struct fscache_object *cachefiles_alloc_object( - struct fscache_cache *_cache, - struct fscache_cookie *cookie) +static +struct cachefiles_object *cachefiles_alloc_object(struct fscache_cookie *cookie) { - struct cachefiles_lookup_data *lookup_data; + struct fscache_volume *vcookie = cookie->volume; + struct cachefiles_volume *volume = vcookie->cache_priv; struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct cachefiles_xattr *auxdata; - unsigned keylen, auxlen; - void *buffer, *p; - char *key; - cache = container_of(_cache, struct cachefiles_cache, cache); + _enter("{%s},%x,", vcookie->key, cookie->debug_id); - _enter("{%s},%x,", cache->cache.identifier, cookie->debug_id); - - lookup_data = kmalloc(sizeof(*lookup_data), cachefiles_gfp); - if (!lookup_data) - goto nomem_lookup_data; - - /* create a new object record and a temporary leaf image */ - object = kmem_cache_alloc(cachefiles_object_jar, cachefiles_gfp); + object = kmem_cache_zalloc(cachefiles_object_jar, GFP_KERNEL); if (!object) - goto nomem_object; - - ASSERTCMP(object->backer, ==, NULL); + return NULL; - BUG_ON(test_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)); - atomic_set(&object->usage, 1); + refcount_set(&object->ref, 1); - fscache_object_init(&object->fscache, cookie, &cache->cache); + spin_lock_init(&object->lock); + INIT_LIST_HEAD(&object->cache_link); + object->volume = volume; + object->debug_id = atomic_inc_return(&cachefiles_object_debug_id); + object->cookie = fscache_get_cookie(cookie, fscache_cookie_get_attach_object); - object->type = cookie->def->type; - - /* get hold of the raw key - * - stick the length on the front and leave space on the back for the - * encoder - */ - buffer = kmalloc((2 + 512) + 3, cachefiles_gfp); - if (!buffer) - goto nomem_buffer; - - keylen = cookie->key_len; - if (keylen <= sizeof(cookie->inline_key)) - p = cookie->inline_key; - else - p = cookie->key; - memcpy(buffer + 2, p, keylen); - - *(uint16_t *)buffer = keylen; - ((char *)buffer)[keylen + 2] = 0; - ((char *)buffer)[keylen + 3] = 0; - ((char *)buffer)[keylen + 4] = 0; - - /* turn the raw key into something that can work with as a filename */ - key = cachefiles_cook_key(buffer, keylen + 2, object->type); - if (!key) - goto nomem_key; - - /* get hold of the auxiliary data and prepend the object type */ - auxdata = buffer; - auxlen = cookie->aux_len; - if (auxlen) { - if (auxlen <= sizeof(cookie->inline_aux)) - p = cookie->inline_aux; - else - p = cookie->aux; - memcpy(auxdata->data, p, auxlen); - } - - auxdata->len = auxlen + 1; - auxdata->type = cookie->type; - - lookup_data->auxdata = auxdata; - lookup_data->key = key; - object->lookup_data = lookup_data; - - _leave(" = %x [%p]", object->fscache.debug_id, lookup_data); - return &object->fscache; - -nomem_key: - kfree(buffer); -nomem_buffer: - BUG_ON(test_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)); - kmem_cache_free(cachefiles_object_jar, object); - fscache_object_destroyed(&cache->cache); -nomem_object: - kfree(lookup_data); -nomem_lookup_data: - _leave(" = -ENOMEM"); - return ERR_PTR(-ENOMEM); + fscache_count_object(vcookie->cache); + trace_cachefiles_ref(object->debug_id, cookie->debug_id, 1, + cachefiles_obj_new); + return object; } /* - * attempt to look up the nominated node in this cache - * - return -ETIMEDOUT to be scheduled again + * Note that an object has been seen. */ -static int cachefiles_lookup_object(struct fscache_object *_object) +void cachefiles_see_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) { - struct cachefiles_lookup_data *lookup_data; - struct cachefiles_object *parent, *object; - struct cachefiles_cache *cache; - const struct cred *saved_cred; - int ret; - - _enter("{OBJ%x}", _object->debug_id); - - cache = container_of(_object->cache, struct cachefiles_cache, cache); - parent = container_of(_object->parent, - struct cachefiles_object, fscache); - object = container_of(_object, struct cachefiles_object, fscache); - lookup_data = object->lookup_data; - - ASSERTCMP(lookup_data, !=, NULL); - - /* look up the key, creating any missing bits */ - cachefiles_begin_secure(cache, &saved_cred); - ret = cachefiles_walk_to_object(parent, object, - lookup_data->key, - lookup_data->auxdata); - cachefiles_end_secure(cache, saved_cred); - - /* polish off by setting the attributes of non-index files */ - if (ret == 0 && - object->fscache.cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) - cachefiles_attr_changed(&object->fscache); - - if (ret < 0 && ret != -ETIMEDOUT) { - if (ret != -ENOBUFS) - pr_warn("Lookup failed error %d\n", ret); - fscache_object_lookup_error(&object->fscache); - } - - _leave(" [%d]", ret); - return ret; + trace_cachefiles_ref(object->debug_id, object->cookie->debug_id, + refcount_read(&object->ref), why); } /* - * indication of lookup completion + * Increment the usage count on an object; */ -static void cachefiles_lookup_complete(struct fscache_object *_object) +struct cachefiles_object *cachefiles_grab_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) { - struct cachefiles_object *object; - - object = container_of(_object, struct cachefiles_object, fscache); - - _enter("{OBJ%x,%p}", object->fscache.debug_id, object->lookup_data); + int r; - if (object->lookup_data) { - kfree(object->lookup_data->key); - kfree(object->lookup_data->auxdata); - kfree(object->lookup_data); - object->lookup_data = NULL; - } + __refcount_inc(&object->ref, &r); + trace_cachefiles_ref(object->debug_id, object->cookie->debug_id, r, why); + return object; } /* - * increment the usage count on an inode object (may fail if unmounting) + * dispose of a reference to an object */ -static -struct fscache_object *cachefiles_grab_object(struct fscache_object *_object, - enum fscache_obj_ref_trace why) +void cachefiles_put_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) { - struct cachefiles_object *object = - container_of(_object, struct cachefiles_object, fscache); - int u; + unsigned int object_debug_id = object->debug_id; + unsigned int cookie_debug_id = object->cookie->debug_id; + struct fscache_cache *cache; + bool done; + int r; + + done = __refcount_dec_and_test(&object->ref, &r); + trace_cachefiles_ref(object_debug_id, cookie_debug_id, r, why); + if (done) { + _debug("- kill object OBJ%x", object_debug_id); + + ASSERTCMP(object->file, ==, NULL); - _enter("{OBJ%x,%d}", _object->debug_id, atomic_read(&object->usage)); + kfree(object->d_name); -#ifdef CACHEFILES_DEBUG_SLAB - ASSERT((atomic_read(&object->usage) & 0xffff0000) != 0x6b6b0000); -#endif + cache = object->volume->cache->cache; + fscache_put_cookie(object->cookie, fscache_cookie_put_object); + object->cookie = NULL; + kmem_cache_free(cachefiles_object_jar, object); + fscache_uncount_object(cache); + } - u = atomic_inc_return(&object->usage); - trace_cachefiles_ref(object, _object->cookie, - (enum cachefiles_obj_ref_trace)why, u); - return &object->fscache; + _leave(""); } /* - * update the auxiliary data for an object object on disk + * Adjust the size of a cache file if necessary to match the DIO size. We keep + * the EOF marker a multiple of DIO blocks so that we don't fall back to doing + * non-DIO for a partial block straddling the EOF, but we also have to be + * careful of someone expanding the file and accidentally accreting the + * padding. */ -static void cachefiles_update_object(struct fscache_object *_object) +static int cachefiles_adjust_size(struct cachefiles_object *object) { - struct cachefiles_object *object; - struct cachefiles_xattr *auxdata; - struct cachefiles_cache *cache; - struct fscache_cookie *cookie; - const struct cred *saved_cred; - const void *aux; - unsigned auxlen; + struct iattr newattrs; + struct file *file = object->file; + uint64_t ni_size; + loff_t oi_size; + int ret; - _enter("{OBJ%x}", _object->debug_id); + ni_size = object->cookie->object_size; + ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); - object = container_of(_object, struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, struct cachefiles_cache, - cache); + _enter("{OBJ%x},[%llu]", + object->debug_id, (unsigned long long) ni_size); - if (!fscache_use_cookie(_object)) { - _leave(" [relinq]"); - return; - } + if (!file) + return -ENOBUFS; - cookie = object->fscache.cookie; - auxlen = cookie->aux_len; + oi_size = i_size_read(file_inode(file)); + if (oi_size == ni_size) + return 0; - if (!auxlen) { - fscache_unuse_cookie(_object); - _leave(" [no aux]"); - return; - } + inode_lock(file_inode(file)); - auxdata = kmalloc(2 + auxlen + 3, cachefiles_gfp); - if (!auxdata) { - fscache_unuse_cookie(_object); - _leave(" [nomem]"); - return; + /* if there's an extension to a partial page at the end of the backing + * file, we need to discard the partial page so that we pick up new + * data after it */ + if (oi_size & ~PAGE_MASK && ni_size > oi_size) { + _debug("discard tail %llx", oi_size); + newattrs.ia_valid = ATTR_SIZE; + newattrs.ia_size = oi_size & PAGE_MASK; + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = notify_change(&init_user_ns, file->f_path.dentry, + &newattrs, NULL); + if (ret < 0) + goto truncate_failed; } - aux = (auxlen <= sizeof(cookie->inline_aux)) ? - cookie->inline_aux : cookie->aux; + newattrs.ia_valid = ATTR_SIZE; + newattrs.ia_size = ni_size; + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = notify_change(&init_user_ns, file->f_path.dentry, + &newattrs, NULL); - memcpy(auxdata->data, aux, auxlen); - fscache_unuse_cookie(_object); +truncate_failed: + inode_unlock(file_inode(file)); - auxdata->len = auxlen + 1; - auxdata->type = cookie->type; + if (ret < 0) + trace_cachefiles_io_error(NULL, file_inode(file), ret, + cachefiles_trace_notify_change_error); + if (ret == -EIO) { + cachefiles_io_error_obj(object, "Size set failed"); + ret = -ENOBUFS; + } - cachefiles_begin_secure(cache, &saved_cred); - cachefiles_update_object_xattr(object, auxdata); - cachefiles_end_secure(cache, saved_cred); - kfree(auxdata); - _leave(""); + _leave(" = %d", ret); + return ret; } /* - * discard the resources pinned by an object and effect retirement if - * requested + * Attempt to look up the nominated node in this cache */ -static void cachefiles_drop_object(struct fscache_object *_object) +static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie) { struct cachefiles_object *object; - struct cachefiles_cache *cache; + struct cachefiles_cache *cache = cookie->volume->cache->cache_priv; const struct cred *saved_cred; - struct inode *inode; - blkcnt_t i_blocks = 0; + bool success; - ASSERT(_object); + object = cachefiles_alloc_object(cookie); + if (!object) + goto fail; - object = container_of(_object, struct cachefiles_object, fscache); + _enter("{OBJ%x}", object->debug_id); - _enter("{OBJ%x,%d}", - object->fscache.debug_id, atomic_read(&object->usage)); + if (!cachefiles_cook_key(object)) + goto fail_put; - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); + cookie->cache_priv = object; -#ifdef CACHEFILES_DEBUG_SLAB - ASSERT((atomic_read(&object->usage) & 0xffff0000) != 0x6b6b0000); -#endif + cachefiles_begin_secure(cache, &saved_cred); - /* We need to tidy the object up if we did in fact manage to open it. - * It's possible for us to get here before the object is fully - * initialised if the parent goes away or the object gets retired - * before we set it up. - */ - if (object->dentry) { - /* delete retired objects */ - if (test_bit(FSCACHE_OBJECT_RETIRED, &object->fscache.flags) && - _object != cache->cache.fsdef - ) { - _debug("- retire object OBJ%x", object->fscache.debug_id); - inode = d_backing_inode(object->dentry); - if (inode) - i_blocks = inode->i_blocks; - - cachefiles_begin_secure(cache, &saved_cred); - cachefiles_delete_object(cache, object); - cachefiles_end_secure(cache, saved_cred); - } + success = cachefiles_look_up_object(object); + if (!success) + goto fail_withdraw; - /* close the filesystem stuff attached to the object */ - if (object->backer != object->dentry) - dput(object->backer); - object->backer = NULL; - } + cachefiles_see_object(object, cachefiles_obj_see_lookup_cookie); + + spin_lock(&cache->object_list_lock); + list_add(&object->cache_link, &cache->object_list); + spin_unlock(&cache->object_list_lock); + cachefiles_adjust_size(object); - /* note that the object is now inactive */ - if (test_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)) - cachefiles_mark_object_inactive(cache, object, i_blocks); + cachefiles_end_secure(cache, saved_cred); + _leave(" = t"); + return true; - dput(object->dentry); - object->dentry = NULL; +fail_withdraw: + cachefiles_end_secure(cache, saved_cred); + cachefiles_see_object(object, cachefiles_obj_see_lookup_failed); + fscache_caching_failed(cookie); + _debug("failed c=%08x o=%08x", cookie->debug_id, object->debug_id); + /* The caller holds an access count on the cookie, so we need them to + * drop it before we can withdraw the object. + */ + return false; - _leave(""); +fail_put: + cachefiles_put_object(object, cachefiles_obj_put_alloc_fail); +fail: + return false; } /* - * dispose of a reference to an object + * Shorten the backing object to discard any dirty data and free up + * any unused granules. */ -void cachefiles_put_object(struct fscache_object *_object, - enum fscache_obj_ref_trace why) +static bool cachefiles_shorten_object(struct cachefiles_object *object, + struct file *file, loff_t new_size) { - struct cachefiles_object *object; - struct fscache_cache *cache; - int u; - - ASSERT(_object); - - object = container_of(_object, struct cachefiles_object, fscache); - - _enter("{OBJ%x,%d}", - object->fscache.debug_id, atomic_read(&object->usage)); - -#ifdef CACHEFILES_DEBUG_SLAB - ASSERT((atomic_read(&object->usage) & 0xffff0000) != 0x6b6b0000); -#endif - - ASSERTIFCMP(object->fscache.parent, - object->fscache.parent->n_children, >, 0); - - u = atomic_dec_return(&object->usage); - trace_cachefiles_ref(object, _object->cookie, - (enum cachefiles_obj_ref_trace)why, u); - ASSERTCMP(u, !=, -1); - if (u == 0) { - _debug("- kill object OBJ%x", object->fscache.debug_id); + struct cachefiles_cache *cache = object->volume->cache; + struct inode *inode = file_inode(file); + loff_t i_size, dio_size; + int ret; - ASSERT(!test_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)); - ASSERTCMP(object->fscache.parent, ==, NULL); - ASSERTCMP(object->backer, ==, NULL); - ASSERTCMP(object->dentry, ==, NULL); - ASSERTCMP(object->fscache.n_ops, ==, 0); - ASSERTCMP(object->fscache.n_children, ==, 0); + dio_size = round_up(new_size, CACHEFILES_DIO_BLOCK_SIZE); + i_size = i_size_read(inode); + + trace_cachefiles_trunc(object, inode, i_size, dio_size, + cachefiles_trunc_shrink); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_truncate(&file->f_path, dio_size); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_trunc_error); + cachefiles_io_error_obj(object, "Trunc-to-size failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return false; + } - if (object->lookup_data) { - kfree(object->lookup_data->key); - kfree(object->lookup_data->auxdata); - kfree(object->lookup_data); - object->lookup_data = NULL; + if (new_size < dio_size) { + trace_cachefiles_trunc(object, inode, dio_size, new_size, + cachefiles_trunc_dio_adjust); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_fallocate(file, FALLOC_FL_ZERO_RANGE, + new_size, dio_size); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_fallocate_error); + cachefiles_io_error_obj(object, "Trunc-to-dio-size failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return false; } - - cache = object->fscache.cache; - fscache_object_destroy(&object->fscache); - kmem_cache_free(cachefiles_object_jar, object); - fscache_object_destroyed(cache); } - _leave(""); + return true; } /* - * sync a cache + * Resize the backing object. */ -static void cachefiles_sync_cache(struct fscache_cache *_cache) +static void cachefiles_resize_cookie(struct netfs_cache_resources *cres, + loff_t new_size) { - struct cachefiles_cache *cache; + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + struct fscache_cookie *cookie = object->cookie; const struct cred *saved_cred; - int ret; + struct file *file = cachefiles_cres_file(cres); + loff_t old_size = cookie->object_size; - _enter("%s", _cache->tag->name); + _enter("%llu->%llu", old_size, new_size); - cache = container_of(_cache, struct cachefiles_cache, cache); - - /* make sure all pages pinned by operations on behalf of the netfs are - * written to disc */ - cachefiles_begin_secure(cache, &saved_cred); - down_read(&cache->mnt->mnt_sb->s_umount); - ret = sync_filesystem(cache->mnt->mnt_sb); - up_read(&cache->mnt->mnt_sb->s_umount); - cachefiles_end_secure(cache, saved_cred); + if (new_size < old_size) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_shorten_object(object, file, new_size); + cachefiles_end_secure(cache, saved_cred); + object->cookie->object_size = new_size; + return; + } - if (ret == -EIO) - cachefiles_io_error(cache, - "Attempt to sync backing fs superblock" - " returned error %d", - ret); + /* The file is being expanded. We don't need to do anything + * particularly. cookie->initial_size doesn't change and so the point + * at which we have to download before doesn't change. + */ + cookie->object_size = new_size; } /* - * check if the backing cache is updated to FS-Cache - * - called by FS-Cache when evaluates if need to invalidate the cache + * Commit changes to the object as we drop it. */ -static int cachefiles_check_consistency(struct fscache_operation *op) +static void cachefiles_commit_object(struct cachefiles_object *object, + struct cachefiles_cache *cache) { - struct cachefiles_object *object; - struct cachefiles_cache *cache; - const struct cred *saved_cred; - int ret; + bool update = false; - _enter("{OBJ%x}", op->object->debug_id); + if (test_and_clear_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) + update = true; + if (test_and_clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags)) + update = true; + if (update) + cachefiles_set_object_xattr(object); - object = container_of(op->object, struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); + if (test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) + cachefiles_commit_tmpfile(cache, object); +} - cachefiles_begin_secure(cache, &saved_cred); - ret = cachefiles_check_auxdata(object); - cachefiles_end_secure(cache, saved_cred); +/* + * Finalise and object and close the VFS structs that we have. + */ +static void cachefiles_clean_up_object(struct cachefiles_object *object, + struct cachefiles_cache *cache) +{ + if (test_bit(FSCACHE_COOKIE_RETIRED, &object->cookie->flags)) { + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + cachefiles_see_object(object, cachefiles_obj_see_clean_delete); + _debug("- inval object OBJ%x", object->debug_id); + cachefiles_delete_object(object, FSCACHE_OBJECT_WAS_RETIRED); + } else { + cachefiles_see_object(object, cachefiles_obj_see_clean_drop_tmp); + _debug("- inval object OBJ%x tmpfile", object->debug_id); + } + } else { + cachefiles_see_object(object, cachefiles_obj_see_clean_commit); + cachefiles_commit_object(object, cache); + } - _leave(" = %d", ret); - return ret; + cachefiles_unmark_inode_in_use(object, object->file); + if (object->file) { + fput(object->file); + object->file = NULL; + } } /* - * notification the attributes on an object have changed - * - called with reads/writes excluded by FS-Cache + * Withdraw caching for a cookie. */ -static int cachefiles_attr_changed(struct fscache_object *_object) +static void cachefiles_withdraw_cookie(struct fscache_cookie *cookie) { - struct cachefiles_object *object; - struct cachefiles_cache *cache; + struct cachefiles_object *object = cookie->cache_priv; + struct cachefiles_cache *cache = object->volume->cache; const struct cred *saved_cred; - struct iattr newattrs; - uint64_t ni_size; - loff_t oi_size; - int ret; - - ni_size = _object->store_limit_l; - - _enter("{OBJ%x},[%llu]", - _object->debug_id, (unsigned long long) ni_size); - - object = container_of(_object, struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - if (ni_size == object->i_size) - return 0; - - if (!object->backer) - return -ENOBUFS; - ASSERT(d_is_reg(object->backer)); + _enter("o=%x", object->debug_id); + cachefiles_see_object(object, cachefiles_obj_see_withdraw_cookie); - fscache_set_store_limit(&object->fscache, ni_size); - - oi_size = i_size_read(d_backing_inode(object->backer)); - if (oi_size == ni_size) - return 0; - - cachefiles_begin_secure(cache, &saved_cred); - inode_lock(d_inode(object->backer)); - - /* if there's an extension to a partial page at the end of the backing - * file, we need to discard the partial page so that we pick up new - * data after it */ - if (oi_size & ~PAGE_MASK && ni_size > oi_size) { - _debug("discard tail %llx", oi_size); - newattrs.ia_valid = ATTR_SIZE; - newattrs.ia_size = oi_size & PAGE_MASK; - ret = notify_change(&init_user_ns, object->backer, &newattrs, NULL); - if (ret < 0) - goto truncate_failed; + if (!list_empty(&object->cache_link)) { + spin_lock(&cache->object_list_lock); + cachefiles_see_object(object, cachefiles_obj_see_withdrawal); + list_del_init(&object->cache_link); + spin_unlock(&cache->object_list_lock); } - newattrs.ia_valid = ATTR_SIZE; - newattrs.ia_size = ni_size; - ret = notify_change(&init_user_ns, object->backer, &newattrs, NULL); - -truncate_failed: - inode_unlock(d_inode(object->backer)); - cachefiles_end_secure(cache, saved_cred); - - if (ret == -EIO) { - fscache_set_store_limit(&object->fscache, 0); - cachefiles_io_error_obj(object, "Size set failed"); - ret = -ENOBUFS; + if (object->file) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_clean_up_object(object, cache); + cachefiles_end_secure(cache, saved_cred); } - _leave(" = %d", ret); - return ret; + cookie->cache_priv = NULL; + cachefiles_put_object(object, cachefiles_obj_put_detach); } /* - * Invalidate an object + * Invalidate the storage associated with a cookie. */ -static void cachefiles_invalidate_object(struct fscache_operation *op) +static bool cachefiles_invalidate_cookie(struct fscache_cookie *cookie) { - struct cachefiles_object *object; - struct cachefiles_cache *cache; - const struct cred *saved_cred; - struct path path; - uint64_t ni_size; - int ret; + struct cachefiles_object *object = cookie->cache_priv; + struct file *new_file, *old_file; + bool old_tmpfile; - object = container_of(op->object, struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); + _enter("o=%x,[%llu]", object->debug_id, object->cookie->object_size); - ni_size = op->object->store_limit_l; + old_tmpfile = test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); - _enter("{OBJ%x},[%llu]", - op->object->debug_id, (unsigned long long)ni_size); + if (!object->file) { + fscache_resume_after_invalidation(cookie); + _leave(" = t [light]"); + return true; + } - if (object->backer) { - ASSERT(d_is_reg(object->backer)); + new_file = cachefiles_create_tmpfile(object); + if (IS_ERR(new_file)) + goto failed; - fscache_set_store_limit(&object->fscache, ni_size); + /* Substitute the VFS target */ + _debug("sub"); + spin_lock(&object->lock); - path.dentry = object->backer; - path.mnt = cache->mnt; + old_file = object->file; + object->file = new_file; + object->content_info = CACHEFILES_CONTENT_NO_DATA; + set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags); - cachefiles_begin_secure(cache, &saved_cred); - ret = vfs_truncate(&path, 0); - if (ret == 0) - ret = vfs_truncate(&path, ni_size); - cachefiles_end_secure(cache, saved_cred); + spin_unlock(&object->lock); + _debug("subbed"); + + /* Allow I/O to take place again */ + fscache_resume_after_invalidation(cookie); + + if (old_file) { + if (!old_tmpfile) { + struct cachefiles_volume *volume = object->volume; + struct dentry *fan = volume->fanout[(u8)cookie->key_hash]; - if (ret != 0) { - fscache_set_store_limit(&object->fscache, 0); - if (ret == -EIO) - cachefiles_io_error_obj(object, - "Invalidate failed"); + inode_lock_nested(d_inode(fan), I_MUTEX_PARENT); + cachefiles_bury_object(volume->cache, object, fan, + old_file->f_path.dentry, + FSCACHE_OBJECT_INVALIDATED); } + fput(old_file); } - fscache_op_complete(op, true); - _leave(""); -} + _leave(" = t"); + return true; -/* - * dissociate a cache from all the pages it was backing - */ -static void cachefiles_dissociate_pages(struct fscache_cache *cache) -{ - _enter(""); +failed: + _leave(" = f"); + return false; } const struct fscache_cache_ops cachefiles_cache_ops = { .name = "cachefiles", - .alloc_object = cachefiles_alloc_object, - .lookup_object = cachefiles_lookup_object, - .lookup_complete = cachefiles_lookup_complete, - .grab_object = cachefiles_grab_object, - .update_object = cachefiles_update_object, - .invalidate_object = cachefiles_invalidate_object, - .drop_object = cachefiles_drop_object, - .put_object = cachefiles_put_object, - .sync_cache = cachefiles_sync_cache, - .attr_changed = cachefiles_attr_changed, - .read_or_alloc_page = cachefiles_read_or_alloc_page, - .read_or_alloc_pages = cachefiles_read_or_alloc_pages, - .allocate_page = cachefiles_allocate_page, - .allocate_pages = cachefiles_allocate_pages, - .write_page = cachefiles_write_page, - .uncache_page = cachefiles_uncache_page, - .dissociate_pages = cachefiles_dissociate_pages, - .check_consistency = cachefiles_check_consistency, - .begin_read_operation = cachefiles_begin_read_operation, + .acquire_volume = cachefiles_acquire_volume, + .free_volume = cachefiles_free_volume, + .lookup_cookie = cachefiles_lookup_cookie, + .withdraw_cookie = cachefiles_withdraw_cookie, + .invalidate_cookie = cachefiles_invalidate_cookie, + .begin_operation = cachefiles_begin_operation, + .resize_cookie = cachefiles_resize_cookie, + .prepare_to_write = cachefiles_prepare_to_write, }; diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index 0a511c36dab8..8dd54d9375b6 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* General netfs cache on cache files internal defs * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -13,58 +13,72 @@ #include <linux/fscache-cache.h> -#include <linux/timer.h> -#include <linux/wait_bit.h> #include <linux/cred.h> -#include <linux/workqueue.h> #include <linux/security.h> +#define CACHEFILES_DIO_BLOCK_SIZE 4096 + struct cachefiles_cache; struct cachefiles_object; -extern unsigned cachefiles_debug; -#define CACHEFILES_DEBUG_KENTER 1 -#define CACHEFILES_DEBUG_KLEAVE 2 -#define CACHEFILES_DEBUG_KDEBUG 4 +enum cachefiles_content { + /* These values are saved on disk */ + CACHEFILES_CONTENT_NO_DATA = 0, /* No content stored */ + CACHEFILES_CONTENT_SINGLE = 1, /* Content is monolithic, all is present */ + CACHEFILES_CONTENT_ALL = 2, /* Content is all present, no map */ + CACHEFILES_CONTENT_BACKFS_MAP = 3, /* Content is piecemeal, mapped through backing fs */ + CACHEFILES_CONTENT_DIRTY = 4, /* Content is dirty (only seen on disk) */ + nr__cachefiles_content +}; -#define cachefiles_gfp (__GFP_RECLAIM | __GFP_NORETRY | __GFP_NOMEMALLOC) +/* + * Cached volume representation. + */ +struct cachefiles_volume { + struct cachefiles_cache *cache; + struct list_head cache_link; /* Link in cache->volumes */ + struct fscache_volume *vcookie; /* The netfs's representation */ + struct dentry *dentry; /* The volume dentry */ + struct dentry *fanout[256]; /* Fanout subdirs */ +}; /* - * node records + * Backing file state. */ struct cachefiles_object { - struct fscache_object fscache; /* fscache handle */ - struct cachefiles_lookup_data *lookup_data; /* cached lookup data */ - struct dentry *dentry; /* the file/dir representing this object */ - struct dentry *backer; /* backing file */ - loff_t i_size; /* object size */ + struct fscache_cookie *cookie; /* Netfs data storage object cookie */ + struct cachefiles_volume *volume; /* Cache volume that holds this object */ + struct list_head cache_link; /* Link in cache->*_list */ + struct file *file; /* The file representing this object */ + char *d_name; /* Backing file name */ + int debug_id; + spinlock_t lock; + refcount_t ref; + u8 d_name_len; /* Length of filename */ + enum cachefiles_content content_info:8; /* Info about content presence */ unsigned long flags; -#define CACHEFILES_OBJECT_ACTIVE 0 /* T if marked active */ - atomic_t usage; /* object usage count */ - uint8_t type; /* object type */ - uint8_t new; /* T if object new */ - spinlock_t work_lock; - struct rb_node active_node; /* link in active tree (dentry is key) */ +#define CACHEFILES_OBJECT_USING_TMPFILE 0 /* Have an unlinked tmpfile */ }; -extern struct kmem_cache *cachefiles_object_jar; - /* * Cache files cache definition */ struct cachefiles_cache { - struct fscache_cache cache; /* FS-Cache record */ + struct fscache_cache *cache; /* Cache cookie */ struct vfsmount *mnt; /* mountpoint holding the cache */ + struct dentry *store; /* Directory into which live objects go */ struct dentry *graveyard; /* directory into which dead objects go */ struct file *cachefilesd; /* manager daemon handle */ + struct list_head volumes; /* List of volume objects */ + struct list_head object_list; /* List of active objects */ + spinlock_t object_list_lock; /* Lock for volumes and object_list */ const struct cred *cache_cred; /* security override for accessing cache */ struct mutex daemon_mutex; /* command serialisation mutex */ wait_queue_head_t daemon_pollwq; /* poll waitqueue for daemon */ - struct rb_root active_nodes; /* active nodes (can't be culled) */ - rwlock_t active_lock; /* lock for active_nodes */ atomic_t gravecounter; /* graveyard uniquifier */ atomic_t f_released; /* number of objects released lately */ atomic_long_t b_released; /* number of blocks released lately */ + atomic_long_t b_writing; /* Number of blocks being written */ unsigned frun_percent; /* when to stop culling (% files) */ unsigned fcull_percent; /* when to start culling (% files) */ unsigned fstop_percent; /* when to stop allocating (% files) */ @@ -89,38 +103,19 @@ struct cachefiles_cache { char *tag; /* cache binding tag */ }; -/* - * backing file read tracking - */ -struct cachefiles_one_read { - wait_queue_entry_t monitor; /* link into monitored waitqueue */ - struct page *back_page; /* backing file page we're waiting for */ - struct page *netfs_page; /* netfs page we're going to fill */ - struct fscache_retrieval *op; /* retrieval op covering this */ - struct list_head op_link; /* link in op's todo list */ -}; - -/* - * backing file write tracking - */ -struct cachefiles_one_write { - struct page *netfs_page; /* netfs page to copy */ - struct cachefiles_object *object; - struct list_head obj_link; /* link in object's lists */ - fscache_rw_complete_t end_io_func; - void *context; -}; +#include <trace/events/cachefiles.h> -/* - * auxiliary data xattr buffer - */ -struct cachefiles_xattr { - uint16_t len; - uint8_t type; - uint8_t data[]; -}; +static inline +struct file *cachefiles_cres_file(struct netfs_cache_resources *cres) +{ + return cres->cache_priv2; +} -#include <trace/events/cachefiles.h> +static inline +struct cachefiles_object *cachefiles_cres_object(struct netfs_cache_resources *cres) +{ + return fscache_cres_cookie(cres)->cache_priv; +} /* * note change of state for daemon @@ -132,74 +127,118 @@ static inline void cachefiles_state_changed(struct cachefiles_cache *cache) } /* - * bind.c + * cache.c */ -extern int cachefiles_daemon_bind(struct cachefiles_cache *cache, char *args); -extern void cachefiles_daemon_unbind(struct cachefiles_cache *cache); +extern int cachefiles_add_cache(struct cachefiles_cache *cache); +extern void cachefiles_withdraw_cache(struct cachefiles_cache *cache); + +enum cachefiles_has_space_for { + cachefiles_has_space_check, + cachefiles_has_space_for_write, + cachefiles_has_space_for_create, +}; +extern int cachefiles_has_space(struct cachefiles_cache *cache, + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason); /* * daemon.c */ extern const struct file_operations cachefiles_daemon_fops; -extern int cachefiles_has_space(struct cachefiles_cache *cache, - unsigned fnr, unsigned bnr); +/* + * error_inject.c + */ +#ifdef CONFIG_CACHEFILES_ERROR_INJECTION +extern unsigned int cachefiles_error_injection_state; +extern int cachefiles_register_error_injection(void); +extern void cachefiles_unregister_error_injection(void); + +#else +#define cachefiles_error_injection_state 0 + +static inline int cachefiles_register_error_injection(void) +{ + return 0; +} + +static inline void cachefiles_unregister_error_injection(void) +{ +} +#endif + + +static inline int cachefiles_inject_read_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : 0; +} + +static inline int cachefiles_inject_write_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : + cachefiles_error_injection_state & 1 ? -ENOSPC : + 0; +} + +static inline int cachefiles_inject_remove_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : 0; +} /* * interface.c */ extern const struct fscache_cache_ops cachefiles_cache_ops; +extern void cachefiles_see_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); +extern struct cachefiles_object *cachefiles_grab_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); +extern void cachefiles_put_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); -void cachefiles_put_object(struct fscache_object *_object, - enum fscache_obj_ref_trace why); +/* + * io.c + */ +extern bool cachefiles_begin_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state); /* * key.c */ -extern char *cachefiles_cook_key(const u8 *raw, int keylen, uint8_t type); +extern bool cachefiles_cook_key(struct cachefiles_object *object); + +/* + * main.c + */ +extern struct kmem_cache *cachefiles_object_jar; /* * namei.c */ -extern void cachefiles_mark_object_inactive(struct cachefiles_cache *cache, - struct cachefiles_object *object, - blkcnt_t i_blocks); -extern int cachefiles_delete_object(struct cachefiles_cache *cache, - struct cachefiles_object *object); -extern int cachefiles_walk_to_object(struct cachefiles_object *parent, - struct cachefiles_object *object, - const char *key, - struct cachefiles_xattr *auxdata); +extern void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct file *file); +extern int cachefiles_bury_object(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, + struct dentry *rep, + enum fscache_why_object_killed why); +extern int cachefiles_delete_object(struct cachefiles_object *object, + enum fscache_why_object_killed why); +extern bool cachefiles_look_up_object(struct cachefiles_object *object); extern struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, struct dentry *dir, - const char *name); + const char *name, + bool *_is_new); +extern void cachefiles_put_directory(struct dentry *dir); extern int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, char *filename); extern int cachefiles_check_in_use(struct cachefiles_cache *cache, struct dentry *dir, char *filename); - -/* - * rdwr.c - */ -extern int cachefiles_read_or_alloc_page(struct fscache_retrieval *, - struct page *, gfp_t); -extern int cachefiles_read_or_alloc_pages(struct fscache_retrieval *, - struct list_head *, unsigned *, - gfp_t); -extern int cachefiles_allocate_page(struct fscache_retrieval *, struct page *, - gfp_t); -extern int cachefiles_allocate_pages(struct fscache_retrieval *, - struct list_head *, unsigned *, gfp_t); -extern int cachefiles_write_page(struct fscache_storage *, struct page *); -extern void cachefiles_uncache_page(struct fscache_object *, struct page *); - -/* - * rdwr2.c - */ -extern int cachefiles_begin_read_operation(struct netfs_read_request *, - struct fscache_retrieval *); +extern struct file *cachefiles_create_tmpfile(struct cachefiles_object *object); +extern bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, + struct cachefiles_object *object); /* * security.c @@ -222,28 +261,32 @@ static inline void cachefiles_end_secure(struct cachefiles_cache *cache, } /* + * volume.c + */ +void cachefiles_acquire_volume(struct fscache_volume *volume); +void cachefiles_free_volume(struct fscache_volume *volume); +void cachefiles_withdraw_volume(struct cachefiles_volume *volume); + +/* * xattr.c */ -extern int cachefiles_check_object_type(struct cachefiles_object *object); -extern int cachefiles_set_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata); -extern int cachefiles_update_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata); -extern int cachefiles_check_auxdata(struct cachefiles_object *object); -extern int cachefiles_check_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata); +extern int cachefiles_set_object_xattr(struct cachefiles_object *object); +extern int cachefiles_check_auxdata(struct cachefiles_object *object, + struct file *file); extern int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, + struct cachefiles_object *object, struct dentry *dentry); - +extern void cachefiles_prepare_to_write(struct fscache_cookie *cookie); +extern bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume); +extern int cachefiles_check_volume_xattr(struct cachefiles_volume *volume); /* - * error handling + * Error handling */ - #define cachefiles_io_error(___cache, FMT, ...) \ do { \ pr_err("I/O Error: " FMT"\n", ##__VA_ARGS__); \ - fscache_io_error(&(___cache)->cache); \ + fscache_io_error((___cache)->cache); \ set_bit(CACHEFILES_DEAD, &(___cache)->flags); \ } while (0) @@ -251,15 +294,20 @@ do { \ do { \ struct cachefiles_cache *___cache; \ \ - ___cache = container_of((object)->fscache.cache, \ - struct cachefiles_cache, cache); \ - cachefiles_io_error(___cache, FMT, ##__VA_ARGS__); \ + ___cache = (object)->volume->cache; \ + cachefiles_io_error(___cache, FMT " [o=%08x]", ##__VA_ARGS__, \ + (object)->debug_id); \ } while (0) /* - * debug tracing + * Debug tracing */ +extern unsigned cachefiles_debug; +#define CACHEFILES_DEBUG_KENTER 1 +#define CACHEFILES_DEBUG_KLEAVE 2 +#define CACHEFILES_DEBUG_KDEBUG 4 + #define dbgprintk(FMT, ...) \ printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index effe37ef8629..60b1eac2ce78 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -9,8 +9,9 @@ #include <linux/slab.h> #include <linux/file.h> #include <linux/uio.h> +#include <linux/falloc.h> #include <linux/sched/mm.h> -#include <linux/netfs.h> +#include <trace/events/fscache.h> #include "internal.h" struct cachefiles_kiocb { @@ -21,14 +22,18 @@ struct cachefiles_kiocb { size_t skipped; size_t len; }; + struct cachefiles_object *object; netfs_io_terminated_t term_func; void *term_func_priv; bool was_async; + unsigned int inval_counter; /* Copy of cookie->inval_counter */ + u64 b_writing; }; static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki) { if (refcount_dec_and_test(&ki->ki_refcnt)) { + cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq); fput(ki->iocb.ki_filp); kfree(ki); } @@ -40,12 +45,22 @@ static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki) static void cachefiles_read_complete(struct kiocb *iocb, long ret) { struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); + struct inode *inode = file_inode(ki->iocb.ki_filp); _enter("%ld", ret); + if (ret < 0) + trace_cachefiles_io_error(ki->object, inode, ret, + cachefiles_trace_read_error); + if (ki->term_func) { - if (ret >= 0) - ret += ki->skipped; + if (ret >= 0) { + if (ki->object->cookie->inval_counter == ki->inval_counter) + ki->skipped += ret; + else + ret = -ESTALE; + } + ki->term_func(ki->term_func_priv, ret, ki->was_async); } @@ -58,16 +73,24 @@ static void cachefiles_read_complete(struct kiocb *iocb, long ret) static int cachefiles_read(struct netfs_cache_resources *cres, loff_t start_pos, struct iov_iter *iter, - bool seek_data, + enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, void *term_func_priv) { + struct cachefiles_object *object; struct cachefiles_kiocb *ki; - struct file *file = cres->cache_priv2; + struct file *file; unsigned int old_nofs; ssize_t ret = -ENOBUFS; size_t len = iov_iter_count(iter), skipped = 0; + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) + goto presubmission_error; + + fscache_count_read(); + object = cachefiles_cres_object(cres); + file = cachefiles_cres_file(cres); + _enter("%pD,%li,%llx,%zx/%llx", file, file_inode(file)->i_ino, start_pos, len, i_size_read(file_inode(file))); @@ -75,10 +98,12 @@ static int cachefiles_read(struct netfs_cache_resources *cres, /* If the caller asked us to seek for data before doing the read, then * we should do that now. If we find a gap, we fill it with zeros. */ - if (seek_data) { + if (read_hole != NETFS_READ_HOLE_IGNORE) { loff_t off = start_pos, off2; - off2 = vfs_llseek(file, off, SEEK_DATA); + off2 = cachefiles_inject_read_error(); + if (off2 == 0) + off2 = vfs_llseek(file, off, SEEK_DATA); if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) { skipped = 0; ret = off2; @@ -90,6 +115,10 @@ static int cachefiles_read(struct netfs_cache_resources *cres, * in the region, so clear the rest of the buffer and * return success. */ + ret = -ENODATA; + if (read_hole == NETFS_READ_HOLE_FAIL) + goto presubmission_error; + iov_iter_zero(len, iter); skipped = len; ret = 0; @@ -100,7 +129,7 @@ static int cachefiles_read(struct netfs_cache_resources *cres, iov_iter_zero(skipped, iter); } - ret = -ENOBUFS; + ret = -ENOMEM; ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL); if (!ki) goto presubmission_error; @@ -112,6 +141,8 @@ static int cachefiles_read(struct netfs_cache_resources *cres, ki->iocb.ki_hint = ki_hint_validate(file_write_hint(file)); ki->iocb.ki_ioprio = get_current_ioprio(); ki->skipped = skipped; + ki->object = object; + ki->inval_counter = cres->inval_counter; ki->term_func = term_func; ki->term_func_priv = term_func_priv; ki->was_async = true; @@ -120,9 +151,13 @@ static int cachefiles_read(struct netfs_cache_resources *cres, ki->iocb.ki_complete = cachefiles_read_complete; get_file(ki->iocb.ki_filp); + cachefiles_grab_object(object, cachefiles_obj_get_ioreq); + trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped); old_nofs = memalloc_nofs_save(); - ret = vfs_iocb_iter_read(file, &ki->iocb, iter); + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_iocb_iter_read(file, &ki->iocb, iter); memalloc_nofs_restore(old_nofs); switch (ret) { case -EIOCBQUEUED: @@ -162,6 +197,7 @@ presubmission_error: static void cachefiles_write_complete(struct kiocb *iocb, long ret) { struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); + struct cachefiles_object *object = ki->object; struct inode *inode = file_inode(ki->iocb.ki_filp); _enter("%ld", ret); @@ -170,9 +206,14 @@ static void cachefiles_write_complete(struct kiocb *iocb, long ret) __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); __sb_end_write(inode->i_sb, SB_FREEZE_WRITE); + if (ret < 0) + trace_cachefiles_io_error(object, inode, ret, + cachefiles_trace_write_error); + + atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing); + set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags); if (ki->term_func) ki->term_func(ki->term_func_priv, ret, ki->was_async); - cachefiles_put_kiocb(ki); } @@ -185,17 +226,27 @@ static int cachefiles_write(struct netfs_cache_resources *cres, netfs_io_terminated_t term_func, void *term_func_priv) { + struct cachefiles_object *object; + struct cachefiles_cache *cache; struct cachefiles_kiocb *ki; struct inode *inode; - struct file *file = cres->cache_priv2; + struct file *file; unsigned int old_nofs; ssize_t ret = -ENOBUFS; size_t len = iov_iter_count(iter); + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) + goto presubmission_error; + fscache_count_write(); + object = cachefiles_cres_object(cres); + cache = object->volume->cache; + file = cachefiles_cres_file(cres); + _enter("%pD,%li,%llx,%zx/%llx", file, file_inode(file)->i_ino, start_pos, len, i_size_read(file_inode(file))); + ret = -ENOMEM; ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL); if (!ki) goto presubmission_error; @@ -206,14 +257,18 @@ static int cachefiles_write(struct netfs_cache_resources *cres, ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE; ki->iocb.ki_hint = ki_hint_validate(file_write_hint(file)); ki->iocb.ki_ioprio = get_current_ioprio(); + ki->object = object; + ki->inval_counter = cres->inval_counter; ki->start = start_pos; ki->len = len; ki->term_func = term_func; ki->term_func_priv = term_func_priv; ki->was_async = true; + ki->b_writing = (len + (1 << cache->bshift)) >> cache->bshift; if (ki->term_func) ki->iocb.ki_complete = cachefiles_write_complete; + atomic_long_add(ki->b_writing, &cache->b_writing); /* Open-code file_start_write here to grab freeze protection, which * will be released by another thread in aio_complete_rw(). Fool @@ -225,9 +280,13 @@ static int cachefiles_write(struct netfs_cache_resources *cres, __sb_writers_release(inode->i_sb, SB_FREEZE_WRITE); get_file(ki->iocb.ki_filp); + cachefiles_grab_object(object, cachefiles_obj_get_ioreq); + trace_cachefiles_write(object, inode, ki->iocb.ki_pos, len); old_nofs = memalloc_nofs_save(); - ret = vfs_iocb_iter_write(file, &ki->iocb, iter); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_iocb_iter_write(file, &ki->iocb, iter); memalloc_nofs_restore(old_nofs); switch (ret) { case -EIOCBQUEUED: @@ -257,8 +316,8 @@ in_progress: presubmission_error: if (term_func) - term_func(term_func_priv, -ENOMEM, false); - return -ENOMEM; + term_func(term_func_priv, ret, false); + return ret; } /* @@ -268,47 +327,82 @@ presubmission_error: static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subrequest *subreq, loff_t i_size) { - struct fscache_retrieval *op = subreq->rreq->cache_resources.cache_priv; + enum cachefiles_prepare_read_trace why; + struct netfs_read_request *rreq = subreq->rreq; + struct netfs_cache_resources *cres = &rreq->cache_resources; struct cachefiles_object *object; struct cachefiles_cache *cache; + struct fscache_cookie *cookie = fscache_cres_cookie(cres); const struct cred *saved_cred; - struct file *file = subreq->rreq->cache_resources.cache_priv2; + struct file *file = cachefiles_cres_file(cres); + enum netfs_read_source ret = NETFS_DOWNLOAD_FROM_SERVER; loff_t off, to; + ino_t ino = file ? file_inode(file)->i_ino : 0; _enter("%zx @%llx/%llx", subreq->len, subreq->start, i_size); - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); + if (subreq->start >= i_size) { + ret = NETFS_FILL_WITH_ZEROES; + why = cachefiles_trace_read_after_eof; + goto out_no_object; + } - if (!file) - goto cache_fail_nosec; + if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) { + __set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); + why = cachefiles_trace_read_no_data; + goto out_no_object; + } - if (subreq->start >= i_size) - return NETFS_FILL_WITH_ZEROES; + /* The object and the file may be being created in the background. */ + if (!file) { + why = cachefiles_trace_read_no_file; + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) + goto out_no_object; + file = cachefiles_cres_file(cres); + if (!file) + goto out_no_object; + ino = file_inode(file)->i_ino; + } + object = cachefiles_cres_object(cres); + cache = object->volume->cache; cachefiles_begin_secure(cache, &saved_cred); - off = vfs_llseek(file, subreq->start, SEEK_DATA); + off = cachefiles_inject_read_error(); + if (off == 0) + off = vfs_llseek(file, subreq->start, SEEK_DATA); if (off < 0 && off >= (loff_t)-MAX_ERRNO) { - if (off == (loff_t)-ENXIO) + if (off == (loff_t)-ENXIO) { + why = cachefiles_trace_read_seek_nxio; goto download_and_store; - goto cache_fail; + } + trace_cachefiles_io_error(object, file_inode(file), off, + cachefiles_trace_seek_error); + why = cachefiles_trace_read_seek_error; + goto out; } - if (off >= subreq->start + subreq->len) + if (off >= subreq->start + subreq->len) { + why = cachefiles_trace_read_found_hole; goto download_and_store; + } if (off > subreq->start) { off = round_up(off, cache->bsize); subreq->len = off - subreq->start; + why = cachefiles_trace_read_found_part; goto download_and_store; } - to = vfs_llseek(file, subreq->start, SEEK_HOLE); - if (to < 0 && to >= (loff_t)-MAX_ERRNO) - goto cache_fail; + to = cachefiles_inject_read_error(); + if (to == 0) + to = vfs_llseek(file, subreq->start, SEEK_HOLE); + if (to < 0 && to >= (loff_t)-MAX_ERRNO) { + trace_cachefiles_io_error(object, file_inode(file), to, + cachefiles_trace_seek_error); + why = cachefiles_trace_read_seek_error; + goto out; + } if (to < subreq->start + subreq->len) { if (subreq->start + subreq->len >= i_size) @@ -318,32 +412,119 @@ static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subreque subreq->len = to - subreq->start; } - cachefiles_end_secure(cache, saved_cred); - return NETFS_READ_FROM_CACHE; + why = cachefiles_trace_read_have_data; + ret = NETFS_READ_FROM_CACHE; + goto out; download_and_store: - if (cachefiles_has_space(cache, 0, (subreq->len + PAGE_SIZE - 1) / PAGE_SIZE) == 0) - __set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); -cache_fail: + __set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); +out: cachefiles_end_secure(cache, saved_cred); -cache_fail_nosec: - return NETFS_DOWNLOAD_FROM_SERVER; +out_no_object: + trace_cachefiles_prep_read(subreq, ret, why, ino); + return ret; } /* * Prepare for a write to occur. */ -static int cachefiles_prepare_write(struct netfs_cache_resources *cres, - loff_t *_start, size_t *_len, loff_t i_size) +static int __cachefiles_prepare_write(struct netfs_cache_resources *cres, + loff_t *_start, size_t *_len, loff_t i_size, + bool no_space_allocated_yet) { - loff_t start = *_start; + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + struct file *file = cachefiles_cres_file(cres); + loff_t start = *_start, pos; size_t len = *_len, down; + int ret; /* Round to DIO size */ down = start - round_down(start, PAGE_SIZE); *_start = start - down; *_len = round_up(down + len, PAGE_SIZE); - return 0; + + /* We need to work out whether there's sufficient disk space to perform + * the write - but we can skip that check if we have space already + * allocated. + */ + if (no_space_allocated_yet) + goto check_space; + + pos = cachefiles_inject_read_error(); + if (pos == 0) + pos = vfs_llseek(file, *_start, SEEK_DATA); + if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { + if (pos == -ENXIO) + goto check_space; /* Unallocated tail */ + trace_cachefiles_io_error(object, file_inode(file), pos, + cachefiles_trace_seek_error); + return pos; + } + if ((u64)pos >= (u64)*_start + *_len) + goto check_space; /* Unallocated region */ + + /* We have a block that's at least partially filled - if we're low on + * space, we need to see if it's fully allocated. If it's not, we may + * want to cull it. + */ + if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_check) == 0) + return 0; /* Enough space to simply overwrite the whole block */ + + pos = cachefiles_inject_read_error(); + if (pos == 0) + pos = vfs_llseek(file, *_start, SEEK_HOLE); + if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { + trace_cachefiles_io_error(object, file_inode(file), pos, + cachefiles_trace_seek_error); + return pos; + } + if ((u64)pos >= (u64)*_start + *_len) + return 0; /* Fully allocated */ + + /* Partially allocated, but insufficient space: cull. */ + fscache_count_no_write_space(); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + *_start, *_len); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_fallocate_error); + cachefiles_io_error_obj(object, + "CacheFiles: fallocate failed (%d)\n", ret); + ret = -EIO; + } + + return ret; + +check_space: + return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_for_write); +} + +static int cachefiles_prepare_write(struct netfs_cache_resources *cres, + loff_t *_start, size_t *_len, loff_t i_size, + bool no_space_allocated_yet) +{ + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + const struct cred *saved_cred; + int ret; + + if (!cachefiles_cres_file(cres)) { + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) + return -ENOBUFS; + if (!cachefiles_cres_file(cres)) + return -ENOBUFS; + } + + cachefiles_begin_secure(cache, &saved_cred); + ret = __cachefiles_prepare_write(cres, _start, _len, i_size, + no_space_allocated_yet); + cachefiles_end_secure(cache, saved_cred); + return ret; } /* @@ -351,19 +532,11 @@ static int cachefiles_prepare_write(struct netfs_cache_resources *cres, */ static void cachefiles_end_operation(struct netfs_cache_resources *cres) { - struct fscache_retrieval *op = cres->cache_priv; - struct file *file = cres->cache_priv2; - - _enter(""); + struct file *file = cachefiles_cres_file(cres); if (file) fput(file); - if (op) { - fscache_op_complete(&op->op, false); - fscache_put_retrieval(op); - } - - _leave(""); + fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end); } static const struct netfs_cache_ops cachefiles_netfs_cache_ops = { @@ -377,44 +550,25 @@ static const struct netfs_cache_ops cachefiles_netfs_cache_ops = { /* * Open the cache file when beginning a cache operation. */ -int cachefiles_begin_read_operation(struct netfs_read_request *rreq, - struct fscache_retrieval *op) +bool cachefiles_begin_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) { - struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct path path; - struct file *file; + struct cachefiles_object *object = cachefiles_cres_object(cres); + + if (!cachefiles_cres_file(cres)) { + cres->ops = &cachefiles_netfs_cache_ops; + if (object->file) { + spin_lock(&object->lock); + if (!cres->cache_priv2 && object->file) + cres->cache_priv2 = get_file(object->file); + spin_unlock(&object->lock); + } + } - _enter(""); - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - path.mnt = cache->mnt; - path.dentry = object->backer; - file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT, - d_inode(object->backer), cache->cache_cred); - if (IS_ERR(file)) - return PTR_ERR(file); - if (!S_ISREG(file_inode(file)->i_mode)) - goto error_file; - if (unlikely(!file->f_op->read_iter) || - unlikely(!file->f_op->write_iter)) { - pr_notice("Cache does not support read_iter and write_iter\n"); - goto error_file; + if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) { + pr_err("failed to get cres->file\n"); + return false; } - fscache_get_retrieval(op); - rreq->cache_resources.cache_priv = op; - rreq->cache_resources.cache_priv2 = file; - rreq->cache_resources.ops = &cachefiles_netfs_cache_ops; - rreq->cache_resources.debug_id = object->fscache.debug_id; - _leave(""); - return 0; - -error_file: - fput(file); - return -EIO; + return true; } diff --git a/fs/cachefiles/key.c b/fs/cachefiles/key.c index 7f94efc97e23..bf935e25bdbe 100644 --- a/fs/cachefiles/key.c +++ b/fs/cachefiles/key.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* Key to pathname encoder * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -22,134 +22,117 @@ static const char cachefiles_filecharmap[256] = { [48 ... 127] = 1, /* '0' -> '~' */ }; +static inline unsigned int how_many_hex_digits(unsigned int x) +{ + return x ? round_up(ilog2(x) + 1, 4) / 4 : 0; +} + /* * turn the raw key into something cooked - * - the raw key should include the length in the two bytes at the front - * - the key may be up to 514 bytes in length (including the length word) + * - the key may be up to NAME_MAX in length (including the length word) * - "base64" encode the strange keys, mapping 3 bytes of raw to four of * cooked * - need to cut the cooked key into 252 char lengths (189 raw bytes) */ -char *cachefiles_cook_key(const u8 *raw, int keylen, uint8_t type) +bool cachefiles_cook_key(struct cachefiles_object *object) { - unsigned char csum, ch; - unsigned int acc; - char *key; - int loop, len, max, seg, mark, print; + const u8 *key = fscache_get_key(object->cookie), *kend; + unsigned char ch; + unsigned int acc, i, n, nle, nbe, keylen = object->cookie->key_len; + unsigned int b64len, len, print, pad; + char *name, sep; - _enter(",%d", keylen); + _enter(",%u,%*phN", keylen, keylen, key); - BUG_ON(keylen < 2 || keylen > 514); + BUG_ON(keylen > NAME_MAX - 3); - csum = raw[0] + raw[1]; print = 1; - for (loop = 2; loop < keylen; loop++) { - ch = raw[loop]; - csum += ch; + for (i = 0; i < keylen; i++) { + ch = key[i]; print &= cachefiles_filecharmap[ch]; } + /* If the path is usable ASCII, then we render it directly */ if (print) { - /* if the path is usable ASCII, then we render it directly */ - max = keylen - 2; - max += 2; /* two base64'd length chars on the front */ - max += 5; /* @checksum/M */ - max += 3 * 2; /* maximum number of segment dividers (".../M") - * is ((514 + 251) / 252) = 3 - */ - max += 1; /* NUL on end */ - } else { - /* calculate the maximum length of the cooked key */ - keylen = (keylen + 2) / 3; - - max = keylen * 4; - max += 5; /* @checksum/M */ - max += 3 * 2; /* maximum number of segment dividers (".../M") - * is ((514 + 188) / 189) = 3 - */ - max += 1; /* NUL on end */ + len = 1 + keylen; + name = kmalloc(len + 1, GFP_KERNEL); + if (!name) + return false; + + name[0] = 'D'; /* Data object type, string encoding */ + memcpy(name + 1, key, keylen); + goto success; } - max += 1; /* 2nd NUL on end */ - - _debug("max: %d", max); - - key = kmalloc(max, cachefiles_gfp); - if (!key) - return NULL; - - len = 0; - - /* build the cooked key */ - sprintf(key, "@%02x%c+", (unsigned) csum, 0); - len = 5; - mark = len - 1; - - if (print) { - acc = *(uint16_t *) raw; - raw += 2; - - key[len + 1] = cachefiles_charmap[acc & 63]; - acc >>= 6; - key[len] = cachefiles_charmap[acc & 63]; - len += 2; - - seg = 250; - for (loop = keylen; loop > 0; loop--) { - if (seg <= 0) { - key[len++] = '\0'; - mark = len; - key[len++] = '+'; - seg = 252; - } - - key[len++] = *raw++; - ASSERT(len < max); - } - - switch (type) { - case FSCACHE_COOKIE_TYPE_INDEX: type = 'I'; break; - case FSCACHE_COOKIE_TYPE_DATAFILE: type = 'D'; break; - default: type = 'S'; break; - } - } else { - seg = 252; - for (loop = keylen; loop > 0; loop--) { - if (seg <= 0) { - key[len++] = '\0'; - mark = len; - key[len++] = '+'; - seg = 252; - } - - acc = *raw++; - acc |= *raw++ << 8; - acc |= *raw++ << 16; - - _debug("acc: %06x", acc); - - key[len++] = cachefiles_charmap[acc & 63]; - acc >>= 6; - key[len++] = cachefiles_charmap[acc & 63]; - acc >>= 6; - key[len++] = cachefiles_charmap[acc & 63]; - acc >>= 6; - key[len++] = cachefiles_charmap[acc & 63]; - - ASSERT(len < max); - } + /* See if it makes sense to encode it as "hex,hex,hex" for each 32-bit + * chunk. We rely on the key having been padded out to a whole number + * of 32-bit words. + */ + n = round_up(keylen, 4); + nbe = nle = 0; + for (i = 0; i < n; i += 4) { + u32 be = be32_to_cpu(*(__be32 *)(key + i)); + u32 le = le32_to_cpu(*(__le32 *)(key + i)); + + nbe += 1 + how_many_hex_digits(be); + nle += 1 + how_many_hex_digits(le); + } - switch (type) { - case FSCACHE_COOKIE_TYPE_INDEX: type = 'J'; break; - case FSCACHE_COOKIE_TYPE_DATAFILE: type = 'E'; break; - default: type = 'T'; break; + b64len = DIV_ROUND_UP(keylen, 3); + pad = b64len * 3 - keylen; + b64len = 2 + b64len * 4; /* Length if we base64-encode it */ + _debug("len=%u nbe=%u nle=%u b64=%u", keylen, nbe, nle, b64len); + if (nbe < b64len || nle < b64len) { + unsigned int nlen = min(nbe, nle) + 1; + name = kmalloc(nlen, GFP_KERNEL); + if (!name) + return false; + sep = (nbe <= nle) ? 'S' : 'T'; /* Encoding indicator */ + len = 0; + for (i = 0; i < n; i += 4) { + u32 x; + if (nbe <= nle) + x = be32_to_cpu(*(__be32 *)(key + i)); + else + x = le32_to_cpu(*(__le32 *)(key + i)); + name[len++] = sep; + if (x != 0) + len += snprintf(name + len, nlen - len, "%x", x); + sep = ','; } + goto success; } - key[mark] = type; - key[len++] = 0; - key[len] = 0; + /* We need to base64-encode it */ + name = kmalloc(b64len + 1, GFP_KERNEL); + if (!name) + return false; + + name[0] = 'E'; + name[1] = '0' + pad; + len = 2; + kend = key + keylen; + do { + acc = *key++; + if (key < kend) { + acc |= *key++ << 8; + if (key < kend) + acc |= *key++ << 16; + } - _leave(" = %s %d", key, len); - return key; + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + } while (key < kend); + +success: + name[len] = 0; + object->d_name = name; + object->d_name_len = len; + _leave(" = %s", object->d_name); + return true; } diff --git a/fs/cachefiles/main.c b/fs/cachefiles/main.c index 9c8d34c49b12..3f369c6f816d 100644 --- a/fs/cachefiles/main.c +++ b/fs/cachefiles/main.c @@ -2,7 +2,7 @@ /* Network filesystem caching backend to use cache files on a premounted * filesystem * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -18,6 +18,8 @@ #include <linux/statfs.h> #include <linux/sysctl.h> #include <linux/miscdevice.h> +#include <linux/netfs.h> +#include <trace/events/netfs.h> #define CREATE_TRACE_POINTS #include "internal.h" @@ -37,14 +39,6 @@ static struct miscdevice cachefiles_dev = { .fops = &cachefiles_daemon_fops, }; -static void cachefiles_object_init_once(void *_object) -{ - struct cachefiles_object *object = _object; - - memset(object, 0, sizeof(*object)); - spin_lock_init(&object->work_lock); -} - /* * initialise the fs caching module */ @@ -52,6 +46,9 @@ static int __init cachefiles_init(void) { int ret; + ret = cachefiles_register_error_injection(); + if (ret < 0) + goto error_einj; ret = misc_register(&cachefiles_dev); if (ret < 0) goto error_dev; @@ -61,9 +58,7 @@ static int __init cachefiles_init(void) cachefiles_object_jar = kmem_cache_create("cachefiles_object_jar", sizeof(struct cachefiles_object), - 0, - SLAB_HWCACHE_ALIGN, - cachefiles_object_init_once); + 0, SLAB_HWCACHE_ALIGN, NULL); if (!cachefiles_object_jar) { pr_notice("Failed to allocate an object jar\n"); goto error_object_jar; @@ -75,6 +70,8 @@ static int __init cachefiles_init(void) error_object_jar: misc_deregister(&cachefiles_dev); error_dev: + cachefiles_unregister_error_injection(); +error_einj: pr_err("failed to register: %d\n", ret); return ret; } @@ -90,6 +87,7 @@ static void __exit cachefiles_exit(void) kmem_cache_destroy(cachefiles_object_jar); misc_deregister(&cachefiles_dev); + cachefiles_unregister_error_injection(); } module_exit(cachefiles_exit); diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index a9aca5ab5970..9bd692870617 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -1,295 +1,268 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* CacheFiles path walking and related routines * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/file.h> #include <linux/fs.h> -#include <linux/fsnotify.h> -#include <linux/quotaops.h> -#include <linux/xattr.h> -#include <linux/mount.h> #include <linux/namei.h> -#include <linux/security.h> -#include <linux/slab.h> #include "internal.h" -#define CACHEFILES_KEYBUF_SIZE 512 - /* - * dump debugging info about an object + * Mark the backing file as being a cache file if it's not already in use. The + * mark tells the culling request command that it's not allowed to cull the + * file or directory. The caller must hold the inode lock. */ -static noinline -void __cachefiles_printk_object(struct cachefiles_object *object, - const char *prefix) +static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, + struct dentry *dentry) { - struct fscache_cookie *cookie; - const u8 *k; - unsigned loop; - - pr_err("%sobject: OBJ%x\n", prefix, object->fscache.debug_id); - pr_err("%sobjstate=%s fl=%lx wbusy=%x ev=%lx[%lx]\n", - prefix, object->fscache.state->name, - object->fscache.flags, work_busy(&object->fscache.work), - object->fscache.events, object->fscache.event_mask); - pr_err("%sops=%u inp=%u exc=%u\n", - prefix, object->fscache.n_ops, object->fscache.n_in_progress, - object->fscache.n_exclusive); - pr_err("%sparent=%x\n", - prefix, object->fscache.parent ? object->fscache.parent->debug_id : 0); - - spin_lock(&object->fscache.lock); - cookie = object->fscache.cookie; - if (cookie) { - pr_err("%scookie=%x [pr=%x nd=%p fl=%lx]\n", - prefix, - cookie->debug_id, - cookie->parent ? cookie->parent->debug_id : 0, - cookie->netfs_data, - cookie->flags); - pr_err("%skey=[%u] '", prefix, cookie->key_len); - k = (cookie->key_len <= sizeof(cookie->inline_key)) ? - cookie->inline_key : cookie->key; - for (loop = 0; loop < cookie->key_len; loop++) - pr_cont("%02x", k[loop]); - pr_cont("'\n"); + struct inode *inode = d_backing_inode(dentry); + bool can_use = false; + + if (!(inode->i_flags & S_KERNEL_FILE)) { + inode->i_flags |= S_KERNEL_FILE; + trace_cachefiles_mark_active(object, inode); + can_use = true; } else { - pr_err("%scookie=NULL\n", prefix); + pr_notice("cachefiles: Inode already in use: %pd\n", dentry); } - spin_unlock(&object->fscache.lock); + + return can_use; } -/* - * dump debugging info about a pair of objects - */ -static noinline void cachefiles_printk_object(struct cachefiles_object *object, - struct cachefiles_object *xobject) +static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object, + struct dentry *dentry) { - if (object) - __cachefiles_printk_object(object, ""); - if (xobject) - __cachefiles_printk_object(xobject, "x"); + struct inode *inode = d_backing_inode(dentry); + bool can_use; + + inode_lock(inode); + can_use = __cachefiles_mark_inode_in_use(object, dentry); + inode_unlock(inode); + return can_use; } /* - * mark the owner of a dentry, if there is one, to indicate that that dentry - * has been preemptively deleted - * - the caller must hold the i_mutex on the dentry's parent as required to - * call vfs_unlink(), vfs_rmdir() or vfs_rename() + * Unmark a backing inode. The caller must hold the inode lock. */ -static void cachefiles_mark_object_buried(struct cachefiles_cache *cache, - struct dentry *dentry, - enum fscache_why_object_killed why) +static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct dentry *dentry) { - struct cachefiles_object *object; - struct rb_node *p; - - _enter(",'%pd'", dentry); + struct inode *inode = d_backing_inode(dentry); - write_lock(&cache->active_lock); + inode->i_flags &= ~S_KERNEL_FILE; + trace_cachefiles_mark_inactive(object, inode); +} - p = cache->active_nodes.rb_node; - while (p) { - object = rb_entry(p, struct cachefiles_object, active_node); - if (object->dentry > dentry) - p = p->rb_left; - else if (object->dentry < dentry) - p = p->rb_right; - else - goto found_dentry; +/* + * Unmark a backing inode and tell cachefilesd that there's something that can + * be culled. + */ +void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct file *file) +{ + struct cachefiles_cache *cache = object->volume->cache; + struct inode *inode = file_inode(file); + + if (inode) { + inode_lock(inode); + __cachefiles_unmark_inode_in_use(object, file->f_path.dentry); + inode_unlock(inode); + + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + atomic_long_add(inode->i_blocks, &cache->b_released); + if (atomic_inc_return(&cache->f_released)) + cachefiles_state_changed(cache); + } } +} - write_unlock(&cache->active_lock); - trace_cachefiles_mark_buried(NULL, dentry, why); - _leave(" [no owner]"); - return; +/* + * get a subdirectory + */ +struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, + struct dentry *dir, + const char *dirname, + bool *_is_new) +{ + struct dentry *subdir; + struct path path; + int ret; - /* found the dentry for */ -found_dentry: - kdebug("preemptive burial: OBJ%x [%s] %pd", - object->fscache.debug_id, - object->fscache.state->name, - dentry); + _enter(",,%s", dirname); - trace_cachefiles_mark_buried(object, dentry, why); + /* search the current directory for the element name */ + inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); - if (fscache_object_is_live(&object->fscache)) { - pr_err("\n"); - pr_err("Error: Can't preemptively bury live object\n"); - cachefiles_printk_object(object, NULL); - } else { - if (why != FSCACHE_OBJECT_IS_STALE) - fscache_object_mark_killed(&object->fscache, why); +retry: + ret = cachefiles_inject_read_error(); + if (ret == 0) + subdir = lookup_one_len(dirname, dir, strlen(dirname)); + else + subdir = ERR_PTR(ret); + if (IS_ERR(subdir)) { + trace_cachefiles_vfs_error(NULL, d_backing_inode(dir), + PTR_ERR(subdir), + cachefiles_trace_lookup_error); + if (PTR_ERR(subdir) == -ENOMEM) + goto nomem_d_alloc; + goto lookup_error; } - write_unlock(&cache->active_lock); - _leave(" [owner marked]"); -} + _debug("subdir -> %pd %s", + subdir, d_backing_inode(subdir) ? "positive" : "negative"); -/* - * record the fact that an object is now active - */ -static int cachefiles_mark_object_active(struct cachefiles_cache *cache, - struct cachefiles_object *object) -{ - struct cachefiles_object *xobject; - struct rb_node **_p, *_parent = NULL; - struct dentry *dentry; + /* we need to create the subdir if it doesn't exist yet */ + if (d_is_negative(subdir)) { + ret = cachefiles_has_space(cache, 1, 0, + cachefiles_has_space_for_create); + if (ret < 0) + goto mkdir_error; - _enter(",%x", object->fscache.debug_id); + _debug("attempt mkdir"); -try_again: - write_lock(&cache->active_lock); + path.mnt = cache->mnt; + path.dentry = dir; + ret = security_path_mkdir(&path, subdir, 0700); + if (ret < 0) + goto mkdir_error; + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_mkdir(&init_user_ns, d_inode(dir), subdir, 0700); + if (ret < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(dir), ret, + cachefiles_trace_mkdir_error); + goto mkdir_error; + } - dentry = object->dentry; - trace_cachefiles_mark_active(object, dentry); + if (unlikely(d_unhashed(subdir))) { + cachefiles_put_directory(subdir); + goto retry; + } + ASSERT(d_backing_inode(subdir)); - if (test_and_set_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)) { - pr_err("Error: Object already active\n"); - cachefiles_printk_object(object, NULL); - BUG(); + _debug("mkdir -> %pd{ino=%lu}", + subdir, d_backing_inode(subdir)->i_ino); + if (_is_new) + *_is_new = true; } - _p = &cache->active_nodes.rb_node; - while (*_p) { - _parent = *_p; - xobject = rb_entry(_parent, - struct cachefiles_object, active_node); + /* Tell rmdir() it's not allowed to delete the subdir */ + inode_lock(d_inode(subdir)); + inode_unlock(d_inode(dir)); - ASSERT(xobject != object); + if (!__cachefiles_mark_inode_in_use(NULL, subdir)) + goto mark_error; - if (xobject->dentry > dentry) - _p = &(*_p)->rb_left; - else if (xobject->dentry < dentry) - _p = &(*_p)->rb_right; - else - goto wait_for_old_object; - } + inode_unlock(d_inode(subdir)); - rb_link_node(&object->active_node, _parent, _p); - rb_insert_color(&object->active_node, &cache->active_nodes); + /* we need to make sure the subdir is a directory */ + ASSERT(d_backing_inode(subdir)); - write_unlock(&cache->active_lock); - _leave(" = 0"); - return 0; + if (!d_can_lookup(subdir)) { + pr_err("%s is not a directory\n", dirname); + ret = -EIO; + goto check_error; + } - /* an old object from a previous incarnation is hogging the slot - we - * need to wait for it to be destroyed */ -wait_for_old_object: - trace_cachefiles_wait_active(object, dentry, xobject); - clear_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags); - - if (fscache_object_is_live(&xobject->fscache)) { - pr_err("\n"); - pr_err("Error: Unexpected object collision\n"); - cachefiles_printk_object(object, xobject); - } - atomic_inc(&xobject->usage); - write_unlock(&cache->active_lock); - - if (test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags)) { - wait_queue_head_t *wq; - - signed long timeout = 60 * HZ; - wait_queue_entry_t wait; - bool requeue; - - /* if the object we're waiting for is queued for processing, - * then just put ourselves on the queue behind it */ - if (work_pending(&xobject->fscache.work)) { - _debug("queue OBJ%x behind OBJ%x immediately", - object->fscache.debug_id, - xobject->fscache.debug_id); - goto requeue; - } + ret = -EPERM; + if (!(d_backing_inode(subdir)->i_opflags & IOP_XATTR) || + !d_backing_inode(subdir)->i_op->lookup || + !d_backing_inode(subdir)->i_op->mkdir || + !d_backing_inode(subdir)->i_op->rename || + !d_backing_inode(subdir)->i_op->rmdir || + !d_backing_inode(subdir)->i_op->unlink) + goto check_error; - /* otherwise we sleep until either the object we're waiting for - * is done, or the fscache_object is congested */ - wq = bit_waitqueue(&xobject->flags, CACHEFILES_OBJECT_ACTIVE); - init_wait(&wait); - requeue = false; - do { - prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); - if (!test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags)) - break; - - requeue = fscache_object_sleep_till_congested(&timeout); - } while (timeout > 0 && !requeue); - finish_wait(wq, &wait); - - if (requeue && - test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags)) { - _debug("queue OBJ%x behind OBJ%x after wait", - object->fscache.debug_id, - xobject->fscache.debug_id); - goto requeue; - } + _leave(" = [%lu]", d_backing_inode(subdir)->i_ino); + return subdir; - if (timeout <= 0) { - pr_err("\n"); - pr_err("Error: Overlong wait for old active object to go away\n"); - cachefiles_printk_object(object, xobject); - goto requeue; - } - } +check_error: + cachefiles_put_directory(subdir); + _leave(" = %d [check]", ret); + return ERR_PTR(ret); - ASSERT(!test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags)); +mark_error: + inode_unlock(d_inode(subdir)); + dput(subdir); + return ERR_PTR(-EBUSY); - cache->cache.ops->put_object(&xobject->fscache, - (enum fscache_obj_ref_trace)cachefiles_obj_put_wait_retry); - goto try_again; +mkdir_error: + inode_unlock(d_inode(dir)); + dput(subdir); + pr_err("mkdir %s failed with error %d\n", dirname, ret); + return ERR_PTR(ret); + +lookup_error: + inode_unlock(d_inode(dir)); + ret = PTR_ERR(subdir); + pr_err("Lookup %s failed with error %d\n", dirname, ret); + return ERR_PTR(ret); -requeue: - cache->cache.ops->put_object(&xobject->fscache, - (enum fscache_obj_ref_trace)cachefiles_obj_put_wait_timeo); - _leave(" = -ETIMEDOUT"); - return -ETIMEDOUT; +nomem_d_alloc: + inode_unlock(d_inode(dir)); + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); } /* - * Mark an object as being inactive. + * Put a subdirectory. */ -void cachefiles_mark_object_inactive(struct cachefiles_cache *cache, - struct cachefiles_object *object, - blkcnt_t i_blocks) +void cachefiles_put_directory(struct dentry *dir) { - struct dentry *dentry = object->dentry; - struct inode *inode = d_backing_inode(dentry); - - trace_cachefiles_mark_inactive(object, dentry, inode); + if (dir) { + inode_lock(dir->d_inode); + __cachefiles_unmark_inode_in_use(NULL, dir); + inode_unlock(dir->d_inode); + dput(dir); + } +} - write_lock(&cache->active_lock); - rb_erase(&object->active_node, &cache->active_nodes); - clear_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags); - write_unlock(&cache->active_lock); +/* + * Remove a regular file from the cache. + */ +static int cachefiles_unlink(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, struct dentry *dentry, + enum fscache_why_object_killed why) +{ + struct path path = { + .mnt = cache->mnt, + .dentry = dir, + }; + int ret; - wake_up_bit(&object->flags, CACHEFILES_OBJECT_ACTIVE); + trace_cachefiles_unlink(object, dentry, why); + ret = security_path_unlink(&path, dentry); + if (ret < 0) { + cachefiles_io_error(cache, "Unlink security error"); + return ret; + } - /* This object can now be culled, so we need to let the daemon know - * that there is something it can remove if it needs to. - */ - atomic_long_add(i_blocks, &cache->b_released); - if (atomic_inc_return(&cache->f_released)) - cachefiles_state_changed(cache); + ret = cachefiles_inject_remove_error(); + if (ret == 0) { + ret = vfs_unlink(&init_user_ns, d_backing_inode(dir), dentry, NULL); + if (ret == -EIO) + cachefiles_io_error(cache, "Unlink failed"); + } + if (ret != 0) + trace_cachefiles_vfs_error(object, d_backing_inode(dir), ret, + cachefiles_trace_unlink_error); + return ret; } /* - * delete an object representation from the cache - * - file backed objects are unlinked - * - directory backed objects are stuffed into the graveyard for userspace to + * Delete an object representation from the cache + * - File backed objects are unlinked + * - Directory backed objects are stuffed into the graveyard for userspace to * delete - * - unlocks the directory mutex */ -static int cachefiles_bury_object(struct cachefiles_cache *cache, - struct cachefiles_object *object, - struct dentry *dir, - struct dentry *rep, - bool preemptive, - enum fscache_why_object_killed why) +int cachefiles_bury_object(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, + struct dentry *rep, + enum fscache_why_object_killed why) { struct dentry *grave, *trap; struct path path, path_to_graveyard; @@ -298,29 +271,21 @@ static int cachefiles_bury_object(struct cachefiles_cache *cache, _enter(",'%pd','%pd'", dir, rep); + if (rep->d_parent != dir) { + inode_unlock(d_inode(dir)); + _leave(" = -ESTALE"); + return -ESTALE; + } + /* non-directories can just be unlinked */ if (!d_is_dir(rep)) { - _debug("unlink stale object"); - - path.mnt = cache->mnt; - path.dentry = dir; - ret = security_path_unlink(&path, rep); - if (ret < 0) { - cachefiles_io_error(cache, "Unlink security error"); - } else { - trace_cachefiles_unlink(object, rep, why); - ret = vfs_unlink(&init_user_ns, d_inode(dir), rep, - NULL); - - if (preemptive) - cachefiles_mark_object_buried(cache, rep, why); - } + dget(rep); /* Stop the dentry being negated if it's only pinned + * by a file struct. + */ + ret = cachefiles_unlink(cache, object, dir, rep, why); + dput(rep); inode_unlock(d_inode(dir)); - - if (ret == -EIO) - cachefiles_io_error(cache, "Unlink failed"); - _leave(" = %d", ret); return ret; } @@ -368,14 +333,16 @@ try_again: grave = lookup_one_len(nbuffer, cache->graveyard, strlen(nbuffer)); if (IS_ERR(grave)) { unlock_rename(cache->graveyard, dir); + trace_cachefiles_vfs_error(object, d_inode(cache->graveyard), + PTR_ERR(grave), + cachefiles_trace_lookup_error); if (PTR_ERR(grave) == -ENOMEM) { _leave(" = -ENOMEM"); return -ENOMEM; } - cachefiles_io_error(cache, "Lookup error %ld", - PTR_ERR(grave)); + cachefiles_io_error(cache, "Lookup error %ld", PTR_ERR(grave)); return -EIO; } @@ -420,15 +387,18 @@ try_again: .new_dentry = grave, }; trace_cachefiles_rename(object, rep, grave, why); - ret = vfs_rename(&rd); + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_rename(&rd); + if (ret != 0) + trace_cachefiles_vfs_error(object, d_inode(dir), ret, + cachefiles_trace_rename_error); if (ret != 0 && ret != -ENOMEM) cachefiles_io_error(cache, "Rename failed with error %d", ret); - - if (preemptive) - cachefiles_mark_object_buried(cache, rep, why); } + __cachefiles_unmark_inode_in_use(object, rep); unlock_rename(cache->graveyard, dir); dput(grave); _leave(" = 0"); @@ -436,493 +406,358 @@ try_again: } /* - * delete an object representation from the cache + * Delete a cache file. */ -int cachefiles_delete_object(struct cachefiles_cache *cache, - struct cachefiles_object *object) +int cachefiles_delete_object(struct cachefiles_object *object, + enum fscache_why_object_killed why) { - struct dentry *dir; + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry = object->file->f_path.dentry; + struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; int ret; - _enter(",OBJ%x{%pd}", object->fscache.debug_id, object->dentry); - - ASSERT(object->dentry); - ASSERT(d_backing_inode(object->dentry)); - ASSERT(object->dentry->d_parent); + _enter(",OBJ%x{%pD}", object->debug_id, object->file); - dir = dget_parent(object->dentry); + /* Stop the dentry being negated if it's only pinned by a file struct. */ + dget(dentry); - inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); - - if (test_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->fscache.flags)) { - /* object allocation for the same key preemptively deleted this - * object's file so that it could create its own file */ - _debug("object preemptively buried"); - inode_unlock(d_inode(dir)); - ret = 0; - } else { - /* we need to check that our parent is _still_ our parent - it - * may have been renamed */ - if (dir == object->dentry->d_parent) { - ret = cachefiles_bury_object(cache, object, dir, - object->dentry, false, - FSCACHE_OBJECT_WAS_RETIRED); - } else { - /* it got moved, presumably by cachefilesd culling it, - * so it's no longer in the key path and we can ignore - * it */ - inode_unlock(d_inode(dir)); - ret = 0; - } - } - - dput(dir); - _leave(" = %d", ret); + inode_lock_nested(d_backing_inode(fan), I_MUTEX_PARENT); + ret = cachefiles_unlink(volume->cache, object, fan, dentry, why); + inode_unlock(d_backing_inode(fan)); + dput(dentry); return ret; } /* - * walk from the parent object to the child object through the backing - * filesystem, creating directories as we go + * Create a temporary file and leave it unattached and un-xattr'd until the + * time comes to discard the object from memory. */ -int cachefiles_walk_to_object(struct cachefiles_object *parent, - struct cachefiles_object *object, - const char *key, - struct cachefiles_xattr *auxdata) +struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) { - struct cachefiles_cache *cache; - struct dentry *dir, *next = NULL; - struct inode *inode; + struct cachefiles_volume *volume = object->volume; + struct cachefiles_cache *cache = volume->cache; + const struct cred *saved_cred; + struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; + struct file *file; struct path path; - const char *name; - int ret, nlen; - - _enter("OBJ%x{%pd},OBJ%x,%s,", - parent->fscache.debug_id, parent->dentry, - object->fscache.debug_id, key); - - cache = container_of(parent->fscache.cache, - struct cachefiles_cache, cache); - path.mnt = cache->mnt; - - ASSERT(parent->dentry); - ASSERT(d_backing_inode(parent->dentry)); - - if (!(d_is_dir(parent->dentry))) { - // TODO: convert file to dir - _leave("looking up in none directory"); - return -ENOBUFS; - } - - dir = dget(parent->dentry); - -advance: - /* attempt to transit the first directory component */ - name = key; - nlen = strlen(key); - - /* key ends in a double NUL */ - key = key + nlen + 1; - if (!*key) - key = NULL; - -lookup_again: - /* search the current directory for the element name */ - _debug("lookup '%s'", name); - - inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); + uint64_t ni_size = object->cookie->object_size; + long ret; - next = lookup_one_len(name, dir, nlen); - if (IS_ERR(next)) { - trace_cachefiles_lookup(object, next, NULL); - goto lookup_error; - } + ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); - inode = d_backing_inode(next); - trace_cachefiles_lookup(object, next, inode); - _debug("next -> %pd %s", next, inode ? "positive" : "negative"); - - if (!key) - object->new = !inode; - - /* if this element of the path doesn't exist, then the lookup phase - * failed, and we can release any readers in the certain knowledge that - * there's nothing for them to actually read */ - if (d_is_negative(next)) - fscache_object_lookup_negative(&object->fscache); - - /* we need to create the object if it's negative */ - if (key || object->type == FSCACHE_COOKIE_TYPE_INDEX) { - /* index objects and intervening tree levels must be subdirs */ - if (d_is_negative(next)) { - ret = cachefiles_has_space(cache, 1, 0); - if (ret < 0) - goto no_space_error; - - path.dentry = dir; - ret = security_path_mkdir(&path, next, 0); - if (ret < 0) - goto create_error; - ret = vfs_mkdir(&init_user_ns, d_inode(dir), next, 0); - if (!key) - trace_cachefiles_mkdir(object, next, ret); - if (ret < 0) - goto create_error; - - if (unlikely(d_unhashed(next))) { - dput(next); - inode_unlock(d_inode(dir)); - goto lookup_again; - } - ASSERT(d_backing_inode(next)); - - _debug("mkdir -> %pd{ino=%lu}", - next, d_backing_inode(next)->i_ino); - - } else if (!d_can_lookup(next)) { - pr_err("inode %lu is not a directory\n", - d_backing_inode(next)->i_ino); - ret = -ENOBUFS; - goto error; - } + cachefiles_begin_secure(cache, &saved_cred); - } else { - /* non-index objects start out life as files */ - if (d_is_negative(next)) { - ret = cachefiles_has_space(cache, 1, 0); - if (ret < 0) - goto no_space_error; - - path.dentry = dir; - ret = security_path_mknod(&path, next, S_IFREG, 0); - if (ret < 0) - goto create_error; - ret = vfs_create(&init_user_ns, d_inode(dir), next, - S_IFREG, true); - trace_cachefiles_create(object, next, ret); - if (ret < 0) - goto create_error; - - ASSERT(d_backing_inode(next)); - - _debug("create -> %pd{ino=%lu}", - next, d_backing_inode(next)->i_ino); - - } else if (!d_can_lookup(next) && - !d_is_reg(next) - ) { - pr_err("inode %lu is not a file or directory\n", - d_backing_inode(next)->i_ino); - ret = -ENOBUFS; - goto error; + path.mnt = cache->mnt; + ret = cachefiles_inject_write_error(); + if (ret == 0) + path.dentry = vfs_tmpfile(&init_user_ns, fan, S_IFREG, O_RDWR); + else + path.dentry = ERR_PTR(ret); + if (IS_ERR(path.dentry)) { + trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(path.dentry), + cachefiles_trace_tmpfile_error); + if (PTR_ERR(path.dentry) == -EIO) + cachefiles_io_error_obj(object, "Failed to create tmpfile"); + file = ERR_CAST(path.dentry); + goto out; + } + + trace_cachefiles_tmpfile(object, d_backing_inode(path.dentry)); + + if (!cachefiles_mark_inode_in_use(object, path.dentry)) { + file = ERR_PTR(-EBUSY); + goto out_dput; + } + + if (ni_size > 0) { + trace_cachefiles_trunc(object, d_backing_inode(path.dentry), 0, ni_size, + cachefiles_trunc_expand_tmpfile); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_truncate(&path, ni_size); + if (ret < 0) { + trace_cachefiles_vfs_error( + object, d_backing_inode(path.dentry), ret, + cachefiles_trace_trunc_error); + file = ERR_PTR(ret); + goto out_dput; } } - /* process the next component */ - if (key) { - _debug("advance"); - inode_unlock(d_inode(dir)); - dput(dir); - dir = next; - next = NULL; - goto advance; + file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT, + d_backing_inode(path.dentry), cache->cache_cred); + if (IS_ERR(file)) { + trace_cachefiles_vfs_error(object, d_backing_inode(path.dentry), + PTR_ERR(file), + cachefiles_trace_open_error); + goto out_dput; + } + if (unlikely(!file->f_op->read_iter) || + unlikely(!file->f_op->write_iter)) { + fput(file); + pr_notice("Cache does not support read_iter and write_iter\n"); + file = ERR_PTR(-EINVAL); } - /* we've found the object we were looking for */ - object->dentry = next; - - /* if we've found that the terminal object exists, then we need to - * check its attributes and delete it if it's out of date */ - if (!object->new) { - _debug("validate '%pd'", next); - - ret = cachefiles_check_object_xattr(object, auxdata); - if (ret == -ESTALE) { - /* delete the object (the deleter drops the directory - * mutex) */ - object->dentry = NULL; +out_dput: + dput(path.dentry); +out: + cachefiles_end_secure(cache, saved_cred); + return file; +} - ret = cachefiles_bury_object(cache, object, dir, next, - true, - FSCACHE_OBJECT_IS_STALE); - dput(next); - next = NULL; +/* + * Create a new file. + */ +static bool cachefiles_create_file(struct cachefiles_object *object) +{ + struct file *file; + int ret; - if (ret < 0) - goto delete_error; + ret = cachefiles_has_space(object->volume->cache, 1, 0, + cachefiles_has_space_for_create); + if (ret < 0) + return false; - _debug("redo lookup"); - fscache_object_retrying_stale(&object->fscache); - goto lookup_again; - } - } + file = cachefiles_create_tmpfile(object); + if (IS_ERR(file)) + return false; - /* note that we're now using this object */ - ret = cachefiles_mark_object_active(cache, object); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags); + set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + _debug("create -> %pD{ino=%lu}", file, file_inode(file)->i_ino); + object->file = file; + return true; +} - inode_unlock(d_inode(dir)); - dput(dir); - dir = NULL; +/* + * Open an existing file, checking its attributes and replacing it if it is + * stale. + */ +static bool cachefiles_open_file(struct cachefiles_object *object, + struct dentry *dentry) +{ + struct cachefiles_cache *cache = object->volume->cache; + struct file *file; + struct path path; + int ret; - if (ret == -ETIMEDOUT) - goto mark_active_timed_out; + _enter("%pd", dentry); - _debug("=== OBTAINED_OBJECT ==="); + if (!cachefiles_mark_inode_in_use(object, dentry)) + return false; - if (object->new) { - /* attach data to a newly constructed terminal object */ - ret = cachefiles_set_object_xattr(object, auxdata); - if (ret < 0) - goto check_error; - } else { - /* always update the atime on an object we've just looked up - * (this is used to keep track of culling, and atimes are only - * updated by read, write and readdir but not lookup or - * open) */ - path.dentry = next; - touch_atime(&path); - } - - /* open a file interface onto a data file */ - if (object->type != FSCACHE_COOKIE_TYPE_INDEX) { - if (d_is_reg(object->dentry)) { - const struct address_space_operations *aops; - - ret = -EPERM; - aops = d_backing_inode(object->dentry)->i_mapping->a_ops; - if (!aops->bmap) - goto check_error; - if (object->dentry->d_sb->s_blocksize > PAGE_SIZE) - goto check_error; - - object->backer = object->dentry; - } else { - BUG(); // TODO: open file in data-class subdir - } + /* We need to open a file interface onto a data file now as we can't do + * it on demand because writeback called from do_exit() sees + * current->fs == NULL - which breaks d_path() called from ext4 open. + */ + path.mnt = cache->mnt; + path.dentry = dentry; + file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT, + d_backing_inode(dentry), cache->cache_cred); + if (IS_ERR(file)) { + trace_cachefiles_vfs_error(object, d_backing_inode(dentry), + PTR_ERR(file), + cachefiles_trace_open_error); + goto error; } - object->new = 0; - fscache_obtained_object(&object->fscache); - - _leave(" = 0 [%lu]", d_backing_inode(object->dentry)->i_ino); - return 0; - -no_space_error: - fscache_object_mark_killed(&object->fscache, FSCACHE_OBJECT_NO_SPACE); -create_error: - _debug("create error %d", ret); - if (ret == -EIO) - cachefiles_io_error(cache, "Create/mkdir failed"); - goto error; + if (unlikely(!file->f_op->read_iter) || + unlikely(!file->f_op->write_iter)) { + pr_notice("Cache does not support read_iter and write_iter\n"); + goto error_fput; + } + _debug("file -> %pd positive", dentry); -mark_active_timed_out: - _debug("mark active timed out"); - goto release_dentry; + ret = cachefiles_check_auxdata(object, file); + if (ret < 0) + goto check_failed; -check_error: - _debug("check error %d", ret); - cachefiles_mark_object_inactive( - cache, object, d_backing_inode(object->dentry)->i_blocks); -release_dentry: - dput(object->dentry); - object->dentry = NULL; - goto error_out; - -delete_error: - _debug("delete error %d", ret); - goto error_out2; + object->file = file; -lookup_error: - _debug("lookup error %ld", PTR_ERR(next)); - ret = PTR_ERR(next); - if (ret == -EIO) - cachefiles_io_error(cache, "Lookup failed"); - next = NULL; + /* Always update the atime on an object we've just looked up (this is + * used to keep track of culling, and atimes are only updated by read, + * write and readdir but not lookup or open). + */ + touch_atime(&file->f_path); + dput(dentry); + return true; + +check_failed: + fscache_cookie_lookup_negative(object->cookie); + cachefiles_unmark_inode_in_use(object, file); + if (ret == -ESTALE) { + fput(file); + dput(dentry); + return cachefiles_create_file(object); + } +error_fput: + fput(file); error: - inode_unlock(d_inode(dir)); - dput(next); -error_out2: - dput(dir); -error_out: - _leave(" = error %d", -ret); - return ret; + dput(dentry); + return false; } /* - * get a subdirectory + * walk from the parent object to the child object through the backing + * filesystem, creating directories as we go */ -struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, - struct dentry *dir, - const char *dirname) +bool cachefiles_look_up_object(struct cachefiles_object *object) { - struct dentry *subdir; - struct path path; + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; int ret; - _enter(",,%s", dirname); - - /* search the current directory for the element name */ - inode_lock(d_inode(dir)); - -retry: - subdir = lookup_one_len(dirname, dir, strlen(dirname)); - if (IS_ERR(subdir)) { - if (PTR_ERR(subdir) == -ENOMEM) - goto nomem_d_alloc; - goto lookup_error; + _enter("OBJ%x,%s,", object->debug_id, object->d_name); + + /* Look up path "cache/vol/fanout/file". */ + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = lookup_positive_unlocked(object->d_name, fan, + object->d_name_len); + else + dentry = ERR_PTR(ret); + trace_cachefiles_lookup(object, dentry); + if (IS_ERR(dentry)) { + if (dentry == ERR_PTR(-ENOENT)) + goto new_file; + if (dentry == ERR_PTR(-EIO)) + cachefiles_io_error_obj(object, "Lookup failed"); + return false; + } + + if (!d_is_reg(dentry)) { + pr_err("%pd is not a file\n", dentry); + inode_lock_nested(d_inode(fan), I_MUTEX_PARENT); + ret = cachefiles_bury_object(volume->cache, object, fan, dentry, + FSCACHE_OBJECT_IS_WEIRD); + dput(dentry); + if (ret < 0) + return false; + goto new_file; } - _debug("subdir -> %pd %s", - subdir, d_backing_inode(subdir) ? "positive" : "negative"); + if (!cachefiles_open_file(object, dentry)) + return false; - /* we need to create the subdir if it doesn't exist yet */ - if (d_is_negative(subdir)) { - ret = cachefiles_has_space(cache, 1, 0); - if (ret < 0) - goto mkdir_error; + _leave(" = t [%lu]", file_inode(object->file)->i_ino); + return true; - _debug("attempt mkdir"); +new_file: + fscache_cookie_lookup_negative(object->cookie); + return cachefiles_create_file(object); +} - path.mnt = cache->mnt; - path.dentry = dir; - ret = security_path_mkdir(&path, subdir, 0700); - if (ret < 0) - goto mkdir_error; - ret = vfs_mkdir(&init_user_ns, d_inode(dir), subdir, 0700); - if (ret < 0) - goto mkdir_error; +/* + * Attempt to link a temporary file into its rightful place in the cache. + */ +bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, + struct cachefiles_object *object) +{ + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; + bool success = false; + int ret; - if (unlikely(d_unhashed(subdir))) { - dput(subdir); - goto retry; + _enter(",%pD", object->file); + + inode_lock_nested(d_inode(fan), I_MUTEX_PARENT); + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = lookup_one_len(object->d_name, fan, object->d_name_len); + else + dentry = ERR_PTR(ret); + if (IS_ERR(dentry)) { + trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), + cachefiles_trace_lookup_error); + _debug("lookup fail %ld", PTR_ERR(dentry)); + goto out_unlock; + } + + if (!d_is_negative(dentry)) { + if (d_backing_inode(dentry) == file_inode(object->file)) { + success = true; + goto out_dput; } - ASSERT(d_backing_inode(subdir)); - _debug("mkdir -> %pd{ino=%lu}", - subdir, d_backing_inode(subdir)->i_ino); - } - - inode_unlock(d_inode(dir)); - - /* we need to make sure the subdir is a directory */ - ASSERT(d_backing_inode(subdir)); + ret = cachefiles_unlink(volume->cache, object, fan, dentry, + FSCACHE_OBJECT_IS_STALE); + if (ret < 0) + goto out_dput; - if (!d_can_lookup(subdir)) { - pr_err("%s is not a directory\n", dirname); - ret = -EIO; - goto check_error; + dput(dentry); + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = lookup_one_len(object->d_name, fan, object->d_name_len); + else + dentry = ERR_PTR(ret); + if (IS_ERR(dentry)) { + trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), + cachefiles_trace_lookup_error); + _debug("lookup fail %ld", PTR_ERR(dentry)); + goto out_unlock; + } } - ret = -EPERM; - if (!(d_backing_inode(subdir)->i_opflags & IOP_XATTR) || - !d_backing_inode(subdir)->i_op->lookup || - !d_backing_inode(subdir)->i_op->mkdir || - !d_backing_inode(subdir)->i_op->create || - !d_backing_inode(subdir)->i_op->rename || - !d_backing_inode(subdir)->i_op->rmdir || - !d_backing_inode(subdir)->i_op->unlink) - goto check_error; - - _leave(" = [%lu]", d_backing_inode(subdir)->i_ino); - return subdir; - -check_error: - dput(subdir); - _leave(" = %d [check]", ret); - return ERR_PTR(ret); - -mkdir_error: - inode_unlock(d_inode(dir)); - dput(subdir); - pr_err("mkdir %s failed with error %d\n", dirname, ret); - return ERR_PTR(ret); - -lookup_error: - inode_unlock(d_inode(dir)); - ret = PTR_ERR(subdir); - pr_err("Lookup %s failed with error %d\n", dirname, ret); - return ERR_PTR(ret); - -nomem_d_alloc: - inode_unlock(d_inode(dir)); - _leave(" = -ENOMEM"); - return ERR_PTR(-ENOMEM); + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_link(object->file->f_path.dentry, &init_user_ns, + d_inode(fan), dentry, NULL); + if (ret < 0) { + trace_cachefiles_vfs_error(object, d_inode(fan), ret, + cachefiles_trace_link_error); + _debug("link fail %d", ret); + } else { + trace_cachefiles_link(object, file_inode(object->file)); + spin_lock(&object->lock); + /* TODO: Do we want to switch the file pointer to the new dentry? */ + clear_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + spin_unlock(&object->lock); + success = true; + } + +out_dput: + dput(dentry); +out_unlock: + inode_unlock(d_inode(fan)); + _leave(" = %u", success); + return success; } /* - * find out if an object is in use or not - * - if finds object and it's not in use: - * - returns a pointer to the object and a reference on it - * - returns with the directory locked + * Look up an inode to be checked or culled. Return -EBUSY if the inode is + * marked in use. */ -static struct dentry *cachefiles_check_active(struct cachefiles_cache *cache, - struct dentry *dir, - char *filename) +static struct dentry *cachefiles_lookup_for_cull(struct cachefiles_cache *cache, + struct dentry *dir, + char *filename) { - struct cachefiles_object *object; - struct rb_node *_n; struct dentry *victim; - int ret; - - //_enter(",%pd/,%s", - // dir, filename); + int ret = -ENOENT; - /* look up the victim */ inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); victim = lookup_one_len(filename, dir, strlen(filename)); if (IS_ERR(victim)) goto lookup_error; - - //_debug("victim -> %pd %s", - // victim, d_backing_inode(victim) ? "positive" : "negative"); - - /* if the object is no longer there then we probably retired the object - * at the netfs's request whilst the cull was in progress - */ - if (d_is_negative(victim)) { - inode_unlock(d_inode(dir)); - dput(victim); - _leave(" = -ENOENT [absent]"); - return ERR_PTR(-ENOENT); - } - - /* check to see if we're using this object */ - read_lock(&cache->active_lock); - - _n = cache->active_nodes.rb_node; - - while (_n) { - object = rb_entry(_n, struct cachefiles_object, active_node); - - if (object->dentry > victim) - _n = _n->rb_left; - else if (object->dentry < victim) - _n = _n->rb_right; - else - goto object_in_use; - } - - read_unlock(&cache->active_lock); - - //_leave(" = %pd", victim); + if (d_is_negative(victim)) + goto lookup_put; + if (d_inode(victim)->i_flags & S_KERNEL_FILE) + goto lookup_busy; return victim; -object_in_use: - read_unlock(&cache->active_lock); +lookup_busy: + ret = -EBUSY; +lookup_put: inode_unlock(d_inode(dir)); dput(victim); - //_leave(" = -EBUSY [in use]"); - return ERR_PTR(-EBUSY); + return ERR_PTR(ret); lookup_error: inode_unlock(d_inode(dir)); ret = PTR_ERR(victim); - if (ret == -ENOENT) { - /* file or dir now absent - probably retired by netfs */ - _leave(" = -ESTALE [absent]"); - return ERR_PTR(-ESTALE); - } + if (ret == -ENOENT) + return ERR_PTR(-ESTALE); /* Probably got retired by the netfs */ if (ret == -EIO) { cachefiles_io_error(cache, "Lookup failed"); @@ -931,46 +766,46 @@ lookup_error: ret = -EIO; } - _leave(" = %d", ret); return ERR_PTR(ret); } /* - * cull an object if it's not in use + * Cull an object if it's not in use * - called only by cache manager daemon */ int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, char *filename) { struct dentry *victim; + struct inode *inode; int ret; _enter(",%pd/,%s", dir, filename); - victim = cachefiles_check_active(cache, dir, filename); + victim = cachefiles_lookup_for_cull(cache, dir, filename); if (IS_ERR(victim)) return PTR_ERR(victim); - _debug("victim -> %pd %s", - victim, d_backing_inode(victim) ? "positive" : "negative"); - - /* okay... the victim is not being used so we can cull it - * - start by marking it as stale - */ - _debug("victim is cullable"); - - ret = cachefiles_remove_object_xattr(cache, victim); + /* check to see if someone is using this object */ + inode = d_inode(victim); + inode_lock(inode); + if (inode->i_flags & S_KERNEL_FILE) { + ret = -EBUSY; + } else { + /* Stop the cache from picking it back up */ + inode->i_flags |= S_KERNEL_FILE; + ret = 0; + } + inode_unlock(inode); if (ret < 0) goto error_unlock; - /* actually remove the victim (drops the dir mutex) */ - _debug("bury"); - - ret = cachefiles_bury_object(cache, NULL, dir, victim, false, + ret = cachefiles_bury_object(cache, NULL, dir, victim, FSCACHE_OBJECT_WAS_CULLED); if (ret < 0) goto error; + fscache_count_culled(); dput(victim); _leave(" = 0"); return 0; @@ -979,11 +814,8 @@ error_unlock: inode_unlock(d_inode(dir)); error: dput(victim); - if (ret == -ENOENT) { - /* file or dir now absent - probably retired by netfs */ - _leave(" = -ESTALE [absent]"); - return -ESTALE; - } + if (ret == -ENOENT) + return -ESTALE; /* Probably got retired by the netfs */ if (ret != -ENOMEM) { pr_err("Internal error: %d\n", ret); @@ -995,7 +827,7 @@ error: } /* - * find out if an object is in use or not + * Find out if an object is in use or not * - called only by cache manager daemon * - returns -EBUSY or 0 to indicate whether an object is in use or not */ @@ -1003,16 +835,13 @@ int cachefiles_check_in_use(struct cachefiles_cache *cache, struct dentry *dir, char *filename) { struct dentry *victim; + int ret = 0; - //_enter(",%pd/,%s", - // dir, filename); - - victim = cachefiles_check_active(cache, dir, filename); + victim = cachefiles_lookup_for_cull(cache, dir, filename); if (IS_ERR(victim)) return PTR_ERR(victim); inode_unlock(d_inode(dir)); dput(victim); - //_leave(" = 0"); - return 0; + return ret; } diff --git a/fs/cachefiles/rdwr.c b/fs/cachefiles/rdwr.c deleted file mode 100644 index fcf4f3b72923..000000000000 --- a/fs/cachefiles/rdwr.c +++ /dev/null @@ -1,972 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Storage object read/write - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include <linux/mount.h> -#include <linux/slab.h> -#include <linux/file.h> -#include <linux/swap.h> -#include "internal.h" - -/* - * detect wake up events generated by the unlocking of pages in which we're - * interested - * - we use this to detect read completion of backing pages - * - the caller holds the waitqueue lock - */ -static int cachefiles_read_waiter(wait_queue_entry_t *wait, unsigned mode, - int sync, void *_key) -{ - struct cachefiles_one_read *monitor = - container_of(wait, struct cachefiles_one_read, monitor); - struct cachefiles_object *object; - struct fscache_retrieval *op = monitor->op; - struct wait_page_key *key = _key; - struct folio *folio = wait->private; - - ASSERT(key); - - _enter("{%lu},%u,%d,{%p,%u}", - monitor->netfs_page->index, mode, sync, - key->folio, key->bit_nr); - - if (key->folio != folio || key->bit_nr != PG_locked) - return 0; - - _debug("--- monitor %p %lx ---", folio, folio->flags); - - if (!folio_test_uptodate(folio) && !folio_test_error(folio)) { - /* unlocked, not uptodate and not erronous? */ - _debug("page probably truncated"); - } - - /* remove from the waitqueue */ - list_del(&wait->entry); - - /* move onto the action list and queue for FS-Cache thread pool */ - ASSERT(op); - - /* We need to temporarily bump the usage count as we don't own a ref - * here otherwise cachefiles_read_copier() may free the op between the - * monitor being enqueued on the op->to_do list and the op getting - * enqueued on the work queue. - */ - fscache_get_retrieval(op); - - object = container_of(op->op.object, struct cachefiles_object, fscache); - spin_lock(&object->work_lock); - list_add_tail(&monitor->op_link, &op->to_do); - fscache_enqueue_retrieval(op); - spin_unlock(&object->work_lock); - - fscache_put_retrieval(op); - return 0; -} - -/* - * handle a probably truncated page - * - check to see if the page is still relevant and reissue the read if - * possible - * - return -EIO on error, -ENODATA if the page is gone, -EINPROGRESS if we - * must wait again and 0 if successful - */ -static int cachefiles_read_reissue(struct cachefiles_object *object, - struct cachefiles_one_read *monitor) -{ - struct address_space *bmapping = d_backing_inode(object->backer)->i_mapping; - struct page *backpage = monitor->back_page, *backpage2; - int ret; - - _enter("{ino=%lx},{%lx,%lx}", - d_backing_inode(object->backer)->i_ino, - backpage->index, backpage->flags); - - /* skip if the page was truncated away completely */ - if (backpage->mapping != bmapping) { - _leave(" = -ENODATA [mapping]"); - return -ENODATA; - } - - backpage2 = find_get_page(bmapping, backpage->index); - if (!backpage2) { - _leave(" = -ENODATA [gone]"); - return -ENODATA; - } - - if (backpage != backpage2) { - put_page(backpage2); - _leave(" = -ENODATA [different]"); - return -ENODATA; - } - - /* the page is still there and we already have a ref on it, so we don't - * need a second */ - put_page(backpage2); - - INIT_LIST_HEAD(&monitor->op_link); - folio_add_wait_queue(page_folio(backpage), &monitor->monitor); - - if (trylock_page(backpage)) { - ret = -EIO; - if (PageError(backpage)) - goto unlock_discard; - ret = 0; - if (PageUptodate(backpage)) - goto unlock_discard; - - _debug("reissue read"); - ret = bmapping->a_ops->readpage(NULL, backpage); - if (ret < 0) - goto discard; - } - - /* but the page may have been read before the monitor was installed, so - * the monitor may miss the event - so we have to ensure that we do get - * one in such a case */ - if (trylock_page(backpage)) { - _debug("jumpstart %p {%lx}", backpage, backpage->flags); - unlock_page(backpage); - } - - /* it'll reappear on the todo list */ - _leave(" = -EINPROGRESS"); - return -EINPROGRESS; - -unlock_discard: - unlock_page(backpage); -discard: - spin_lock_irq(&object->work_lock); - list_del(&monitor->op_link); - spin_unlock_irq(&object->work_lock); - _leave(" = %d", ret); - return ret; -} - -/* - * copy data from backing pages to netfs pages to complete a read operation - * - driven by FS-Cache's thread pool - */ -static void cachefiles_read_copier(struct fscache_operation *_op) -{ - struct cachefiles_one_read *monitor; - struct cachefiles_object *object; - struct fscache_retrieval *op; - int error, max; - - op = container_of(_op, struct fscache_retrieval, op); - object = container_of(op->op.object, - struct cachefiles_object, fscache); - - _enter("{ino=%lu}", d_backing_inode(object->backer)->i_ino); - - max = 8; - spin_lock_irq(&object->work_lock); - - while (!list_empty(&op->to_do)) { - monitor = list_entry(op->to_do.next, - struct cachefiles_one_read, op_link); - list_del(&monitor->op_link); - - spin_unlock_irq(&object->work_lock); - - _debug("- copy {%lu}", monitor->back_page->index); - - recheck: - if (test_bit(FSCACHE_COOKIE_INVALIDATING, - &object->fscache.cookie->flags)) { - error = -ESTALE; - } else if (PageUptodate(monitor->back_page)) { - copy_highpage(monitor->netfs_page, monitor->back_page); - fscache_mark_page_cached(monitor->op, - monitor->netfs_page); - error = 0; - } else if (!PageError(monitor->back_page)) { - /* the page has probably been truncated */ - error = cachefiles_read_reissue(object, monitor); - if (error == -EINPROGRESS) - goto next; - goto recheck; - } else { - cachefiles_io_error_obj( - object, - "Readpage failed on backing file %lx", - (unsigned long) monitor->back_page->flags); - error = -EIO; - } - - put_page(monitor->back_page); - - fscache_end_io(op, monitor->netfs_page, error); - put_page(monitor->netfs_page); - fscache_retrieval_complete(op, 1); - fscache_put_retrieval(op); - kfree(monitor); - - next: - /* let the thread pool have some air occasionally */ - max--; - if (max < 0 || need_resched()) { - if (!list_empty(&op->to_do)) - fscache_enqueue_retrieval(op); - _leave(" [maxed out]"); - return; - } - - spin_lock_irq(&object->work_lock); - } - - spin_unlock_irq(&object->work_lock); - _leave(""); -} - -/* - * read the corresponding page to the given set from the backing file - * - an uncertain page is simply discarded, to be tried again another time - */ -static int cachefiles_read_backing_file_one(struct cachefiles_object *object, - struct fscache_retrieval *op, - struct page *netpage) -{ - struct cachefiles_one_read *monitor; - struct address_space *bmapping; - struct page *newpage, *backpage; - int ret; - - _enter(""); - - _debug("read back %p{%lu,%d}", - netpage, netpage->index, page_count(netpage)); - - monitor = kzalloc(sizeof(*monitor), cachefiles_gfp); - if (!monitor) - goto nomem; - - monitor->netfs_page = netpage; - monitor->op = fscache_get_retrieval(op); - - init_waitqueue_func_entry(&monitor->monitor, cachefiles_read_waiter); - - /* attempt to get hold of the backing page */ - bmapping = d_backing_inode(object->backer)->i_mapping; - newpage = NULL; - - for (;;) { - backpage = find_get_page(bmapping, netpage->index); - if (backpage) - goto backing_page_already_present; - - if (!newpage) { - newpage = __page_cache_alloc(cachefiles_gfp); - if (!newpage) - goto nomem_monitor; - } - - ret = add_to_page_cache_lru(newpage, bmapping, - netpage->index, cachefiles_gfp); - if (ret == 0) - goto installed_new_backing_page; - if (ret != -EEXIST) - goto nomem_page; - } - - /* we've installed a new backing page, so now we need to start - * it reading */ -installed_new_backing_page: - _debug("- new %p", newpage); - - backpage = newpage; - newpage = NULL; - -read_backing_page: - ret = bmapping->a_ops->readpage(NULL, backpage); - if (ret < 0) - goto read_error; - - /* set the monitor to transfer the data across */ -monitor_backing_page: - _debug("- monitor add"); - - /* install the monitor */ - get_page(monitor->netfs_page); - get_page(backpage); - monitor->back_page = backpage; - monitor->monitor.private = backpage; - folio_add_wait_queue(page_folio(backpage), &monitor->monitor); - monitor = NULL; - - /* but the page may have been read before the monitor was installed, so - * the monitor may miss the event - so we have to ensure that we do get - * one in such a case */ - if (trylock_page(backpage)) { - _debug("jumpstart %p {%lx}", backpage, backpage->flags); - unlock_page(backpage); - } - goto success; - - /* if the backing page is already present, it can be in one of - * three states: read in progress, read failed or read okay */ -backing_page_already_present: - _debug("- present"); - - if (newpage) { - put_page(newpage); - newpage = NULL; - } - - if (PageError(backpage)) - goto io_error; - - if (PageUptodate(backpage)) - goto backing_page_already_uptodate; - - if (!trylock_page(backpage)) - goto monitor_backing_page; - _debug("read %p {%lx}", backpage, backpage->flags); - goto read_backing_page; - - /* the backing page is already up to date, attach the netfs - * page to the pagecache and LRU and copy the data across */ -backing_page_already_uptodate: - _debug("- uptodate"); - - fscache_mark_page_cached(op, netpage); - - copy_highpage(netpage, backpage); - fscache_end_io(op, netpage, 0); - fscache_retrieval_complete(op, 1); - -success: - _debug("success"); - ret = 0; - -out: - if (backpage) - put_page(backpage); - if (monitor) { - fscache_put_retrieval(monitor->op); - kfree(monitor); - } - _leave(" = %d", ret); - return ret; - -read_error: - _debug("read error %d", ret); - if (ret == -ENOMEM) { - fscache_retrieval_complete(op, 1); - goto out; - } -io_error: - cachefiles_io_error_obj(object, "Page read error on backing file"); - fscache_retrieval_complete(op, 1); - ret = -ENOBUFS; - goto out; - -nomem_page: - put_page(newpage); -nomem_monitor: - fscache_put_retrieval(monitor->op); - kfree(monitor); -nomem: - fscache_retrieval_complete(op, 1); - _leave(" = -ENOMEM"); - return -ENOMEM; -} - -/* - * read a page from the cache or allocate a block in which to store it - * - cache withdrawal is prevented by the caller - * - returns -EINTR if interrupted - * - returns -ENOMEM if ran out of memory - * - returns -ENOBUFS if no buffers can be made available - * - returns -ENOBUFS if page is beyond EOF - * - if the page is backed by a block in the cache: - * - a read will be started which will call the callback on completion - * - 0 will be returned - * - else if the page is unbacked: - * - the metadata will be retained - * - -ENODATA will be returned - */ -int cachefiles_read_or_alloc_page(struct fscache_retrieval *op, - struct page *page, - gfp_t gfp) -{ - struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct inode *inode; - sector_t block; - unsigned shift; - int ret, ret2; - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - _enter("{%p},{%lx},,,", object, page->index); - - if (!object->backer) - goto enobufs; - - inode = d_backing_inode(object->backer); - ASSERT(S_ISREG(inode->i_mode)); - - /* calculate the shift required to use bmap */ - shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits; - - op->op.flags &= FSCACHE_OP_KEEP_FLAGS; - op->op.flags |= FSCACHE_OP_ASYNC; - op->op.processor = cachefiles_read_copier; - - /* we assume the absence or presence of the first block is a good - * enough indication for the page as a whole - * - TODO: don't use bmap() for this as it is _not_ actually good - * enough for this as it doesn't indicate errors, but it's all we've - * got for the moment - */ - block = page->index; - block <<= shift; - - ret2 = bmap(inode, &block); - ASSERT(ret2 == 0); - - _debug("%llx -> %llx", - (unsigned long long) (page->index << shift), - (unsigned long long) block); - - if (block) { - /* submit the apparently valid page to the backing fs to be - * read from disk */ - ret = cachefiles_read_backing_file_one(object, op, page); - } else if (cachefiles_has_space(cache, 0, 1) == 0) { - /* there's space in the cache we can use */ - fscache_mark_page_cached(op, page); - fscache_retrieval_complete(op, 1); - ret = -ENODATA; - } else { - goto enobufs; - } - - _leave(" = %d", ret); - return ret; - -enobufs: - fscache_retrieval_complete(op, 1); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} - -/* - * read the corresponding pages to the given set from the backing file - * - any uncertain pages are simply discarded, to be tried again another time - */ -static int cachefiles_read_backing_file(struct cachefiles_object *object, - struct fscache_retrieval *op, - struct list_head *list) -{ - struct cachefiles_one_read *monitor = NULL; - struct address_space *bmapping = d_backing_inode(object->backer)->i_mapping; - struct page *newpage = NULL, *netpage, *_n, *backpage = NULL; - int ret = 0; - - _enter(""); - - list_for_each_entry_safe(netpage, _n, list, lru) { - list_del(&netpage->lru); - - _debug("read back %p{%lu,%d}", - netpage, netpage->index, page_count(netpage)); - - if (!monitor) { - monitor = kzalloc(sizeof(*monitor), cachefiles_gfp); - if (!monitor) - goto nomem; - - monitor->op = fscache_get_retrieval(op); - init_waitqueue_func_entry(&monitor->monitor, - cachefiles_read_waiter); - } - - for (;;) { - backpage = find_get_page(bmapping, netpage->index); - if (backpage) - goto backing_page_already_present; - - if (!newpage) { - newpage = __page_cache_alloc(cachefiles_gfp); - if (!newpage) - goto nomem; - } - - ret = add_to_page_cache_lru(newpage, bmapping, - netpage->index, - cachefiles_gfp); - if (ret == 0) - goto installed_new_backing_page; - if (ret != -EEXIST) - goto nomem; - } - - /* we've installed a new backing page, so now we need - * to start it reading */ - installed_new_backing_page: - _debug("- new %p", newpage); - - backpage = newpage; - newpage = NULL; - - reread_backing_page: - ret = bmapping->a_ops->readpage(NULL, backpage); - if (ret < 0) - goto read_error; - - /* add the netfs page to the pagecache and LRU, and set the - * monitor to transfer the data across */ - monitor_backing_page: - _debug("- monitor add"); - - ret = add_to_page_cache_lru(netpage, op->mapping, - netpage->index, cachefiles_gfp); - if (ret < 0) { - if (ret == -EEXIST) { - put_page(backpage); - backpage = NULL; - put_page(netpage); - netpage = NULL; - fscache_retrieval_complete(op, 1); - continue; - } - goto nomem; - } - - /* install a monitor */ - get_page(netpage); - monitor->netfs_page = netpage; - - get_page(backpage); - monitor->back_page = backpage; - monitor->monitor.private = backpage; - folio_add_wait_queue(page_folio(backpage), &monitor->monitor); - monitor = NULL; - - /* but the page may have been read before the monitor was - * installed, so the monitor may miss the event - so we have to - * ensure that we do get one in such a case */ - if (trylock_page(backpage)) { - _debug("2unlock %p {%lx}", backpage, backpage->flags); - unlock_page(backpage); - } - - put_page(backpage); - backpage = NULL; - - put_page(netpage); - netpage = NULL; - continue; - - /* if the backing page is already present, it can be in one of - * three states: read in progress, read failed or read okay */ - backing_page_already_present: - _debug("- present %p", backpage); - - if (PageError(backpage)) - goto io_error; - - if (PageUptodate(backpage)) - goto backing_page_already_uptodate; - - _debug("- not ready %p{%lx}", backpage, backpage->flags); - - if (!trylock_page(backpage)) - goto monitor_backing_page; - - if (PageError(backpage)) { - _debug("error %lx", backpage->flags); - unlock_page(backpage); - goto io_error; - } - - if (PageUptodate(backpage)) - goto backing_page_already_uptodate_unlock; - - /* we've locked a page that's neither up to date nor erroneous, - * so we need to attempt to read it again */ - goto reread_backing_page; - - /* the backing page is already up to date, attach the netfs - * page to the pagecache and LRU and copy the data across */ - backing_page_already_uptodate_unlock: - _debug("uptodate %lx", backpage->flags); - unlock_page(backpage); - backing_page_already_uptodate: - _debug("- uptodate"); - - ret = add_to_page_cache_lru(netpage, op->mapping, - netpage->index, cachefiles_gfp); - if (ret < 0) { - if (ret == -EEXIST) { - put_page(backpage); - backpage = NULL; - put_page(netpage); - netpage = NULL; - fscache_retrieval_complete(op, 1); - continue; - } - goto nomem; - } - - copy_highpage(netpage, backpage); - - put_page(backpage); - backpage = NULL; - - fscache_mark_page_cached(op, netpage); - - /* the netpage is unlocked and marked up to date here */ - fscache_end_io(op, netpage, 0); - put_page(netpage); - netpage = NULL; - fscache_retrieval_complete(op, 1); - continue; - } - - netpage = NULL; - - _debug("out"); - -out: - /* tidy up */ - if (newpage) - put_page(newpage); - if (netpage) - put_page(netpage); - if (backpage) - put_page(backpage); - if (monitor) { - fscache_put_retrieval(op); - kfree(monitor); - } - - list_for_each_entry_safe(netpage, _n, list, lru) { - list_del(&netpage->lru); - put_page(netpage); - fscache_retrieval_complete(op, 1); - } - - _leave(" = %d", ret); - return ret; - -nomem: - _debug("nomem"); - ret = -ENOMEM; - goto record_page_complete; - -read_error: - _debug("read error %d", ret); - if (ret == -ENOMEM) - goto record_page_complete; -io_error: - cachefiles_io_error_obj(object, "Page read error on backing file"); - ret = -ENOBUFS; -record_page_complete: - fscache_retrieval_complete(op, 1); - goto out; -} - -/* - * read a list of pages from the cache or allocate blocks in which to store - * them - */ -int cachefiles_read_or_alloc_pages(struct fscache_retrieval *op, - struct list_head *pages, - unsigned *nr_pages, - gfp_t gfp) -{ - struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct list_head backpages; - struct pagevec pagevec; - struct inode *inode; - struct page *page, *_n; - unsigned shift, nrbackpages; - int ret, ret2, space; - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - _enter("{OBJ%x,%d},,%d,,", - object->fscache.debug_id, atomic_read(&op->op.usage), - *nr_pages); - - if (!object->backer) - goto all_enobufs; - - space = 1; - if (cachefiles_has_space(cache, 0, *nr_pages) < 0) - space = 0; - - inode = d_backing_inode(object->backer); - ASSERT(S_ISREG(inode->i_mode)); - - /* calculate the shift required to use bmap */ - shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits; - - pagevec_init(&pagevec); - - op->op.flags &= FSCACHE_OP_KEEP_FLAGS; - op->op.flags |= FSCACHE_OP_ASYNC; - op->op.processor = cachefiles_read_copier; - - INIT_LIST_HEAD(&backpages); - nrbackpages = 0; - - ret = space ? -ENODATA : -ENOBUFS; - list_for_each_entry_safe(page, _n, pages, lru) { - sector_t block; - - /* we assume the absence or presence of the first block is a - * good enough indication for the page as a whole - * - TODO: don't use bmap() for this as it is _not_ actually - * good enough for this as it doesn't indicate errors, but - * it's all we've got for the moment - */ - block = page->index; - block <<= shift; - - ret2 = bmap(inode, &block); - ASSERT(ret2 == 0); - - _debug("%llx -> %llx", - (unsigned long long) (page->index << shift), - (unsigned long long) block); - - if (block) { - /* we have data - add it to the list to give to the - * backing fs */ - list_move(&page->lru, &backpages); - (*nr_pages)--; - nrbackpages++; - } else if (space && pagevec_add(&pagevec, page) == 0) { - fscache_mark_pages_cached(op, &pagevec); - fscache_retrieval_complete(op, 1); - ret = -ENODATA; - } else { - fscache_retrieval_complete(op, 1); - } - } - - if (pagevec_count(&pagevec) > 0) - fscache_mark_pages_cached(op, &pagevec); - - if (list_empty(pages)) - ret = 0; - - /* submit the apparently valid pages to the backing fs to be read from - * disk */ - if (nrbackpages > 0) { - ret2 = cachefiles_read_backing_file(object, op, &backpages); - if (ret2 == -ENOMEM || ret2 == -EINTR) - ret = ret2; - } - - _leave(" = %d [nr=%u%s]", - ret, *nr_pages, list_empty(pages) ? " empty" : ""); - return ret; - -all_enobufs: - fscache_retrieval_complete(op, *nr_pages); - return -ENOBUFS; -} - -/* - * allocate a block in the cache in which to store a page - * - cache withdrawal is prevented by the caller - * - returns -EINTR if interrupted - * - returns -ENOMEM if ran out of memory - * - returns -ENOBUFS if no buffers can be made available - * - returns -ENOBUFS if page is beyond EOF - * - otherwise: - * - the metadata will be retained - * - 0 will be returned - */ -int cachefiles_allocate_page(struct fscache_retrieval *op, - struct page *page, - gfp_t gfp) -{ - struct cachefiles_object *object; - struct cachefiles_cache *cache; - int ret; - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - _enter("%p,{%lx},", object, page->index); - - ret = cachefiles_has_space(cache, 0, 1); - if (ret == 0) - fscache_mark_page_cached(op, page); - else - ret = -ENOBUFS; - - fscache_retrieval_complete(op, 1); - _leave(" = %d", ret); - return ret; -} - -/* - * allocate blocks in the cache in which to store a set of pages - * - cache withdrawal is prevented by the caller - * - returns -EINTR if interrupted - * - returns -ENOMEM if ran out of memory - * - returns -ENOBUFS if some buffers couldn't be made available - * - returns -ENOBUFS if some pages are beyond EOF - * - otherwise: - * - -ENODATA will be returned - * - metadata will be retained for any page marked - */ -int cachefiles_allocate_pages(struct fscache_retrieval *op, - struct list_head *pages, - unsigned *nr_pages, - gfp_t gfp) -{ - struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct pagevec pagevec; - struct page *page; - int ret; - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - _enter("%p,,,%d,", object, *nr_pages); - - ret = cachefiles_has_space(cache, 0, *nr_pages); - if (ret == 0) { - pagevec_init(&pagevec); - - list_for_each_entry(page, pages, lru) { - if (pagevec_add(&pagevec, page) == 0) - fscache_mark_pages_cached(op, &pagevec); - } - - if (pagevec_count(&pagevec) > 0) - fscache_mark_pages_cached(op, &pagevec); - ret = -ENODATA; - } else { - ret = -ENOBUFS; - } - - fscache_retrieval_complete(op, *nr_pages); - _leave(" = %d", ret); - return ret; -} - -/* - * request a page be stored in the cache - * - cache withdrawal is prevented by the caller - * - this request may be ignored if there's no cache block available, in which - * case -ENOBUFS will be returned - * - if the op is in progress, 0 will be returned - */ -int cachefiles_write_page(struct fscache_storage *op, struct page *page) -{ - struct cachefiles_object *object; - struct cachefiles_cache *cache; - struct file *file; - struct path path; - loff_t pos, eof; - size_t len; - void *data; - int ret = -ENOBUFS; - - ASSERT(op != NULL); - ASSERT(page != NULL); - - object = container_of(op->op.object, - struct cachefiles_object, fscache); - - _enter("%p,%p{%lx},,,", object, page, page->index); - - if (!object->backer) { - _leave(" = -ENOBUFS"); - return -ENOBUFS; - } - - ASSERT(d_is_reg(object->backer)); - - cache = container_of(object->fscache.cache, - struct cachefiles_cache, cache); - - pos = (loff_t)page->index << PAGE_SHIFT; - - /* We mustn't write more data than we have, so we have to beware of a - * partial page at EOF. - */ - eof = object->fscache.store_limit_l; - if (pos >= eof) - goto error; - - /* write the page to the backing filesystem and let it store it in its - * own time */ - path.mnt = cache->mnt; - path.dentry = object->backer; - file = dentry_open(&path, O_RDWR | O_LARGEFILE, cache->cache_cred); - if (IS_ERR(file)) { - ret = PTR_ERR(file); - goto error_2; - } - - len = PAGE_SIZE; - if (eof & ~PAGE_MASK) { - if (eof - pos < PAGE_SIZE) { - _debug("cut short %llx to %llx", - pos, eof); - len = eof - pos; - ASSERTCMP(pos + len, ==, eof); - } - } - - data = kmap(page); - ret = kernel_write(file, data, len, &pos); - kunmap(page); - fput(file); - if (ret != len) - goto error_eio; - - _leave(" = 0"); - return 0; - -error_eio: - ret = -EIO; -error_2: - if (ret == -EIO) - cachefiles_io_error_obj(object, - "Write page to backing file failed"); -error: - _leave(" = -ENOBUFS [%d]", ret); - return -ENOBUFS; -} - -/* - * detach a backing block from a page - * - cache withdrawal is prevented by the caller - */ -void cachefiles_uncache_page(struct fscache_object *_object, struct page *page) - __releases(&object->fscache.cookie->lock) -{ - struct cachefiles_object *object; - - object = container_of(_object, struct cachefiles_object, fscache); - - _enter("%p,{%lu}", object, page->index); - - spin_unlock(&object->fscache.cookie->lock); -} diff --git a/fs/cachefiles/security.c b/fs/cachefiles/security.c index aec13fd94692..fe777164f1d8 100644 --- a/fs/cachefiles/security.c +++ b/fs/cachefiles/security.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* CacheFiles security management * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ diff --git a/fs/cachefiles/volume.c b/fs/cachefiles/volume.c new file mode 100644 index 000000000000..89df0ba8ba5e --- /dev/null +++ b/fs/cachefiles/volume.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/fs.h> +#include <linux/slab.h> +#include "internal.h" +#include <trace/events/fscache.h> + +/* + * Allocate and set up a volume representation. We make sure all the fanout + * directories are created and pinned. + */ +void cachefiles_acquire_volume(struct fscache_volume *vcookie) +{ + struct cachefiles_volume *volume; + struct cachefiles_cache *cache = vcookie->cache->cache_priv; + const struct cred *saved_cred; + struct dentry *vdentry, *fan; + size_t len; + char *name; + bool is_new = false; + int ret, n_accesses, i; + + _enter(""); + + volume = kzalloc(sizeof(struct cachefiles_volume), GFP_KERNEL); + if (!volume) + return; + volume->vcookie = vcookie; + volume->cache = cache; + INIT_LIST_HEAD(&volume->cache_link); + + cachefiles_begin_secure(cache, &saved_cred); + + len = vcookie->key[0]; + name = kmalloc(len + 3, GFP_NOFS); + if (!name) + goto error_vol; + name[0] = 'I'; + memcpy(name + 1, vcookie->key + 1, len); + name[len + 1] = 0; + +retry: + vdentry = cachefiles_get_directory(cache, cache->store, name, &is_new); + if (IS_ERR(vdentry)) + goto error_name; + volume->dentry = vdentry; + + if (is_new) { + if (!cachefiles_set_volume_xattr(volume)) + goto error_dir; + } else { + ret = cachefiles_check_volume_xattr(volume); + if (ret < 0) { + if (ret != -ESTALE) + goto error_dir; + inode_lock_nested(d_inode(cache->store), I_MUTEX_PARENT); + cachefiles_bury_object(cache, NULL, cache->store, vdentry, + FSCACHE_VOLUME_IS_WEIRD); + cachefiles_put_directory(volume->dentry); + cond_resched(); + goto retry; + } + } + + for (i = 0; i < 256; i++) { + sprintf(name, "@%02x", i); + fan = cachefiles_get_directory(cache, vdentry, name, NULL); + if (IS_ERR(fan)) + goto error_fan; + volume->fanout[i] = fan; + } + + cachefiles_end_secure(cache, saved_cred); + + vcookie->cache_priv = volume; + n_accesses = atomic_inc_return(&vcookie->n_accesses); /* Stop wakeups on dec-to-0 */ + trace_fscache_access_volume(vcookie->debug_id, 0, + refcount_read(&vcookie->ref), + n_accesses, fscache_access_cache_pin); + + spin_lock(&cache->object_list_lock); + list_add(&volume->cache_link, &volume->cache->volumes); + spin_unlock(&cache->object_list_lock); + + kfree(name); + return; + +error_fan: + for (i = 0; i < 256; i++) + cachefiles_put_directory(volume->fanout[i]); +error_dir: + cachefiles_put_directory(volume->dentry); +error_name: + kfree(name); +error_vol: + kfree(volume); + cachefiles_end_secure(cache, saved_cred); +} + +/* + * Release a volume representation. + */ +static void __cachefiles_free_volume(struct cachefiles_volume *volume) +{ + int i; + + _enter(""); + + volume->vcookie->cache_priv = NULL; + + for (i = 0; i < 256; i++) + cachefiles_put_directory(volume->fanout[i]); + cachefiles_put_directory(volume->dentry); + kfree(volume); +} + +void cachefiles_free_volume(struct fscache_volume *vcookie) +{ + struct cachefiles_volume *volume = vcookie->cache_priv; + + if (volume) { + spin_lock(&volume->cache->object_list_lock); + list_del_init(&volume->cache_link); + spin_unlock(&volume->cache->object_list_lock); + __cachefiles_free_volume(volume); + } +} + +void cachefiles_withdraw_volume(struct cachefiles_volume *volume) +{ + fscache_withdraw_volume(volume->vcookie); + cachefiles_set_volume_xattr(volume); + __cachefiles_free_volume(volume); +} diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index 9e82de668595..83f41bd0c3a9 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* CacheFiles extended attribute management * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -15,310 +15,245 @@ #include <linux/slab.h> #include "internal.h" +#define CACHEFILES_COOKIE_TYPE_DATA 1 + +struct cachefiles_xattr { + __be64 object_size; /* Actual size of the object */ + __be64 zero_point; /* Size after which server has no data not written by us */ + __u8 type; /* Type of object */ + __u8 content; /* Content presence (enum cachefiles_content) */ + __u8 data[]; /* netfs coherency data */ +} __packed; + static const char cachefiles_xattr_cache[] = XATTR_USER_PREFIX "CacheFiles.cache"; /* - * check the type label on an object - * - done using xattrs + * set the state xattr on a cache file */ -int cachefiles_check_object_type(struct cachefiles_object *object) +int cachefiles_set_object_xattr(struct cachefiles_object *object) { - struct dentry *dentry = object->dentry; - char type[3], xtype[3]; + struct cachefiles_xattr *buf; + struct dentry *dentry; + struct file *file = object->file; + unsigned int len = object->cookie->aux_len; int ret; - ASSERT(dentry); - ASSERT(d_backing_inode(dentry)); - - if (!object->fscache.cookie) - strcpy(type, "C3"); - else - snprintf(type, 3, "%02x", object->fscache.cookie->def->type); - - _enter("%x{%s}", object->fscache.debug_id, type); + if (!file) + return -ESTALE; + dentry = file->f_path.dentry; - /* attempt to install a type label directly */ - ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, type, - 2, XATTR_CREATE); - if (ret == 0) { - _debug("SET"); /* we succeeded */ - goto error; - } + _enter("%x,#%d", object->debug_id, len); - if (ret != -EEXIST) { - pr_err("Can't set xattr on %pd [%lu] (err %d)\n", - dentry, d_backing_inode(dentry)->i_ino, - -ret); - goto error; - } + buf = kmalloc(sizeof(struct cachefiles_xattr) + len, GFP_KERNEL); + if (!buf) + return -ENOMEM; - /* read the current type label */ - ret = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, xtype, - 3); + buf->object_size = cpu_to_be64(object->cookie->object_size); + buf->zero_point = 0; + buf->type = CACHEFILES_COOKIE_TYPE_DATA; + buf->content = object->content_info; + if (test_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) + buf->content = CACHEFILES_CONTENT_DIRTY; + if (len > 0) + memcpy(buf->data, fscache_get_aux(object->cookie), len); + + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, + buf, sizeof(struct cachefiles_xattr) + len, 0); if (ret < 0) { - if (ret == -ERANGE) - goto bad_type_length; - - pr_err("Can't read xattr on %pd [%lu] (err %d)\n", - dentry, d_backing_inode(dentry)->i_ino, - -ret); - goto error; + trace_cachefiles_vfs_error(object, file_inode(file), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + buf->content, + cachefiles_coherency_set_fail); + if (ret != -ENOMEM) + cachefiles_io_error_obj( + object, + "Failed to set xattr with error %d", ret); + } else { + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + buf->content, + cachefiles_coherency_set_ok); } - /* check the type is what we're expecting */ - if (ret != 2) - goto bad_type_length; - - if (xtype[0] != type[0] || xtype[1] != type[1]) - goto bad_type; - - ret = 0; - -error: + kfree(buf); _leave(" = %d", ret); return ret; - -bad_type_length: - pr_err("Cache object %lu type xattr length incorrect\n", - d_backing_inode(dentry)->i_ino); - ret = -EIO; - goto error; - -bad_type: - xtype[2] = 0; - pr_err("Cache object %pd [%lu] type %s not %s\n", - dentry, d_backing_inode(dentry)->i_ino, - xtype, type); - ret = -EIO; - goto error; } /* - * set the state xattr on a cache file + * check the consistency between the backing cache and the FS-Cache cookie */ -int cachefiles_set_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata) +int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file) { - struct dentry *dentry = object->dentry; - int ret; - - ASSERT(dentry); - - _enter("%p,#%d", object, auxdata->len); + struct cachefiles_xattr *buf; + struct dentry *dentry = file->f_path.dentry; + unsigned int len = object->cookie->aux_len, tlen; + const void *p = fscache_get_aux(object->cookie); + enum cachefiles_coherency_trace why; + ssize_t xlen; + int ret = -ESTALE; - /* attempt to install the cache metadata directly */ - _debug("SET #%u", auxdata->len); + tlen = sizeof(struct cachefiles_xattr) + len; + buf = kmalloc(tlen, GFP_KERNEL); + if (!buf) + return -ENOMEM; - clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); - ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, - &auxdata->type, auxdata->len, XATTR_CREATE); - if (ret < 0 && ret != -ENOMEM) - cachefiles_io_error_obj( - object, - "Failed to set xattr with error %d", ret); + xlen = cachefiles_inject_read_error(); + if (xlen == 0) + xlen = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, buf, tlen); + if (xlen != tlen) { + if (xlen < 0) + trace_cachefiles_vfs_error(object, file_inode(file), xlen, + cachefiles_trace_getxattr_error); + if (xlen == -EIO) + cachefiles_io_error_obj( + object, + "Failed to read aux with error %zd", xlen); + why = cachefiles_coherency_check_xattr; + } else if (buf->type != CACHEFILES_COOKIE_TYPE_DATA) { + why = cachefiles_coherency_check_type; + } else if (memcmp(buf->data, p, len) != 0) { + why = cachefiles_coherency_check_aux; + } else if (be64_to_cpu(buf->object_size) != object->cookie->object_size) { + why = cachefiles_coherency_check_objsize; + } else if (buf->content == CACHEFILES_CONTENT_DIRTY) { + // TODO: Begin conflict resolution + pr_warn("Dirty object in cache\n"); + why = cachefiles_coherency_check_dirty; + } else { + why = cachefiles_coherency_check_ok; + ret = 0; + } - _leave(" = %d", ret); + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + buf->content, why); + kfree(buf); return ret; } /* - * update the state xattr on a cache file + * remove the object's xattr to mark it stale */ -int cachefiles_update_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata) +int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dentry) { - struct dentry *dentry = object->dentry; int ret; - if (!dentry) - return -ESTALE; - - _enter("%x,#%d", object->fscache.debug_id, auxdata->len); - - /* attempt to install the cache metadata directly */ - _debug("SET #%u", auxdata->len); - - clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); - ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, - &auxdata->type, auxdata->len, XATTR_REPLACE); - if (ret < 0 && ret != -ENOMEM) - cachefiles_io_error_obj( - object, - "Failed to update xattr with error %d", ret); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_removexattr(&init_user_ns, dentry, cachefiles_xattr_cache); + if (ret < 0) { + trace_cachefiles_vfs_error(object, d_inode(dentry), ret, + cachefiles_trace_remxattr_error); + if (ret == -ENOENT || ret == -ENODATA) + ret = 0; + else if (ret != -ENOMEM) + cachefiles_io_error(cache, + "Can't remove xattr from %lu" + " (error %d)", + d_backing_inode(dentry)->i_ino, -ret); + } _leave(" = %d", ret); return ret; } /* - * check the consistency between the backing cache and the FS-Cache cookie + * Stick a marker on the cache object to indicate that it's dirty. */ -int cachefiles_check_auxdata(struct cachefiles_object *object) +void cachefiles_prepare_to_write(struct fscache_cookie *cookie) { - struct cachefiles_xattr *auxbuf; - enum fscache_checkaux validity; - struct dentry *dentry = object->dentry; - ssize_t xlen; - int ret; - - ASSERT(dentry); - ASSERT(d_backing_inode(dentry)); - ASSERT(object->fscache.cookie->def->check_aux); - - auxbuf = kmalloc(sizeof(struct cachefiles_xattr) + 512, GFP_KERNEL); - if (!auxbuf) - return -ENOMEM; + const struct cred *saved_cred; + struct cachefiles_object *object = cookie->cache_priv; + struct cachefiles_cache *cache = object->volume->cache; - xlen = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, - &auxbuf->type, 512 + 1); - ret = -ESTALE; - if (xlen < 1 || - auxbuf->type != object->fscache.cookie->def->type) - goto error; + _enter("c=%08x", object->cookie->debug_id); - xlen--; - validity = fscache_check_aux(&object->fscache, &auxbuf->data, xlen, - i_size_read(d_backing_inode(dentry))); - if (validity != FSCACHE_CHECKAUX_OKAY) - goto error; - - ret = 0; -error: - kfree(auxbuf); - return ret; + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_set_object_xattr(object); + cachefiles_end_secure(cache, saved_cred); + } } /* - * check the state xattr on a cache file - * - return -ESTALE if the object should be deleted + * Set the state xattr on a volume directory. */ -int cachefiles_check_object_xattr(struct cachefiles_object *object, - struct cachefiles_xattr *auxdata) +bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume) { - struct cachefiles_xattr *auxbuf; - struct dentry *dentry = object->dentry; + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + struct dentry *dentry = volume->dentry; int ret; - _enter("%p,#%d", object, auxdata->len); - - ASSERT(dentry); - ASSERT(d_backing_inode(dentry)); - - auxbuf = kmalloc(sizeof(struct cachefiles_xattr) + 512, cachefiles_gfp); - if (!auxbuf) { - _leave(" = -ENOMEM"); - return -ENOMEM; - } + _enter("%x,#%d", volume->vcookie->debug_id, len); - /* read the current type label */ - ret = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, - &auxbuf->type, 512 + 1); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, + p, len, 0); if (ret < 0) { - if (ret == -ENODATA) - goto stale; /* no attribute - power went off - * mid-cull? */ - - if (ret == -ERANGE) - goto bad_type_length; - - cachefiles_io_error_obj(object, - "Can't read xattr on %lu (err %d)", - d_backing_inode(dentry)->i_ino, -ret); - goto error; + trace_cachefiles_vfs_error(NULL, d_inode(dentry), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_fail); + if (ret != -ENOMEM) + cachefiles_io_error( + volume->cache, "Failed to set xattr with error %d", ret); + } else { + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_ok); } - /* check the on-disk object */ - if (ret < 1) - goto bad_type_length; - - if (auxbuf->type != auxdata->type) - goto stale; - - auxbuf->len = ret; - - /* consult the netfs */ - if (object->fscache.cookie->def->check_aux) { - enum fscache_checkaux result; - unsigned int dlen; - - dlen = auxbuf->len - 1; - - _debug("checkaux %s #%u", - object->fscache.cookie->def->name, dlen); - - result = fscache_check_aux(&object->fscache, - &auxbuf->data, dlen, - i_size_read(d_backing_inode(dentry))); - - switch (result) { - /* entry okay as is */ - case FSCACHE_CHECKAUX_OKAY: - goto okay; - - /* entry requires update */ - case FSCACHE_CHECKAUX_NEEDS_UPDATE: - break; - - /* entry requires deletion */ - case FSCACHE_CHECKAUX_OBSOLETE: - goto stale; - - default: - BUG(); - } - - /* update the current label */ - ret = vfs_setxattr(&init_user_ns, dentry, - cachefiles_xattr_cache, &auxdata->type, - auxdata->len, XATTR_REPLACE); - if (ret < 0) { - cachefiles_io_error_obj(object, - "Can't update xattr on %lu" - " (error %d)", - d_backing_inode(dentry)->i_ino, -ret); - goto error; - } - } - -okay: - ret = 0; - -error: - kfree(auxbuf); _leave(" = %d", ret); - return ret; - -bad_type_length: - pr_err("Cache object %lu xattr length incorrect\n", - d_backing_inode(dentry)->i_ino); - ret = -EIO; - goto error; - -stale: - ret = -ESTALE; - goto error; + return ret == 0; } /* - * remove the object's xattr to mark it stale + * Check the consistency between the backing cache and the volume cookie. */ -int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, - struct dentry *dentry) +int cachefiles_check_volume_xattr(struct cachefiles_volume *volume) { - int ret; + struct cachefiles_xattr *buf; + struct dentry *dentry = volume->dentry; + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + enum cachefiles_coherency_trace why; + ssize_t xlen; + int ret = -ESTALE; - ret = vfs_removexattr(&init_user_ns, dentry, cachefiles_xattr_cache); - if (ret < 0) { - if (ret == -ENOENT || ret == -ENODATA) - ret = 0; - else if (ret != -ENOMEM) - cachefiles_io_error(cache, - "Can't remove xattr from %lu" - " (error %d)", - d_backing_inode(dentry)->i_ino, -ret); + _enter(""); + + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + xlen = cachefiles_inject_read_error(); + if (xlen == 0) + xlen = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, buf, len); + if (xlen != len) { + if (xlen < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(dentry), xlen, + cachefiles_trace_getxattr_error); + if (xlen == -EIO) + cachefiles_io_error( + volume->cache, + "Failed to read xattr with error %zd", xlen); + } + why = cachefiles_coherency_vol_check_xattr; + } else if (memcmp(buf->data, p, len) != 0) { + why = cachefiles_coherency_vol_check_cmp; + } else { + why = cachefiles_coherency_vol_check_ok; + ret = 0; } + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, why); + kfree(buf); _leave(" = %d", ret); return ret; } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index e53c8541f5b2..b3d9459c9bbd 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -4,8 +4,8 @@ #include <linux/backing-dev.h> #include <linux/fs.h> #include <linux/mm.h> +#include <linux/swap.h> #include <linux/pagemap.h> -#include <linux/writeback.h> /* generic_writepages */ #include <linux/slab.h> #include <linux/pagevec.h> #include <linux/task_io_accounting_ops.h> @@ -126,7 +126,7 @@ static int ceph_set_page_dirty(struct page *page) BUG_ON(PagePrivate(page)); attach_page_private(page, snapc); - return __set_page_dirty_nobuffers(page); + return ceph_fscache_set_page_dirty(page); } /* @@ -141,8 +141,6 @@ static void ceph_invalidatepage(struct page *page, unsigned int offset, struct ceph_inode_info *ci; struct ceph_snap_context *snapc; - wait_on_page_fscache(page); - inode = page->mapping->host; ci = ceph_inode(inode); @@ -153,28 +151,36 @@ static void ceph_invalidatepage(struct page *page, unsigned int offset, } WARN_ON(!PageLocked(page)); - if (!PagePrivate(page)) - return; + if (PagePrivate(page)) { + dout("%p invalidatepage %p idx %lu full dirty page\n", + inode, page, page->index); - dout("%p invalidatepage %p idx %lu full dirty page\n", - inode, page, page->index); + snapc = detach_page_private(page); + ceph_put_wrbuffer_cap_refs(ci, 1, snapc); + ceph_put_snap_context(snapc); + } - snapc = detach_page_private(page); - ceph_put_wrbuffer_cap_refs(ci, 1, snapc); - ceph_put_snap_context(snapc); + wait_on_page_fscache(page); } static int ceph_releasepage(struct page *page, gfp_t gfp) { - dout("%p releasepage %p idx %lu (%sdirty)\n", page->mapping->host, - page, page->index, PageDirty(page) ? "" : "not "); + struct inode *inode = page->mapping->host; + + dout("%llx:%llx releasepage %p idx %lu (%sdirty)\n", + ceph_vinop(inode), page, + page->index, PageDirty(page) ? "" : "not "); + + if (PagePrivate(page)) + return 0; if (PageFsCache(page)) { - if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) + if (current_is_kswapd() || !(gfp & __GFP_FS)) return 0; wait_on_page_fscache(page); } - return !PagePrivate(page); + ceph_fscache_note_page_release(inode); + return 1; } static void ceph_netfs_expand_readahead(struct netfs_read_request *rreq) @@ -378,6 +384,38 @@ static void ceph_readahead(struct readahead_control *ractl) netfs_readahead(ractl, &ceph_netfs_read_ops, (void *)(uintptr_t)got); } +#ifdef CONFIG_CEPH_FSCACHE +static void ceph_set_page_fscache(struct page *page) +{ + set_page_fscache(page); +} + +static void ceph_fscache_write_terminated(void *priv, ssize_t error, bool was_async) +{ + struct inode *inode = priv; + + if (IS_ERR_VALUE(error) && error != -ENOBUFS) + ceph_fscache_invalidate(inode, false); +} + +static void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct fscache_cookie *cookie = ceph_fscache_cookie(ci); + + fscache_write_to_cache(cookie, inode->i_mapping, off, len, i_size_read(inode), + ceph_fscache_write_terminated, inode, caching); +} +#else +static inline void ceph_set_page_fscache(struct page *page) +{ +} + +static inline void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) +{ +} +#endif /* CONFIG_CEPH_FSCACHE */ + struct ceph_writeback_ctl { loff_t i_size; @@ -493,6 +531,7 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc) struct ceph_writeback_ctl ceph_wbc; struct ceph_osd_client *osdc = &fsc->client->osdc; struct ceph_osd_request *req; + bool caching = ceph_is_cache_enabled(inode); dout("writepage %p idx %lu\n", page, page->index); @@ -531,16 +570,17 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc) CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); - set_page_writeback(page); req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode), page_off, &len, 0, 1, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, snapc, ceph_wbc.truncate_seq, ceph_wbc.truncate_size, true); - if (IS_ERR(req)) { - redirty_page_for_writepage(wbc, page); - end_page_writeback(page); + if (IS_ERR(req)) return PTR_ERR(req); - } + + set_page_writeback(page); + if (caching) + ceph_set_page_fscache(page); + ceph_fscache_write_to_cache(inode, page_off, len, caching); /* it may be a short write due to an object boundary */ WARN_ON_ONCE(len > thp_size(page)); @@ -599,6 +639,9 @@ static int ceph_writepage(struct page *page, struct writeback_control *wbc) struct inode *inode = page->mapping->host; BUG_ON(!inode); ihold(inode); + + wait_on_page_fscache(page); + err = writepage_nounlock(page, wbc); if (err == -ERESTARTSYS) { /* direct memory reclaimer was killed by SIGKILL. return 0 @@ -720,6 +763,7 @@ static int ceph_writepages_start(struct address_space *mapping, struct ceph_writeback_ctl ceph_wbc; bool should_loop, range_whole = false; bool done = false; + bool caching = ceph_is_cache_enabled(inode); dout("writepages_start %p (mode=%s)\n", inode, wbc->sync_mode == WB_SYNC_NONE ? "NONE" : @@ -843,7 +887,7 @@ get_more_pages: unlock_page(page); break; } - if (PageWriteback(page)) { + if (PageWriteback(page) || PageFsCache(page)) { if (wbc->sync_mode == WB_SYNC_NONE) { dout("%p under writeback\n", page); unlock_page(page); @@ -851,6 +895,7 @@ get_more_pages: } dout("waiting on writeback %p\n", page); wait_on_page_writeback(page); + wait_on_page_fscache(page); } if (!clear_page_dirty_for_io(page)) { @@ -983,9 +1028,19 @@ new_request: op_idx = 0; for (i = 0; i < locked_pages; i++) { u64 cur_offset = page_offset(pages[i]); + /* + * Discontinuity in page range? Ceph can handle that by just passing + * multiple extents in the write op. + */ if (offset + len != cur_offset) { + /* If it's full, stop here */ if (op_idx + 1 == req->r_num_ops) break; + + /* Kick off an fscache write with what we have so far. */ + ceph_fscache_write_to_cache(inode, offset, len, caching); + + /* Start a new extent */ osd_req_op_extent_dup_last(req, op_idx, cur_offset - offset); dout("writepages got pages at %llu~%llu\n", @@ -996,14 +1051,17 @@ new_request: osd_req_op_extent_update(req, op_idx, len); len = 0; - offset = cur_offset; + offset = cur_offset; data_pages = pages + i; op_idx++; } set_page_writeback(pages[i]); + if (caching) + ceph_set_page_fscache(pages[i]); len += thp_size(page); } + ceph_fscache_write_to_cache(inode, offset, len, caching); if (ceph_wbc.size_stable) { len = min(len, ceph_wbc.i_size - offset); diff --git a/fs/ceph/cache.c b/fs/ceph/cache.c index 457afda5498a..7d22850623ef 100644 --- a/fs/ceph/cache.c +++ b/fs/ceph/cache.c @@ -12,199 +12,99 @@ #include "super.h" #include "cache.h" -struct fscache_netfs ceph_cache_netfs = { - .name = "ceph", - .version = 0, -}; - -static DEFINE_MUTEX(ceph_fscache_lock); -static LIST_HEAD(ceph_fscache_list); - -struct ceph_fscache_entry { - struct list_head list; - struct fscache_cookie *fscache; - size_t uniq_len; - /* The following members must be last */ - struct ceph_fsid fsid; - char uniquifier[]; -}; - -static const struct fscache_cookie_def ceph_fscache_fsid_object_def = { - .name = "CEPH.fsid", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -int __init ceph_fscache_register(void) -{ - return fscache_register_netfs(&ceph_cache_netfs); -} - -void ceph_fscache_unregister(void) -{ - fscache_unregister_netfs(&ceph_cache_netfs); -} - -int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc) +void ceph_fscache_register_inode_cookie(struct inode *inode) { - const struct ceph_fsid *fsid = &fsc->client->fsid; - const char *fscache_uniq = fsc->mount_options->fscache_uniq; - size_t uniq_len = fscache_uniq ? strlen(fscache_uniq) : 0; - struct ceph_fscache_entry *ent; - int err = 0; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_client(inode); - mutex_lock(&ceph_fscache_lock); - list_for_each_entry(ent, &ceph_fscache_list, list) { - if (memcmp(&ent->fsid, fsid, sizeof(*fsid))) - continue; - if (ent->uniq_len != uniq_len) - continue; - if (uniq_len && memcmp(ent->uniquifier, fscache_uniq, uniq_len)) - continue; - - errorfc(fc, "fscache cookie already registered for fsid %pU, use fsc=<uniquifier> option", - fsid); - err = -EBUSY; - goto out_unlock; - } + /* No caching for filesystem? */ + if (!fsc->fscache) + return; - ent = kzalloc(sizeof(*ent) + uniq_len, GFP_KERNEL); - if (!ent) { - err = -ENOMEM; - goto out_unlock; - } + /* Regular files only */ + if (!S_ISREG(inode->i_mode)) + return; - memcpy(&ent->fsid, fsid, sizeof(*fsid)); - if (uniq_len > 0) { - memcpy(&ent->uniquifier, fscache_uniq, uniq_len); - ent->uniq_len = uniq_len; - } + /* Only new inodes! */ + if (!(inode->i_state & I_NEW)) + return; - fsc->fscache = fscache_acquire_cookie(ceph_cache_netfs.primary_index, - &ceph_fscache_fsid_object_def, - &ent->fsid, sizeof(ent->fsid) + uniq_len, - NULL, 0, - fsc, 0, true); + WARN_ON_ONCE(ci->fscache); - if (fsc->fscache) { - ent->fscache = fsc->fscache; - list_add_tail(&ent->list, &ceph_fscache_list); - } else { - kfree(ent); - errorfc(fc, "unable to register fscache cookie for fsid %pU", - fsid); - /* all other fs ignore this error */ - } -out_unlock: - mutex_unlock(&ceph_fscache_lock); - return err; + ci->fscache = fscache_acquire_cookie(fsc->fscache, 0, + &ci->i_vino, sizeof(ci->i_vino), + &ci->i_version, sizeof(ci->i_version), + i_size_read(inode)); } -static enum fscache_checkaux ceph_fscache_inode_check_aux( - void *cookie_netfs_data, const void *data, uint16_t dlen, - loff_t object_size) +void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) { - struct ceph_inode_info* ci = cookie_netfs_data; - struct inode* inode = &ci->vfs_inode; + struct fscache_cookie *cookie = ci->fscache; - if (dlen != sizeof(ci->i_version) || - i_size_read(inode) != object_size) - return FSCACHE_CHECKAUX_OBSOLETE; + fscache_relinquish_cookie(cookie, false); +} - if (*(u64 *)data != ci->i_version) - return FSCACHE_CHECKAUX_OBSOLETE; +void ceph_fscache_use_cookie(struct inode *inode, bool will_modify) +{ + struct ceph_inode_info *ci = ceph_inode(inode); - dout("ceph inode 0x%p cached okay\n", ci); - return FSCACHE_CHECKAUX_OKAY; + fscache_use_cookie(ci->fscache, will_modify); } -static const struct fscache_cookie_def ceph_fscache_inode_object_def = { - .name = "CEPH.inode", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .check_aux = ceph_fscache_inode_check_aux, -}; - -void ceph_fscache_register_inode_cookie(struct inode *inode) +void ceph_fscache_unuse_cookie(struct inode *inode, bool update) { struct ceph_inode_info *ci = ceph_inode(inode); - struct ceph_fs_client *fsc = ceph_inode_to_client(inode); - - /* No caching for filesystem */ - if (!fsc->fscache) - return; - /* Only cache for regular files that are read only */ - if (!S_ISREG(inode->i_mode)) - return; + if (update) { + loff_t i_size = i_size_read(inode); - inode_lock_nested(inode, I_MUTEX_CHILD); - if (!ci->fscache) { - ci->fscache = fscache_acquire_cookie(fsc->fscache, - &ceph_fscache_inode_object_def, - &ci->i_vino, sizeof(ci->i_vino), - &ci->i_version, sizeof(ci->i_version), - ci, i_size_read(inode), false); + fscache_unuse_cookie(ci->fscache, &ci->i_version, &i_size); + } else { + fscache_unuse_cookie(ci->fscache, NULL, NULL); } - inode_unlock(inode); } -void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) +void ceph_fscache_update(struct inode *inode) { - struct fscache_cookie* cookie; - - if ((cookie = ci->fscache) == NULL) - return; - - ci->fscache = NULL; + struct ceph_inode_info *ci = ceph_inode(inode); + loff_t i_size = i_size_read(inode); - fscache_relinquish_cookie(cookie, &ci->i_vino, false); + fscache_update_cookie(ci->fscache, &ci->i_version, &i_size); } -static bool ceph_fscache_can_enable(void *data) +void ceph_fscache_invalidate(struct inode *inode, bool dio_write) { - struct inode *inode = data; - return !inode_is_open_for_write(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + + fscache_invalidate(ceph_inode(inode)->fscache, + &ci->i_version, i_size_read(inode), + dio_write ? FSCACHE_INVAL_DIO_WRITE : 0); } -void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp) +int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc) { - struct ceph_inode_info *ci = ceph_inode(inode); + const struct ceph_fsid *fsid = &fsc->client->fsid; + const char *fscache_uniq = fsc->mount_options->fscache_uniq; + size_t uniq_len = fscache_uniq ? strlen(fscache_uniq) : 0; + char *name; + int err = 0; - if (!fscache_cookie_valid(ci->fscache)) - return; + name = kasprintf(GFP_KERNEL, "ceph,%pU%s%s", fsid, uniq_len ? "," : "", + uniq_len ? fscache_uniq : ""); + if (!name) + return -ENOMEM; - if (inode_is_open_for_write(inode)) { - dout("fscache_file_set_cookie %p %p disabling cache\n", - inode, filp); - fscache_disable_cookie(ci->fscache, &ci->i_vino, false); - } else { - fscache_enable_cookie(ci->fscache, &ci->i_vino, i_size_read(inode), - ceph_fscache_can_enable, inode); - if (fscache_cookie_enabled(ci->fscache)) { - dout("fscache_file_set_cookie %p %p enabling cache\n", - inode, filp); - } + fsc->fscache = fscache_acquire_volume(name, NULL, NULL, 0); + if (IS_ERR_OR_NULL(fsc->fscache)) { + errorfc(fc, "Unable to register fscache cookie for %s", name); + err = fsc->fscache ? PTR_ERR(fsc->fscache) : -EOPNOTSUPP; + fsc->fscache = NULL; } + kfree(name); + return err; } void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) { - if (fscache_cookie_valid(fsc->fscache)) { - struct ceph_fscache_entry *ent; - bool found = false; - - mutex_lock(&ceph_fscache_lock); - list_for_each_entry(ent, &ceph_fscache_list, list) { - if (ent->fscache == fsc->fscache) { - list_del(&ent->list); - kfree(ent); - found = true; - break; - } - } - WARN_ON_ONCE(!found); - mutex_unlock(&ceph_fscache_lock); - - __fscache_relinquish_cookie(fsc->fscache, NULL, false); - } - fsc->fscache = NULL; + fscache_relinquish_volume(fsc->fscache, NULL, false); } diff --git a/fs/ceph/cache.h b/fs/ceph/cache.h index 058ea2a04376..09164389fa66 100644 --- a/fs/ceph/cache.h +++ b/fs/ceph/cache.h @@ -12,19 +12,19 @@ #include <linux/netfs.h> #ifdef CONFIG_CEPH_FSCACHE - -extern struct fscache_netfs ceph_cache_netfs; - -int ceph_fscache_register(void); -void ceph_fscache_unregister(void); +#include <linux/fscache.h> int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc); void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc); void ceph_fscache_register_inode_cookie(struct inode *inode); void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci); -void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp); -void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci); + +void ceph_fscache_use_cookie(struct inode *inode, bool will_modify); +void ceph_fscache_unuse_cookie(struct inode *inode, bool update); + +void ceph_fscache_update(struct inode *inode); +void ceph_fscache_invalidate(struct inode *inode, bool dio_write); static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci) { @@ -36,37 +36,51 @@ static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info return ci->fscache; } -static inline void ceph_fscache_invalidate(struct inode *inode) +static inline void ceph_fscache_resize(struct inode *inode, loff_t to) { - fscache_invalidate(ceph_inode(inode)->fscache); + struct ceph_inode_info *ci = ceph_inode(inode); + struct fscache_cookie *cookie = ceph_fscache_cookie(ci); + + if (cookie) { + ceph_fscache_use_cookie(inode, true); + fscache_resize_cookie(cookie, to); + ceph_fscache_unuse_cookie(inode, true); + } } -static inline bool ceph_is_cache_enabled(struct inode *inode) +static inline void ceph_fscache_unpin_writeback(struct inode *inode, + struct writeback_control *wbc) { - struct fscache_cookie *cookie = ceph_fscache_cookie(ceph_inode(inode)); + fscache_unpin_writeback(wbc, ceph_fscache_cookie(ceph_inode(inode))); +} + +static inline int ceph_fscache_set_page_dirty(struct page *page) +{ + struct inode *inode = page->mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); - if (!cookie) - return false; - return fscache_cookie_enabled(cookie); + return fscache_set_page_dirty(page, ceph_fscache_cookie(ci)); } static inline int ceph_begin_cache_operation(struct netfs_read_request *rreq) { struct fscache_cookie *cookie = ceph_fscache_cookie(ceph_inode(rreq->inode)); - return fscache_begin_read_operation(rreq, cookie); + return fscache_begin_read_operation(&rreq->cache_resources, cookie); } -#else -static inline int ceph_fscache_register(void) +static inline bool ceph_is_cache_enabled(struct inode *inode) { - return 0; + return fscache_cookie_enabled(ceph_fscache_cookie(ceph_inode(inode))); } -static inline void ceph_fscache_unregister(void) +static inline void ceph_fscache_note_page_release(struct inode *inode) { -} + struct ceph_inode_info *ci = ceph_inode(inode); + fscache_note_page_release(ceph_fscache_cookie(ci)); +} +#else /* CONFIG_CEPH_FSCACHE */ static inline int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc) { @@ -81,28 +95,49 @@ static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci) { } -static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info *ci) +static inline void ceph_fscache_register_inode_cookie(struct inode *inode) { - return NULL; } -static inline void ceph_fscache_register_inode_cookie(struct inode *inode) +static inline void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) { } -static inline void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) +static inline void ceph_fscache_use_cookie(struct inode *inode, bool will_modify) { } -static inline void ceph_fscache_file_set_cookie(struct inode *inode, - struct file *filp) +static inline void ceph_fscache_unuse_cookie(struct inode *inode, bool update) { } -static inline void ceph_fscache_invalidate(struct inode *inode) +static inline void ceph_fscache_update(struct inode *inode) { } +static inline void ceph_fscache_invalidate(struct inode *inode, bool dio_write) +{ +} + +static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info *ci) +{ + return NULL; +} + +static inline void ceph_fscache_resize(struct inode *inode, loff_t to) +{ +} + +static inline void ceph_fscache_unpin_writeback(struct inode *inode, + struct writeback_control *wbc) +{ +} + +static inline int ceph_fscache_set_page_dirty(struct page *page) +{ + return __set_page_dirty_nobuffers(page); +} + static inline bool ceph_is_cache_enabled(struct inode *inode) { return false; @@ -112,6 +147,10 @@ static inline int ceph_begin_cache_operation(struct netfs_read_request *rreq) { return -ENOBUFS; } -#endif -#endif /* _CEPH_CACHE_H */ +static inline void ceph_fscache_note_page_release(struct inode *inode) +{ +} +#endif /* CONFIG_CEPH_FSCACHE */ + +#endif diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index c447fa2e2d1f..7d2c33cdbac6 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -1856,7 +1856,7 @@ static int try_nonblocking_invalidate(struct inode *inode) u32 invalidating_gen = ci->i_rdcache_gen; spin_unlock(&ci->i_ceph_lock); - ceph_fscache_invalidate(inode); + ceph_fscache_invalidate(inode, false); invalidate_mapping_pages(&inode->i_data, 0, -1); spin_lock(&ci->i_ceph_lock); @@ -2388,6 +2388,7 @@ int ceph_write_inode(struct inode *inode, struct writeback_control *wbc) int wait = (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync); dout("write_inode %p wait=%d\n", inode, wait); + ceph_fscache_unpin_writeback(inode, wbc); if (wait) { dirty = try_flush_caps(inode, &flush_tid); if (dirty) diff --git a/fs/ceph/file.c b/fs/ceph/file.c index c138e8126286..9d9304e712d9 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -248,8 +248,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) switch (inode->i_mode & S_IFMT) { case S_IFREG: - ceph_fscache_register_inode_cookie(inode); - ceph_fscache_file_set_cookie(inode, file); + ceph_fscache_use_cookie(inode, file->f_mode & FMODE_WRITE); fallthrough; case S_IFDIR: ret = ceph_init_file_info(inode, file, fmode, @@ -822,6 +821,7 @@ int ceph_release(struct inode *inode, struct file *file) dout("release inode %p regular file %p\n", inode, file); WARN_ON(!list_empty(&fi->rw_contexts)); + ceph_fscache_unuse_cookie(inode, file->f_mode & FMODE_WRITE); ceph_put_fmode(ci, fi->fmode, 1); kmem_cache_free(ceph_file_cachep, fi); @@ -1218,7 +1218,11 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter, snapc, snapc ? snapc->seq : 0); if (write) { - int ret2 = invalidate_inode_pages2_range(inode->i_mapping, + int ret2; + + ceph_fscache_invalidate(inode, true); + + ret2 = invalidate_inode_pages2_range(inode->i_mapping, pos >> PAGE_SHIFT, (pos + count - 1) >> PAGE_SHIFT); if (ret2 < 0) @@ -1429,6 +1433,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos, if (ret < 0) return ret; + ceph_fscache_invalidate(inode, false); ret = invalidate_inode_pages2_range(inode->i_mapping, pos >> PAGE_SHIFT, (pos + count - 1) >> PAGE_SHIFT); @@ -2113,6 +2118,7 @@ static long ceph_fallocate(struct file *file, int mode, goto unlock; filemap_invalidate_lock(inode->i_mapping); + ceph_fscache_invalidate(inode, false); ceph_zero_pagecache_range(inode, offset, length); ret = ceph_zero_objects(inode, offset, length); @@ -2437,6 +2443,7 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, goto out_caps; /* Drop dst file cached pages */ + ceph_fscache_invalidate(dst_inode, false); ret = invalidate_inode_pages2_range(dst_inode->i_mapping, dst_off >> PAGE_SHIFT, (dst_off + len) >> PAGE_SHIFT); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index e3322fcb2e8d..ef4a980a7bf3 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -564,6 +564,8 @@ void ceph_evict_inode(struct inode *inode) percpu_counter_dec(&mdsc->metric.total_inodes); truncate_inode_pages_final(&inode->i_data); + if (inode->i_state & I_PINNING_FSCACHE_WB) + ceph_fscache_unuse_cookie(inode, true); clear_inode(inode); ceph_fscache_unregister_inode_cookie(ci); @@ -634,6 +636,12 @@ int ceph_fill_file_size(struct inode *inode, int issued, } i_size_write(inode, size); inode->i_blocks = calc_inode_blocks(size); + /* + * If we're expanding, then we should be able to just update + * the existing cookie. + */ + if (size > isize) + ceph_fscache_update(inode); ci->i_reported_size = size; if (truncate_seq != ci->i_truncate_seq) { dout("truncate_seq %u -> %u\n", @@ -666,10 +674,6 @@ int ceph_fill_file_size(struct inode *inode, int issued, truncate_size); ci->i_truncate_size = truncate_size; } - - if (queue_trunc) - ceph_fscache_invalidate(inode); - return queue_trunc; } @@ -1053,6 +1057,8 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, spin_unlock(&ci->i_ceph_lock); + ceph_fscache_register_inode_cookie(inode); + if (fill_inline) ceph_fill_inline_data(inode, locked_page, iinfo->inline_data, iinfo->inline_len); @@ -1814,11 +1820,13 @@ bool ceph_inode_set_size(struct inode *inode, loff_t size) spin_lock(&ci->i_ceph_lock); dout("set_size %p %llu -> %llu\n", inode, i_size_read(inode), size); i_size_write(inode, size); + ceph_fscache_update(inode); inode->i_blocks = calc_inode_blocks(size); ret = __ceph_should_report_size(ci); spin_unlock(&ci->i_ceph_lock); + return ret; } @@ -1844,6 +1852,8 @@ static void ceph_do_invalidate_pages(struct inode *inode) u32 orig_gen; int check = 0; + ceph_fscache_invalidate(inode, false); + mutex_lock(&ci->i_truncate_mutex); if (ceph_inode_is_shutdown(inode)) { @@ -1868,7 +1878,7 @@ static void ceph_do_invalidate_pages(struct inode *inode) orig_gen = ci->i_rdcache_gen; spin_unlock(&ci->i_ceph_lock); - ceph_fscache_invalidate(inode); + ceph_fscache_invalidate(inode, false); if (invalidate_inode_pages2(inode->i_mapping) < 0) { pr_err("invalidate_inode_pages2 %llx.%llx failed\n", ceph_vinop(inode)); @@ -1937,6 +1947,7 @@ retry: ci->i_truncate_pending, to); spin_unlock(&ci->i_ceph_lock); + ceph_fscache_resize(inode, to); truncate_pagecache(inode, to); spin_lock(&ci->i_ceph_lock); @@ -2184,7 +2195,6 @@ int __ceph_setattr(struct inode *inode, struct iattr *attr) if (inode_dirty_flags) __mark_inode_dirty(inode, inode_dirty_flags); - if (mask) { req->r_inode = inode; ihold(inode); diff --git a/fs/ceph/super.c b/fs/ceph/super.c index bab61232dc5a..bea89bdb534a 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -787,16 +787,10 @@ static int __init init_caches(void) if (!ceph_wb_pagevec_pool) goto bad_pagevec_pool; - error = ceph_fscache_register(); - if (error) - goto bad_fscache; - return 0; -bad_fscache: - kmem_cache_destroy(ceph_mds_request_cachep); bad_pagevec_pool: - mempool_destroy(ceph_wb_pagevec_pool); + kmem_cache_destroy(ceph_mds_request_cachep); bad_mds_req: kmem_cache_destroy(ceph_dir_file_cachep); bad_dir_file: @@ -828,8 +822,6 @@ static void destroy_caches(void) kmem_cache_destroy(ceph_dir_file_cachep); kmem_cache_destroy(ceph_mds_request_cachep); mempool_destroy(ceph_wb_pagevec_pool); - - ceph_fscache_unregister(); } static void __ceph_umount_begin(struct ceph_fs_client *fsc) diff --git a/fs/ceph/super.h b/fs/ceph/super.h index ac331aa07cfa..d0142cc5c41b 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -21,7 +21,6 @@ #include <linux/ceph/libceph.h> #ifdef CONFIG_CEPH_FSCACHE -#define FSCACHE_USE_NEW_IO_API #include <linux/fscache.h> #endif @@ -135,7 +134,7 @@ struct ceph_fs_client { #endif #ifdef CONFIG_CEPH_FSCACHE - struct fscache_cookie *fscache; + struct fscache_volume *fscache; #endif }; diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig index 3b7e3b9e4fd2..346ae8716deb 100644 --- a/fs/cifs/Kconfig +++ b/fs/cifs/Kconfig @@ -188,7 +188,7 @@ config CIFS_SMB_DIRECT config CIFS_FSCACHE bool "Provide CIFS client caching support" - depends on CIFS=m && FSCACHE || CIFS=y && FSCACHE=y + depends on CIFS=m && FSCACHE_OLD_API || CIFS=y && FSCACHE_OLD_API=y help Makes CIFS FS-Cache capable. Say Y here if you want your CIFS data to be cached locally on disk through the general filesystem cache diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 4f680f848c8b..f8d7fe6db989 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1668,6 +1668,13 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) inode->i_state |= I_DIRTY_PAGES; + else if (unlikely(inode->i_state & I_PINNING_FSCACHE_WB)) { + if (!(inode->i_state & I_DIRTY_PAGES)) { + inode->i_state &= ~I_PINNING_FSCACHE_WB; + wbc->unpinned_fscache_wb = true; + dirty |= I_PINNING_FSCACHE_WB; /* Cause write_inode */ + } + } spin_unlock(&inode->i_lock); @@ -1677,6 +1684,7 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) if (ret == 0) ret = err; } + wbc->unpinned_fscache_wb = false; trace_writeback_single_inode(inode, wbc, nr_to_write); return ret; } diff --git a/fs/fscache/Kconfig b/fs/fscache/Kconfig index b313a978ae0a..76316c4a3fb7 100644 --- a/fs/fscache/Kconfig +++ b/fs/fscache/Kconfig @@ -38,3 +38,6 @@ config FSCACHE_DEBUG enabled by setting bits in /sys/modules/fscache/parameter/debug. See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_OLD_API + bool diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index 03a871d689bb..afb090ea16c4 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -6,13 +6,9 @@ fscache-y := \ cache.o \ cookie.o \ - fsdef.o \ io.o \ main.o \ - netfs.o \ - object.o \ - operation.o \ - page.o + volume.o fscache-$(CONFIG_PROC_FS) += proc.o fscache-$(CONFIG_FSCACHE_STATS) += stats.o diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index bd4f44c1cce0..2749933852a9 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -1,209 +1,229 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* FS-Cache cache handling * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ #define FSCACHE_DEBUG_LEVEL CACHE -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include "internal.h" -LIST_HEAD(fscache_cache_list); +static LIST_HEAD(fscache_caches); DECLARE_RWSEM(fscache_addremove_sem); -DECLARE_WAIT_QUEUE_HEAD(fscache_cache_cleared_wq); -EXPORT_SYMBOL(fscache_cache_cleared_wq); +EXPORT_SYMBOL(fscache_addremove_sem); +DECLARE_WAIT_QUEUE_HEAD(fscache_clearance_waiters); +EXPORT_SYMBOL(fscache_clearance_waiters); -static LIST_HEAD(fscache_cache_tag_list); +static atomic_t fscache_cache_debug_id; /* - * look up a cache tag + * Allocate a cache cookie. */ -struct fscache_cache_tag *__fscache_lookup_cache_tag(const char *name) +static struct fscache_cache *fscache_alloc_cache(const char *name) { - struct fscache_cache_tag *tag, *xtag; - - /* firstly check for the existence of the tag under read lock */ - down_read(&fscache_addremove_sem); - - list_for_each_entry(tag, &fscache_cache_tag_list, link) { - if (strcmp(tag->name, name) == 0) { - atomic_inc(&tag->usage); - up_read(&fscache_addremove_sem); - return tag; - } - } - - up_read(&fscache_addremove_sem); - - /* the tag does not exist - create a candidate */ - xtag = kzalloc(sizeof(*xtag) + strlen(name) + 1, GFP_KERNEL); - if (!xtag) - /* return a dummy tag if out of memory */ - return ERR_PTR(-ENOMEM); - - atomic_set(&xtag->usage, 1); - strcpy(xtag->name, name); - - /* write lock, search again and add if still not present */ - down_write(&fscache_addremove_sem); + struct fscache_cache *cache; - list_for_each_entry(tag, &fscache_cache_tag_list, link) { - if (strcmp(tag->name, name) == 0) { - atomic_inc(&tag->usage); - up_write(&fscache_addremove_sem); - kfree(xtag); - return tag; + cache = kzalloc(sizeof(*cache), GFP_KERNEL); + if (cache) { + if (name) { + cache->name = kstrdup(name, GFP_KERNEL); + if (!cache->name) { + kfree(cache); + return NULL; + } } + refcount_set(&cache->ref, 1); + INIT_LIST_HEAD(&cache->cache_link); + cache->debug_id = atomic_inc_return(&fscache_cache_debug_id); } - - list_add_tail(&xtag->link, &fscache_cache_tag_list); - up_write(&fscache_addremove_sem); - return xtag; + return cache; } -/* - * release a reference to a cache tag - */ -void __fscache_release_cache_tag(struct fscache_cache_tag *tag) +static bool fscache_get_cache_maybe(struct fscache_cache *cache, + enum fscache_cache_trace where) { - if (tag != ERR_PTR(-ENOMEM)) { - down_write(&fscache_addremove_sem); + bool success; + int ref; - if (atomic_dec_and_test(&tag->usage)) - list_del_init(&tag->link); - else - tag = NULL; - - up_write(&fscache_addremove_sem); - - kfree(tag); - } + success = __refcount_inc_not_zero(&cache->ref, &ref); + if (success) + trace_fscache_cache(cache->debug_id, ref + 1, where); + return success; } /* - * select a cache in which to store an object - * - the cache addremove semaphore must be at least read-locked by the caller - * - the object will never be an index + * Look up a cache cookie. */ -struct fscache_cache *fscache_select_cache_for_object( - struct fscache_cookie *cookie) +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache) { - struct fscache_cache_tag *tag; - struct fscache_object *object; - struct fscache_cache *cache; + struct fscache_cache *candidate, *cache, *unnamed = NULL; - _enter(""); + /* firstly check for the existence of the cache under read lock */ + down_read(&fscache_addremove_sem); - if (list_empty(&fscache_cache_list)) { - _leave(" = NULL [no cache]"); - return NULL; + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + if (!cache->name && !name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; } - /* we check the parent to determine the cache to use */ - spin_lock(&cookie->lock); + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + } - /* the first in the parent's backing list should be the preferred - * cache */ - if (!hlist_empty(&cookie->backing_objects)) { - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); + up_read(&fscache_addremove_sem); - cache = object->cache; - if (fscache_object_is_dying(object) || - test_bit(FSCACHE_IOERROR, &cache->flags)) - cache = NULL; + /* the cache does not exist - create a candidate */ + candidate = fscache_alloc_cache(name); + if (!candidate) + return ERR_PTR(-ENOMEM); - spin_unlock(&cookie->lock); - _leave(" = %s [parent]", cache ? cache->tag->name : "NULL"); - return cache; - } + /* write lock, search again and add if still not present */ + down_write(&fscache_addremove_sem); - /* the parent is unbacked */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - /* cookie not an index and is unbacked */ - spin_unlock(&cookie->lock); - _leave(" = NULL [cookie ub,ni]"); - return NULL; + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + if (!cache->name) { + unnamed = cache; + if (!name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } } - spin_unlock(&cookie->lock); + if (unnamed && is_cache && + fscache_get_cache_maybe(unnamed, fscache_cache_get_acquire)) + goto use_unnamed_cache; - if (!cookie->def->select_cache) - goto no_preference; + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } - /* ask the netfs for its preference */ - tag = cookie->def->select_cache(cookie->parent->netfs_data, - cookie->netfs_data); - if (!tag) - goto no_preference; + list_add_tail(&candidate->cache_link, &fscache_caches); + trace_fscache_cache(candidate->debug_id, + refcount_read(&candidate->ref), + fscache_cache_new_acquire); + up_write(&fscache_addremove_sem); + return candidate; - if (tag == ERR_PTR(-ENOMEM)) { - _leave(" = NULL [nomem tag]"); - return NULL; - } +got_cache_r: + up_read(&fscache_addremove_sem); + return cache; +use_unnamed_cache: + cache = unnamed; + cache->name = candidate->name; + candidate->name = NULL; +got_cache_w: + up_write(&fscache_addremove_sem); + kfree(candidate->name); + kfree(candidate); + return cache; +} - if (!tag->cache) { - _leave(" = NULL [unbacked tag]"); - return NULL; - } +/** + * fscache_acquire_cache - Acquire a cache-level cookie. + * @name: The name of the cache. + * + * Get a cookie to represent an actual cache. If a name is given and there is + * a nameless cache record available, this will acquire that and set its name, + * directing all the volumes using it to this cache. + * + * The cache will be switched over to the preparing state if not currently in + * use, otherwise -EBUSY will be returned. + */ +struct fscache_cache *fscache_acquire_cache(const char *name) +{ + struct fscache_cache *cache; - if (test_bit(FSCACHE_IOERROR, &tag->cache->flags)) - return NULL; + ASSERT(name); + cache = fscache_lookup_cache(name, true); + if (IS_ERR(cache)) + return cache; - _leave(" = %s [specific]", tag->name); - return tag->cache; + if (!fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_NOT_PRESENT, + FSCACHE_CACHE_IS_PREPARING)) { + pr_warn("Cache tag %s in use\n", name); + fscache_put_cache(cache, fscache_cache_put_cache); + return ERR_PTR(-EBUSY); + } -no_preference: - /* netfs has no preference - just select first cache */ - cache = list_entry(fscache_cache_list.next, - struct fscache_cache, link); - _leave(" = %s [first]", cache->tag->name); return cache; } +EXPORT_SYMBOL(fscache_acquire_cache); /** - * fscache_init_cache - Initialise a cache record - * @cache: The cache record to be initialised - * @ops: The cache operations to be installed in that record - * @idfmt: Format string to define identifier - * @...: sprintf-style arguments + * fscache_put_cache - Release a cache-level cookie. + * @cache: The cache cookie to be released + * @where: An indication of where the release happened * - * Initialise a record of a cache and fill in the name. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. + * Release the caller's reference on a cache-level cookie. The @where + * indication should give information about the circumstances in which the call + * occurs and will be logged through a tracepoint. */ -void fscache_init_cache(struct fscache_cache *cache, - const struct fscache_cache_ops *ops, - const char *idfmt, - ...) +void fscache_put_cache(struct fscache_cache *cache, + enum fscache_cache_trace where) { - va_list va; + unsigned int debug_id = cache->debug_id; + bool zero; + int ref; - memset(cache, 0, sizeof(*cache)); + if (IS_ERR_OR_NULL(cache)) + return; - cache->ops = ops; + zero = __refcount_dec_and_test(&cache->ref, &ref); + trace_fscache_cache(debug_id, ref - 1, where); - va_start(va, idfmt); - vsnprintf(cache->identifier, sizeof(cache->identifier), idfmt, va); - va_end(va); + if (zero) { + down_write(&fscache_addremove_sem); + list_del_init(&cache->cache_link); + up_write(&fscache_addremove_sem); + kfree(cache->name); + kfree(cache); + } +} - INIT_WORK(&cache->op_gc, fscache_operation_gc); - INIT_LIST_HEAD(&cache->link); - INIT_LIST_HEAD(&cache->object_list); - INIT_LIST_HEAD(&cache->op_gc_list); - spin_lock_init(&cache->object_list_lock); - spin_lock_init(&cache->op_gc_list_lock); +/** + * fscache_relinquish_cache - Reset cache state and release cookie + * @cache: The cache cookie to be released + * + * Reset the state of a cache and release the caller's reference on a cache + * cookie. + */ +void fscache_relinquish_cache(struct fscache_cache *cache) +{ + enum fscache_cache_trace where = + (cache->state == FSCACHE_CACHE_IS_PREPARING) ? + fscache_cache_put_prep_failed : + fscache_cache_put_relinquish; + + cache->ops = NULL; + cache->cache_priv = NULL; + smp_store_release(&cache->state, FSCACHE_CACHE_IS_NOT_PRESENT); + fscache_put_cache(cache, where); } -EXPORT_SYMBOL(fscache_init_cache); +EXPORT_SYMBOL(fscache_relinquish_cache); /** * fscache_add_cache - Declare a cache as being open for business - * @cache: The record describing the cache - * @ifsdef: The record of the cache object describing the top-level index - * @tagname: The tag describing this cache + * @cache: The cache-level cookie representing the cache + * @ops: Table of cache operations to use + * @cache_priv: Private data for the cache record * * Add a cache to the system, making it available for netfs's to use. * @@ -211,93 +231,97 @@ EXPORT_SYMBOL(fscache_init_cache); * description. */ int fscache_add_cache(struct fscache_cache *cache, - struct fscache_object *ifsdef, - const char *tagname) + const struct fscache_cache_ops *ops, + void *cache_priv) { - struct fscache_cache_tag *tag; - - ASSERTCMP(ifsdef->cookie, ==, &fscache_fsdef_index); - BUG_ON(!cache->ops); - BUG_ON(!ifsdef); + int n_accesses; - cache->flags = 0; - ifsdef->event_mask = - ((1 << NR_FSCACHE_OBJECT_EVENTS) - 1) & - ~(1 << FSCACHE_OBJECT_EV_CLEARED); - __set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &ifsdef->flags); + _enter("{%s,%s}", ops->name, cache->name); - if (!tagname) - tagname = cache->identifier; + BUG_ON(fscache_cache_state(cache) != FSCACHE_CACHE_IS_PREPARING); - BUG_ON(!tagname[0]); - - _enter("{%s.%s},,%s", cache->ops->name, cache->identifier, tagname); - - /* we use the cache tag to uniquely identify caches */ - tag = __fscache_lookup_cache_tag(tagname); - if (IS_ERR(tag)) - goto nomem; - - if (test_and_set_bit(FSCACHE_TAG_RESERVED, &tag->flags)) - goto tag_in_use; - - cache->kobj = kobject_create_and_add(tagname, fscache_root); - if (!cache->kobj) - goto error; - - ifsdef->cache = cache; - cache->fsdef = ifsdef; + /* Get a ref on the cache cookie and keep its n_accesses counter raised + * by 1 to prevent wakeups from transitioning it to 0 until we're + * withdrawing caching services from it. + */ + n_accesses = atomic_inc_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_pin); down_write(&fscache_addremove_sem); - tag->cache = cache; - cache->tag = tag; - - /* add the cache to the list */ - list_add(&cache->link, &fscache_cache_list); - - /* add the cache's netfs definition index object to the cache's - * list */ - spin_lock(&cache->object_list_lock); - list_add_tail(&ifsdef->cache_link, &cache->object_list); - spin_unlock(&cache->object_list_lock); - - /* add the cache's netfs definition index object to the top level index - * cookie as a known backing object */ - spin_lock(&fscache_fsdef_index.lock); - - hlist_add_head(&ifsdef->cookie_link, - &fscache_fsdef_index.backing_objects); - - refcount_inc(&fscache_fsdef_index.ref); + cache->ops = ops; + cache->cache_priv = cache_priv; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_ACTIVE); - /* done */ - spin_unlock(&fscache_fsdef_index.lock); up_write(&fscache_addremove_sem); - - pr_notice("Cache \"%s\" added (type %s)\n", - cache->tag->name, cache->ops->name); - kobject_uevent(cache->kobj, KOBJ_ADD); - - _leave(" = 0 [%s]", cache->identifier); + pr_notice("Cache \"%s\" added (type %s)\n", cache->name, ops->name); + _leave(" = 0 [%s]", cache->name); return 0; +} +EXPORT_SYMBOL(fscache_add_cache); -tag_in_use: - pr_err("Cache tag '%s' already in use\n", tagname); - __fscache_release_cache_tag(tag); - _leave(" = -EXIST"); - return -EEXIST; - -error: - __fscache_release_cache_tag(tag); - _leave(" = -EINVAL"); - return -EINVAL; +/** + * fscache_begin_cache_access - Pin a cache so it can be accessed + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing it and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the n_accesses count and + * then recheck the liveness, ending the access if it ceased to be live. + * + * (3) When we end the access, we decrement n_accesses and wake up the any + * waiters if it reaches 0. + * + * (4) Whilst the cache is caching, n_accesses is kept artificially + * incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, n_accesses is decremented and we wait for n_accesses to + * become 0. + */ +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + if (!fscache_cache_is_live(cache)) + return false; + + n_accesses = atomic_inc_return(&cache->n_accesses); + smp_mb__after_atomic(); /* Reread live flag after n_accesses */ + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (!fscache_cache_is_live(cache)) { + fscache_end_cache_access(cache, fscache_access_unlive); + return false; + } + return true; +} -nomem: - _leave(" = -ENOMEM"); - return -ENOMEM; +/** + * fscache_end_cache_access - Unpin a cache at the end of an access. + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache after we've accessed it. The @why indicator is merely + * provided for tracing purposes. + */ +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&cache->n_accesses); } -EXPORT_SYMBOL(fscache_add_cache); /** * fscache_io_error - Note a cache I/O error @@ -311,106 +335,94 @@ EXPORT_SYMBOL(fscache_add_cache); */ void fscache_io_error(struct fscache_cache *cache) { - if (!test_and_set_bit(FSCACHE_IOERROR, &cache->flags)) + if (fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_ACTIVE, + FSCACHE_CACHE_GOT_IOERROR)) pr_err("Cache '%s' stopped due to I/O error\n", - cache->ops->name); + cache->name); } EXPORT_SYMBOL(fscache_io_error); -/* - * request withdrawal of all the objects in a cache - * - all the objects being withdrawn are moved onto the supplied list +/** + * fscache_withdraw_cache - Withdraw a cache from the active service + * @cache: The cache cookie + * + * Begin the process of withdrawing a cache from service. This stops new + * cache-level and volume-level accesses from taking place and waits for + * currently ongoing cache-level accesses to end. */ -static void fscache_withdraw_all_objects(struct fscache_cache *cache, - struct list_head *dying_objects) +void fscache_withdraw_cache(struct fscache_cache *cache) { - struct fscache_object *object; + int n_accesses; - while (!list_empty(&cache->object_list)) { - spin_lock(&cache->object_list_lock); + pr_notice("Withdrawing cache \"%s\" (%u objs)\n", + cache->name, atomic_read(&cache->object_count)); - if (!list_empty(&cache->object_list)) { - object = list_entry(cache->object_list.next, - struct fscache_object, cache_link); - list_move_tail(&object->cache_link, dying_objects); + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_WITHDRAWN); - _debug("withdraw %x", object->cookie->debug_id); + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_unpin); - /* This must be done under object_list_lock to prevent - * a race with fscache_drop_object(). - */ - fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL); - } - - spin_unlock(&cache->object_list_lock); - cond_resched(); - } + wait_var_event(&cache->n_accesses, + atomic_read(&cache->n_accesses) == 0); } +EXPORT_SYMBOL(fscache_withdraw_cache); -/** - * fscache_withdraw_cache - Withdraw a cache from the active service - * @cache: The record describing the cache - * - * Withdraw a cache from service, unbinding all its cache objects from the - * netfs cookies they're currently representing. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. +#ifdef CONFIG_PROC_FS +static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] = "-PAEW"; + +/* + * Generate a list of caches in /proc/fs/fscache/caches */ -void fscache_withdraw_cache(struct fscache_cache *cache) +static int fscache_caches_seq_show(struct seq_file *m, void *v) { - LIST_HEAD(dying_objects); + struct fscache_cache *cache; - _enter(""); + if (v == &fscache_caches) { + seq_puts(m, + "CACHE REF VOLS OBJS ACCES S NAME\n" + "======== ===== ===== ===== ===== = ===============\n" + ); + return 0; + } - pr_notice("Withdrawing cache \"%s\"\n", - cache->tag->name); + cache = list_entry(v, struct fscache_cache, cache_link); + seq_printf(m, + "%08x %5d %5d %5d %5d %c %s\n", + cache->debug_id, + refcount_read(&cache->ref), + atomic_read(&cache->n_volumes), + atomic_read(&cache->object_count), + atomic_read(&cache->n_accesses), + fscache_cache_states[cache->state], + cache->name ?: "-"); + return 0; +} - /* make the cache unavailable for cookie acquisition */ - if (test_and_set_bit(FSCACHE_CACHE_WITHDRAWN, &cache->flags)) - BUG(); +static void *fscache_caches_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_caches, *_pos); +} - down_write(&fscache_addremove_sem); - list_del_init(&cache->link); - cache->tag->cache = NULL; - up_write(&fscache_addremove_sem); +static void *fscache_caches_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_caches, _pos); +} - /* make sure all pages pinned by operations on behalf of the netfs are - * written to disk */ - fscache_stat(&fscache_n_cop_sync_cache); - cache->ops->sync_cache(cache); - fscache_stat_d(&fscache_n_cop_sync_cache); - - /* dissociate all the netfs pages backed by this cache from the block - * mappings in the cache */ - fscache_stat(&fscache_n_cop_dissociate_pages); - cache->ops->dissociate_pages(cache); - fscache_stat_d(&fscache_n_cop_dissociate_pages); - - /* we now have to destroy all the active objects pertaining to this - * cache - which we do by passing them off to thread pool to be - * disposed of */ - _debug("destroy"); - - fscache_withdraw_all_objects(cache, &dying_objects); - - /* wait for all extant objects to finish their outstanding operations - * and go away */ - _debug("wait for finish"); - wait_event(fscache_cache_cleared_wq, - atomic_read(&cache->object_count) == 0); - _debug("wait for clearance"); - wait_event(fscache_cache_cleared_wq, - list_empty(&cache->object_list)); - _debug("cleared"); - ASSERT(list_empty(&dying_objects)); - - kobject_put(cache->kobj); - - clear_bit(FSCACHE_TAG_RESERVED, &cache->tag->flags); - fscache_release_cache_tag(cache->tag); - cache->tag = NULL; - - _leave(""); +static void fscache_caches_seq_stop(struct seq_file *m, void *v) + __releases(fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); } -EXPORT_SYMBOL(fscache_withdraw_cache); + +const struct seq_operations fscache_caches_seq_ops = { + .start = fscache_caches_seq_start, + .next = fscache_caches_seq_next, + .stop = fscache_caches_seq_stop, + .show = fscache_caches_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index cd42be646ed3..9bb1ab5fe5ed 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* netfs cookie management * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * * See Documentation/filesystems/caching/netfs-api.rst for more information on @@ -15,70 +15,258 @@ struct kmem_cache *fscache_cookie_jar; -static atomic_t fscache_object_debug_id = ATOMIC_INIT(0); +static void fscache_cookie_lru_timed_out(struct timer_list *timer); +static void fscache_cookie_lru_worker(struct work_struct *work); +static void fscache_cookie_worker(struct work_struct *work); +static void fscache_unhash_cookie(struct fscache_cookie *cookie); +static void fscache_perform_invalidation(struct fscache_cookie *cookie); #define fscache_cookie_hash_shift 15 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; static LIST_HEAD(fscache_cookies); static DEFINE_RWLOCK(fscache_cookies_lock); - -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, - loff_t object_size); -static int fscache_alloc_object(struct fscache_cache *cache, - struct fscache_cookie *cookie); -static int fscache_attach_object(struct fscache_cookie *cookie, - struct fscache_object *object); - -static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) +static LIST_HEAD(fscache_cookie_lru); +static DEFINE_SPINLOCK(fscache_cookie_lru_lock); +DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); +static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; +unsigned int fscache_lru_cookie_timeout = 10 * HZ; + +void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) { - struct fscache_object *object; - struct hlist_node *o; const u8 *k; - unsigned loop; - pr_err("%c-cookie c=%08x [p=%08x fl=%lx nc=%u na=%u]\n", + pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", prefix, cookie->debug_id, - cookie->parent ? cookie->parent->debug_id : 0, cookie->flags, - atomic_read(&cookie->n_children), - atomic_read(&cookie->n_active)); - pr_err("%c-cookie d=%p{%s} n=%p\n", + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state]); + pr_err("%c-cookie V=%08x [%s]\n", prefix, - cookie->def, - cookie->def ? cookie->def->name : "?", - cookie->netfs_data); - - o = READ_ONCE(cookie->backing_objects.first); - if (o) { - object = hlist_entry(o, struct fscache_object, cookie_link); - pr_err("%c-cookie o=%u\n", prefix, object->debug_id); - } + cookie->volume->debug_id, + cookie->volume->key); - pr_err("%c-key=[%u] '", prefix, cookie->key_len); k = (cookie->key_len <= sizeof(cookie->inline_key)) ? cookie->inline_key : cookie->key; - for (loop = 0; loop < cookie->key_len; loop++) - pr_cont("%02x", k[loop]); - pr_cont("'\n"); + pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); } -void fscache_free_cookie(struct fscache_cookie *cookie) +static void fscache_free_cookie(struct fscache_cookie *cookie) { - if (cookie) { - BUG_ON(!hlist_empty(&cookie->backing_objects)); - write_lock(&fscache_cookies_lock); - list_del(&cookie->proc_link); - write_unlock(&fscache_cookies_lock); - if (cookie->aux_len > sizeof(cookie->inline_aux)) - kfree(cookie->aux); - if (cookie->key_len > sizeof(cookie->inline_key)) - kfree(cookie->key); - kmem_cache_free(fscache_cookie_jar, cookie); + if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { + spin_lock(&fscache_cookie_lru_lock); + list_del_init(&cookie->commit_link); + spin_unlock(&fscache_cookie_lru_lock); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_removed); + } + + if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { + fscache_print_cookie(cookie, 'F'); + return; } + + write_lock(&fscache_cookies_lock); + list_del(&cookie->proc_link); + write_unlock(&fscache_cookies_lock); + if (cookie->aux_len > sizeof(cookie->inline_aux)) + kfree(cookie->aux); + if (cookie->key_len > sizeof(cookie->inline_key)) + kfree(cookie->key); + fscache_stat_d(&fscache_n_cookies); + kmem_cache_free(fscache_cookie_jar, cookie); +} + +static void __fscache_queue_cookie(struct fscache_cookie *cookie) +{ + if (!queue_work(fscache_wq, &cookie->work)) + fscache_put_cookie(cookie, fscache_cookie_put_over_queued); +} + +static void fscache_queue_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + fscache_get_cookie(cookie, where); + __fscache_queue_cookie(cookie); } /* + * Initialise the access gate on a cookie by setting a flag to prevent the + * state machine from being queued when the access counter transitions to 0. + * We're only interested in this when we withdraw caching services from the + * cookie. + */ +static void fscache_init_access_gate(struct fscache_cookie *cookie) +{ + int n_accesses; + + n_accesses = atomic_read(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_pin); + set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); +} + +/** + * fscache_end_cookie_access - Unpin a cache at the end of an access. + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache cookie after we've accessed it and bring a deferred + * relinquishment or withdrawal state into effect. + * + * The @why indicator is provided for tracing purposes. + */ +void fscache_end_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); + if (n_accesses == 0 && + !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} +EXPORT_SYMBOL(fscache_end_cookie_access); + +/* + * Pin the cache behind a cookie so that we can access it. + */ +static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&cookie->n_accesses); + smp_mb__after_atomic(); /* (Future) read state after is-caching. + * Reread n_accesses after is-caching + */ + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); +} + +/** + * fscache_begin_cookie_access - Pin a cache so data can be accessed + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing data and returns true if successful. This works as follows: + * + * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not + * set), we return false to indicate access was not permitted. + * + * (2) If the cookie is being cached, we increment its n_accesses count and + * then recheck the IS_CACHING flag, ending the access if it got cleared. + * + * (3) When we end the access, we decrement the cookie's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cookie is actively being cached, its n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline or if the cookie is culled, the flag is + * cleared to prevent new accesses, the cookie's n_accesses is decremented + * and we wait for it to become 0. + * + * The @why indicator are merely provided for tracing purposes. + */ +bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return false; + __fscache_begin_cookie_access(cookie, why); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || + !fscache_cache_is_live(cookie->volume->cache)) { + fscache_end_cookie_access(cookie, fscache_access_unlive); + return false; + } + return true; +} + +static inline void wake_up_cookie_state(struct fscache_cookie *cookie) +{ + /* Use a barrier to ensure that waiters see the state variable + * change, as spin_unlock doesn't guarantee a barrier. + * + * See comments over wake_up_bit() and waitqueue_active(). + */ + smp_mb(); + wake_up_var(&cookie->state); +} + +/* + * Change the state a cookie is at and wake up anyone waiting for that. Impose + * an ordering between the stuff stored in the cookie and the state member. + * Paired with fscache_cookie_state(). + */ +static void __fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + smp_store_release(&cookie->state, state); +} + +static void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + spin_lock(&cookie->lock); + __fscache_set_cookie_state(cookie, state); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); +} + +/** + * fscache_cookie_lookup_negative - Note negative lookup + * @cookie: The cookie that was being looked up + * + * Note that some part of the metadata path in the cache doesn't exist and so + * we can release any waiting readers in the certain knowledge that there's + * nothing for them to actually read. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); +} +EXPORT_SYMBOL(fscache_cookie_lookup_negative); + +/** + * fscache_resume_after_invalidation - Allow I/O to resume after invalidation + * @cookie: The cookie that was invalidated + * + * Tell fscache that invalidation is sufficiently complete that I/O can be + * allowed again. + */ +void fscache_resume_after_invalidation(struct fscache_cookie *cookie) +{ + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); +} +EXPORT_SYMBOL(fscache_resume_after_invalidation); + +/** + * fscache_caching_failed - Report that a failure stopped caching on a cookie + * @cookie: The cookie that was affected + * + * Tell fscache that caching on a cookie needs to be stopped due to some sort + * of failure. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_caching_failed(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); +} +EXPORT_SYMBOL(fscache_caching_failed); + +/* * Set the index key in a cookie. The cookie struct has space for a 16-byte * key plus length and hash, but if that's not big enough, it's instead a * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then @@ -87,38 +275,35 @@ void fscache_free_cookie(struct fscache_cookie *cookie) static int fscache_set_key(struct fscache_cookie *cookie, const void *index_key, size_t index_key_len) { - u32 *buf; - int bufs; + void *buf; + size_t buf_size; - bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf)); + buf_size = round_up(index_key_len, sizeof(__le32)); if (index_key_len > sizeof(cookie->inline_key)) { - buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL); + buf = kzalloc(buf_size, GFP_KERNEL); if (!buf) return -ENOMEM; cookie->key = buf; } else { - buf = (u32 *)cookie->inline_key; + buf = cookie->inline_key; } memcpy(buf, index_key, index_key_len); - cookie->key_hash = fscache_hash(0, buf, bufs); + cookie->key_hash = fscache_hash(cookie->volume->key_hash, + buf, buf_size); return 0; } -static long fscache_compare_cookie(const struct fscache_cookie *a, - const struct fscache_cookie *b) +static bool fscache_cookie_same(const struct fscache_cookie *a, + const struct fscache_cookie *b) { const void *ka, *kb; - if (a->key_hash != b->key_hash) - return (long)a->key_hash - (long)b->key_hash; - if (a->parent != b->parent) - return (long)a->parent - (long)b->parent; - if (a->key_len != b->key_len) - return (long)a->key_len - (long)b->key_len; - if (a->type != b->type) - return (long)a->type - (long)b->type; + if (a->key_hash != b->key_hash || + a->volume != b->volume || + a->key_len != b->key_len) + return false; if (a->key_len <= sizeof(a->inline_key)) { ka = &a->inline_key; @@ -127,7 +312,7 @@ static long fscache_compare_cookie(const struct fscache_cookie *a, ka = a->key; kb = b->key; } - return memcmp(ka, kb, a->key_len); + return memcmp(ka, kb, a->key_len) == 0; } static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); @@ -135,12 +320,11 @@ static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); /* * Allocate a cookie. */ -struct fscache_cookie *fscache_alloc_cookie( - struct fscache_cookie *parent, - const struct fscache_cookie_def *def, +static struct fscache_cookie *fscache_alloc_cookie( + struct fscache_volume *volume, + u8 advice, const void *index_key, size_t index_key_len, const void *aux_data, size_t aux_data_len, - void *netfs_data, loff_t object_size) { struct fscache_cookie *cookie; @@ -149,9 +333,15 @@ struct fscache_cookie *fscache_alloc_cookie( cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); if (!cookie) return NULL; + fscache_stat(&fscache_n_cookies); - cookie->key_len = index_key_len; - cookie->aux_len = aux_data_len; + cookie->volume = volume; + cookie->advice = advice; + cookie->key_len = index_key_len; + cookie->aux_len = aux_data_len; + cookie->object_size = object_size; + if (object_size == 0) + __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); if (fscache_set_key(cookie, index_key, index_key_len) < 0) goto nomem; @@ -165,30 +355,16 @@ struct fscache_cookie *fscache_alloc_cookie( } refcount_set(&cookie->ref, 1); - atomic_set(&cookie->n_children, 0); cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); - - /* We keep the active count elevated until relinquishment to prevent an - * attempt to wake up every time the object operations queue quiesces. - */ - atomic_set(&cookie->n_active, 1); - - cookie->def = def; - cookie->parent = parent; - cookie->netfs_data = netfs_data; - cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); - cookie->type = def->type; spin_lock_init(&cookie->lock); - spin_lock_init(&cookie->stores_lock); - INIT_HLIST_HEAD(&cookie->backing_objects); - - /* radix tree insertion won't use the preallocation pool unless it's - * told it may not wait */ - INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); + INIT_LIST_HEAD(&cookie->commit_link); + INIT_WORK(&cookie->work, fscache_cookie_worker); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); write_lock(&fscache_cookies_lock); list_add_tail(&cookie->proc_link, &fscache_cookies); write_unlock(&fscache_cookies_lock); + fscache_see_cookie(cookie, fscache_cookie_new_acquire); return cookie; nomem: @@ -196,13 +372,28 @@ nomem: return NULL; } +static void fscache_wait_on_collision(struct fscache_cookie *candidate, + struct fscache_cookie *wait_for) +{ + enum fscache_cookie_state *statep = &wait_for->state; + + wait_var_event_timeout(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED, + 20 * HZ); + if (READ_ONCE(*statep) != FSCACHE_COOKIE_STATE_DROPPED) { + pr_notice("Potential collision c=%08x old: c=%08x", + candidate->debug_id, wait_for->debug_id); + wait_var_event(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED); + } +} + /* * Attempt to insert the new cookie into the hash. If there's a collision, we - * return the old cookie if it's not in use and an error otherwise. + * wait for the old cookie to complete if it's being relinquished and an error + * otherwise. */ -struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate) +static bool fscache_hash_cookie(struct fscache_cookie *candidate) { - struct fscache_cookie *cursor; + struct fscache_cookie *cursor, *wait_for = NULL; struct hlist_bl_head *h; struct hlist_bl_node *p; unsigned int bucket; @@ -212,64 +403,53 @@ struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate) hlist_bl_lock(h); hlist_bl_for_each_entry(cursor, p, h, hash_link) { - if (fscache_compare_cookie(candidate, cursor) == 0) - goto collision; + if (fscache_cookie_same(candidate, cursor)) { + if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) + goto collision; + wait_for = fscache_get_cookie(cursor, + fscache_cookie_get_hash_collision); + break; + } } - __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags); - fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent); - atomic_inc(&candidate->parent->n_children); + fscache_get_volume(candidate->volume, fscache_volume_get_cookie); + atomic_inc(&candidate->volume->n_cookies); hlist_bl_add_head(&candidate->hash_link, h); + set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); hlist_bl_unlock(h); - return candidate; -collision: - if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) { - trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), - fscache_cookie_collision); - pr_err("Duplicate cookie detected\n"); - fscache_print_cookie(cursor, 'O'); - fscache_print_cookie(candidate, 'N'); - hlist_bl_unlock(h); - return NULL; + if (wait_for) { + fscache_wait_on_collision(candidate, wait_for); + fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); } + return true; - fscache_cookie_get(cursor, fscache_cookie_get_reacquire); +collision: + trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), + fscache_cookie_collision); + pr_err("Duplicate cookie detected\n"); + fscache_print_cookie(cursor, 'O'); + fscache_print_cookie(candidate, 'N'); hlist_bl_unlock(h); - return cursor; + return false; } /* - * request a cookie to represent an object (index, datafile, xattr, etc) - * - parent specifies the parent object - * - the top level index cookie for each netfs is stored in the fscache_netfs - * struct upon registration - * - def points to the definition - * - the netfs_data will be passed to the functions pointed to in *def - * - all attached caches will be searched to see if they contain this object - * - index objects aren't stored on disk until there's a dependent file that - * needs storing - * - other objects are stored in a selected cache immediately, and all the - * indices forming the path to it are instantiated if necessary - * - we never let on to the netfs about errors - * - we may set a negative cookie pointer, but that's okay + * Request a cookie to represent a data storage object within a volume. + * + * We never let on to the netfs about errors. We may set a negative cookie + * pointer, but that's okay */ struct fscache_cookie *__fscache_acquire_cookie( - struct fscache_cookie *parent, - const struct fscache_cookie_def *def, + struct fscache_volume *volume, + u8 advice, const void *index_key, size_t index_key_len, const void *aux_data, size_t aux_data_len, - void *netfs_data, - loff_t object_size, - bool enable) + loff_t object_size) { - struct fscache_cookie *candidate, *cookie; - - BUG_ON(!def); + struct fscache_cookie *cookie; - _enter("{%s},{%s},%p,%u", - parent ? (char *) parent->def->name : "<no-parent>", - def->name, netfs_data, enable); + _enter("V=%x", volume->debug_id); if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) return NULL; @@ -280,563 +460,440 @@ struct fscache_cookie *__fscache_acquire_cookie( fscache_stat(&fscache_n_acquires); - /* if there's no parent cookie, then we don't create one here either */ - if (!parent) { - fscache_stat(&fscache_n_acquires_null); - _leave(" [no parent]"); - return NULL; - } - - /* validate the definition */ - BUG_ON(!def->name[0]); - - BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX && - parent->type != FSCACHE_COOKIE_TYPE_INDEX); - - candidate = fscache_alloc_cookie(parent, def, - index_key, index_key_len, - aux_data, aux_data_len, - netfs_data, object_size); - if (!candidate) { + cookie = fscache_alloc_cookie(volume, advice, + index_key, index_key_len, + aux_data, aux_data_len, + object_size); + if (!cookie) { fscache_stat(&fscache_n_acquires_oom); - _leave(" [ENOMEM]"); return NULL; } - cookie = fscache_hash_cookie(candidate); - if (!cookie) { - trace_fscache_cookie(candidate->debug_id, 1, - fscache_cookie_discard); - goto out; - } - - if (cookie == candidate) - candidate = NULL; - - switch (cookie->type) { - case FSCACHE_COOKIE_TYPE_INDEX: - fscache_stat(&fscache_n_cookie_index); - break; - case FSCACHE_COOKIE_TYPE_DATAFILE: - fscache_stat(&fscache_n_cookie_data); - break; - default: - fscache_stat(&fscache_n_cookie_special); - break; + if (!fscache_hash_cookie(cookie)) { + fscache_see_cookie(cookie, fscache_cookie_discard); + fscache_free_cookie(cookie); + return NULL; } trace_fscache_acquire(cookie); - - if (enable) { - /* if the object is an index then we need do nothing more here - * - we create indices on disk when we need them as an index - * may exist in multiple caches */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } else { - atomic_dec(&parent->n_children); - fscache_cookie_put(cookie, - fscache_cookie_put_acquire_nobufs); - fscache_stat(&fscache_n_acquires_nobufs); - _leave(" = NULL"); - return NULL; - } - } else { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } - } - fscache_stat(&fscache_n_acquires_ok); - -out: - fscache_free_cookie(candidate); + _leave(" = c=%08x", cookie->debug_id); return cookie; } EXPORT_SYMBOL(__fscache_acquire_cookie); /* - * Enable a cookie to permit it to accept new operations. + * Prepare a cache object to be written to. */ -void __fscache_enable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - loff_t object_size, - bool (*can_enable)(void *data), - void *data) +static void fscache_prepare_to_write(struct fscache_cookie *cookie) { - _enter("%x", cookie->debug_id); - - trace_fscache_enable(cookie); - - wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, - TASK_UNINTERRUPTIBLE); - - fscache_update_aux(cookie, aux_data); - - if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) - goto out_unlock; - - if (can_enable && !can_enable(data)) { - /* The netfs decided it didn't want to enable after all */ - } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - /* Wait for outstanding disablement to complete */ - __fscache_wait_on_invalidate(cookie); - - if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } else { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } - -out_unlock: - clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); + cookie->volume->cache->ops->prepare_to_write(cookie); } -EXPORT_SYMBOL(__fscache_enable_cookie); /* - * acquire a non-index cookie - * - this must make sure the index chain is instantiated and instantiate the - * object representation too + * Look up a cookie in the cache. */ -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, - loff_t object_size) +static void fscache_perform_lookup(struct fscache_cookie *cookie) { - struct fscache_object *object; - struct fscache_cache *cache; - int ret; + enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; + bool need_withdraw = false; _enter(""); - set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - /* now we need to see whether the backing objects for this cookie yet - * exist, if not there'll be nothing to search */ - down_read(&fscache_addremove_sem); - - if (list_empty(&fscache_cache_list)) { - up_read(&fscache_addremove_sem); - _leave(" = 0 [no caches]"); - return 0; - } - - /* select a cache in which to store the object */ - cache = fscache_select_cache_for_object(cookie->parent); - if (!cache) { - up_read(&fscache_addremove_sem); - fscache_stat(&fscache_n_acquires_no_cache); - _leave(" = -ENOMEDIUM [no cache]"); - return -ENOMEDIUM; - } - - _debug("cache %s", cache->tag->name); - - set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - - /* ask the cache to allocate objects for this cookie and its parent - * chain */ - ret = fscache_alloc_object(cache, cookie); - if (ret < 0) { - up_read(&fscache_addremove_sem); - _leave(" = %d", ret); - return ret; - } - - spin_lock(&cookie->lock); - if (hlist_empty(&cookie->backing_objects)) { - spin_unlock(&cookie->lock); - goto unavailable; + if (!cookie->volume->cache_priv) { + fscache_create_volume(cookie->volume, true); + if (!cookie->volume->cache_priv) { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + goto out; + } } - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - fscache_set_store_limit(object, object_size); - - /* initiate the process of looking up all the objects in the chain - * (done by fscache_initialise_object()) */ - fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD); - - spin_unlock(&cookie->lock); - - /* we may be required to wait for lookup to complete at this point */ - if (!fscache_defer_lookup) { - wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, - TASK_UNINTERRUPTIBLE); - if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags)) - goto unavailable; + if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { + if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + need_withdraw = true; + _leave(" [fail]"); + goto out; } - up_read(&fscache_addremove_sem); - _leave(" = 0 [deferred]"); - return 0; + fscache_see_cookie(cookie, fscache_cookie_see_active); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); + trace = fscache_access_lookup_cookie_end; -unavailable: - up_read(&fscache_addremove_sem); - _leave(" = -ENOBUFS"); - return -ENOBUFS; +out: + fscache_end_cookie_access(cookie, trace); + if (need_withdraw) + fscache_withdraw_cookie(cookie); + fscache_end_volume_access(cookie->volume, cookie, trace); } /* - * recursively allocate cache object records for a cookie/cache combination - * - caller must be holding the addremove sem + * Begin the process of looking up a cookie. We offload the actual process to + * a worker thread. */ -static int fscache_alloc_object(struct fscache_cache *cache, - struct fscache_cookie *cookie) +static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) { - struct fscache_object *object; - int ret; - - _enter("%s,%x{%s}", cache->tag->name, cookie->debug_id, cookie->def->name); - - spin_lock(&cookie->lock); - hlist_for_each_entry(object, &cookie->backing_objects, - cookie_link) { - if (object->cache == cache) - goto object_already_extant; + if (will_modify) { + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); } - spin_unlock(&cookie->lock); - - /* ask the cache to allocate an object (we may end up with duplicate - * objects at this stage, but we sort that out later) */ - fscache_stat(&fscache_n_cop_alloc_object); - object = cache->ops->alloc_object(cache, cookie); - fscache_stat_d(&fscache_n_cop_alloc_object); - if (IS_ERR(object)) { - fscache_stat(&fscache_n_object_no_alloc); - ret = PTR_ERR(object); - goto error; - } - - ASSERTCMP(object->cookie, ==, cookie); - fscache_stat(&fscache_n_object_alloc); - - object->debug_id = atomic_inc_return(&fscache_object_debug_id); - - _debug("ALLOC OBJ%x: %s {%lx}", - object->debug_id, cookie->def->name, object->events); - - ret = fscache_alloc_object(cache, cookie->parent); - if (ret < 0) - goto error_put; - - /* only attach if we managed to allocate all we needed, otherwise - * discard the object we just allocated and instead use the one - * attached to the cookie */ - if (fscache_attach_object(cookie, object) < 0) { - fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object, fscache_obj_put_attach_fail); - fscache_stat_d(&fscache_n_cop_put_object); - } - - _leave(" = 0"); - return 0; - -object_already_extant: - ret = -ENOBUFS; - if (fscache_object_is_dying(object) || - fscache_cache_is_broken(object)) { - spin_unlock(&cookie->lock); - goto error; - } - spin_unlock(&cookie->lock); - _leave(" = 0 [found]"); - return 0; - -error_put: - fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object, fscache_obj_put_alloc_fail); - fscache_stat_d(&fscache_n_cop_put_object); -error: - _leave(" = %d", ret); - return ret; + if (!fscache_begin_volume_access(cookie->volume, cookie, + fscache_access_lookup_cookie)) + return false; + + __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); + set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); + return true; } /* - * attach a cache object to a cookie + * Start using the cookie for I/O. This prevents the backing object from being + * reaped by VM pressure. */ -static int fscache_attach_object(struct fscache_cookie *cookie, - struct fscache_object *object) +void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) { - struct fscache_object *p; - struct fscache_cache *cache = object->cache; - int ret; + enum fscache_cookie_state state; + bool queue = false; + int n_active; - _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id); + _enter("c=%08x", cookie->debug_id); - ASSERTCMP(object->cookie, ==, cookie); + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to use relinquished cookie\n")) + return; spin_lock(&cookie->lock); - /* there may be multiple initial creations of this object, but we only - * want one */ - ret = -EEXIST; - hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) { - if (p->cache == object->cache) { - if (fscache_object_is_dying(p)) - ret = -ENOBUFS; - goto cant_attach_object; - } - } + n_active = atomic_inc_return(&cookie->n_active); + trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), + n_active, atomic_read(&cookie->n_accesses), + will_modify ? + fscache_active_use_modify : fscache_active_use); + +again: + state = fscache_cookie_state(cookie); + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + queue = fscache_begin_lookup(cookie, will_modify); + break; - /* pin the parent object */ - spin_lock_nested(&cookie->parent->lock, 1); - hlist_for_each_entry(p, &cookie->parent->backing_objects, - cookie_link) { - if (p->cache == object->cache) { - if (fscache_object_is_dying(p)) { - ret = -ENOBUFS; - spin_unlock(&cookie->parent->lock); - goto cant_attach_object; - } - object->parent = p; - spin_lock(&p->lock); - p->n_children++; - spin_unlock(&p->lock); - break; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + if (will_modify) + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + break; + case FSCACHE_COOKIE_STATE_ACTIVE: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (will_modify && + !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + queue = true; } - } - spin_unlock(&cookie->parent->lock); - - /* attach to the cache's object list */ - if (list_empty(&object->cache_link)) { - spin_lock(&cache->object_list_lock); - list_add(&object->cache_link, &cache->object_list); - spin_unlock(&cache->object_list_lock); - } - - /* Attach to the cookie. The object already has a ref on it. */ - hlist_add_head(&object->cookie_link, &cookie->backing_objects); - ret = 0; - -cant_attach_object: - spin_unlock(&cookie->lock); - _leave(" = %d", ret); - return ret; -} - -/* - * Invalidate an object. Callable with spinlocks held. - */ -void __fscache_invalidate(struct fscache_cookie *cookie) -{ - struct fscache_object *object; - - _enter("{%s}", cookie->def->name); - - fscache_stat(&fscache_n_invalidates); + break; - /* Only permit invalidation of data files. Invalidating an index will - * require the caller to release all its attachments to the tree rooted - * there, and if it's doing that, it may as well just retire the - * cookie. - */ - ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); + case FSCACHE_COOKIE_STATE_FAILED: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + break; - /* If there's an object, we tell the object state machine to handle the - * invalidation on our behalf, otherwise there's nothing to do. - */ - if (!hlist_empty(&cookie->backing_objects)) { + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + spin_unlock(&cookie->lock); + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != + FSCACHE_COOKIE_STATE_LRU_DISCARDING); spin_lock(&cookie->lock); + goto again; - if (fscache_cookie_enabled(cookie) && - !hlist_empty(&cookie->backing_objects) && - !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING, - &cookie->flags)) { - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, - cookie_link); - if (fscache_object_is_live(object)) - fscache_raise_event( - object, FSCACHE_OBJECT_EV_INVALIDATE); - } - - spin_unlock(&cookie->lock); + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", state); + break; } + spin_unlock(&cookie->lock); + if (queue) + fscache_queue_cookie(cookie, fscache_cookie_get_use_work); _leave(""); } -EXPORT_SYMBOL(__fscache_invalidate); +EXPORT_SYMBOL(__fscache_use_cookie); -/* - * Wait for object invalidation to complete. - */ -void __fscache_wait_on_invalidate(struct fscache_cookie *cookie) +static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) { - _enter("%x", cookie->debug_id); + clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return; - wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING, - TASK_UNINTERRUPTIBLE); + cookie->unused_at = jiffies; + spin_lock(&fscache_cookie_lru_lock); + if (list_empty(&cookie->commit_link)) { + fscache_get_cookie(cookie, fscache_cookie_get_lru); + fscache_stat(&fscache_n_cookies_lru); + } + list_move_tail(&cookie->commit_link, &fscache_cookie_lru); - _leave(""); + spin_unlock(&fscache_cookie_lru_lock); + timer_reduce(&fscache_cookie_lru_timer, + jiffies + fscache_lru_cookie_timeout); } -EXPORT_SYMBOL(__fscache_wait_on_invalidate); /* - * update the index entries backing a cookie + * Stop using the cookie for I/O. */ -void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data) +void __fscache_unuse_cookie(struct fscache_cookie *cookie, + const void *aux_data, const loff_t *object_size) { - struct fscache_object *object; - - fscache_stat(&fscache_n_updates); - - if (!cookie) { - fscache_stat(&fscache_n_updates_null); - _leave(" [no cookie]"); + unsigned int debug_id = cookie->debug_id; + unsigned int r = refcount_read(&cookie->ref); + unsigned int a = atomic_read(&cookie->n_accesses); + unsigned int c; + + if (aux_data || object_size) + __fscache_update_cookie(cookie, aux_data, object_size); + + /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ + c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); + if (c != 1) { + trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); return; } - _enter("{%s}", cookie->def->name); - spin_lock(&cookie->lock); - - fscache_update_aux(cookie, aux_data); - - if (fscache_cookie_enabled(cookie)) { - /* update the index entry on disk in each cache backing this - * cookie. - */ - hlist_for_each_entry(object, - &cookie->backing_objects, cookie_link) { - fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE); - } - } - + r = refcount_read(&cookie->ref); + a = atomic_read(&cookie->n_accesses); + c = atomic_dec_return(&cookie->n_active); + trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); + if (c == 0) + fscache_unuse_cookie_locked(cookie); spin_unlock(&cookie->lock); - _leave(""); } -EXPORT_SYMBOL(__fscache_update_cookie); +EXPORT_SYMBOL(__fscache_unuse_cookie); /* - * Disable a cookie to stop it from accepting new requests from the netfs. + * Perform work upon the cookie, such as committing its cache state, + * relinquishing it or withdrawing the backing cache. We're protected from the + * cache going away under us as object withdrawal must come through this + * non-reentrant work item. */ -void __fscache_disable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool invalidate) +static void fscache_cookie_state_machine(struct fscache_cookie *cookie) { - struct fscache_object *object; - bool awaken = false; + enum fscache_cookie_state state; + bool wake = false; - _enter("%x,%u", cookie->debug_id, invalidate); + _enter("c=%x", cookie->debug_id); - trace_fscache_disable(cookie); - - ASSERTCMP(atomic_read(&cookie->n_active), >, 0); - - if (atomic_read(&cookie->n_children) != 0) { - pr_err("Cookie '%s' still has children\n", - cookie->def->name); - BUG(); - } +again: + spin_lock(&cookie->lock); +again_locked: + state = cookie->state; + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + /* The QUIESCENT state is jumped to the LOOKING_UP state by + * fscache_use_cookie(). + */ - wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, - TASK_UNINTERRUPTIBLE); + if (atomic_read(&cookie->n_accesses) == 0 && + test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + break; - fscache_update_aux(cookie, aux_data); + case FSCACHE_COOKIE_STATE_LOOKING_UP: + spin_unlock(&cookie->lock); + fscache_init_access_gate(cookie); + fscache_perform_lookup(cookie); + goto again; - if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) - goto out_unlock_enable; + case FSCACHE_COOKIE_STATE_INVALIDATING: + spin_unlock(&cookie->lock); + fscache_perform_invalidation(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { + spin_unlock(&cookie->lock); + fscache_prepare_to_write(cookie); + spin_lock(&cookie->lock); + } + if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + wake = true; + goto again_locked; + } + fallthrough; - /* If the cookie is being invalidated, wait for that to complete first - * so that we can reuse the flag. - */ - __fscache_wait_on_invalidate(cookie); + case FSCACHE_COOKIE_STATE_FAILED: + if (atomic_read(&cookie->n_accesses) != 0) + break; + if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_WITHDRAWING); + wake = true; + goto again_locked; + } + break; - /* Dispose of the backing objects */ - set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags); + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + if (cookie->cache_priv) { + spin_unlock(&cookie->lock); + cookie->volume->cache->ops->withdraw_cookie(cookie); + spin_lock(&cookie->lock); + } - spin_lock(&cookie->lock); - if (!hlist_empty(&cookie->backing_objects)) { - hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) { - if (invalidate) - set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL); + switch (state) { + case FSCACHE_COOKIE_STATE_RELINQUISHING: + fscache_see_cookie(cookie, fscache_cookie_see_relinquish); + fscache_unhash_cookie(cookie); + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_DROPPED); + wake = true; + goto out; + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); + break; + case FSCACHE_COOKIE_STATE_WITHDRAWING: + fscache_see_cookie(cookie, fscache_cookie_see_withdraw); + break; + default: + BUG(); } - } else { - if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - awaken = true; - } - spin_unlock(&cookie->lock); - if (awaken) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - /* Wait for cessation of activity requiring access to the netfs (when - * n_active reaches 0). This makes sure outstanding reads and writes - * have completed. - */ - if (!atomic_dec_and_test(&cookie->n_active)) { - wait_var_event(&cookie->n_active, - !atomic_read(&cookie->n_active)); - } + clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + wake = true; + goto again_locked; - /* Make sure any pending writes are cancelled. */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) - fscache_invalidate_writes(cookie); + case FSCACHE_COOKIE_STATE_DROPPED: + break; - /* Reset the cookie state if it wasn't relinquished */ - if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) { - atomic_inc(&cookie->n_active); - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); + default: + WARN_ONCE(1, "Cookie %x in unexpected state %u\n", + cookie->debug_id, state); + break; } -out_unlock_enable: - clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); +out: + spin_unlock(&cookie->lock); + if (wake) + wake_up_cookie_state(cookie); _leave(""); } -EXPORT_SYMBOL(__fscache_disable_cookie); + +static void fscache_cookie_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); + + fscache_see_cookie(cookie, fscache_cookie_see_work); + fscache_cookie_state_machine(cookie); + fscache_put_cookie(cookie, fscache_cookie_put_work); +} /* - * release a cookie back to the cache - * - the object will be marked as recyclable on disk if retire is true - * - all dependents of this cookie must have already been unregistered - * (indices/files/pages) + * Wait for the object to become inactive. The cookie's work item will be + * scheduled when someone transitions n_accesses to 0 - but if someone's + * already done that, schedule it anyway. */ -void __fscache_relinquish_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool retire) +static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) { - fscache_stat(&fscache_n_relinquishes); - if (retire) - fscache_stat(&fscache_n_relinquishes_retire); + int n_accesses; + bool unpinned; + + unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); + + /* Need to read the access count after unpinning */ + n_accesses = atomic_read(&cookie->n_accesses); + if (unpinned) + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_unpin); + if (n_accesses == 0) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} - if (!cookie) { - fscache_stat(&fscache_n_relinquishes_null); - _leave(" [no cookie]"); - return; - } +static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) +{ + fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); - _enter("%x{%s,%d},%d", - cookie->debug_id, cookie->def->name, - atomic_read(&cookie->n_active), retire); + spin_lock(&cookie->lock); + if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || + time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || + atomic_read(&cookie->n_active) > 0) { + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_removed); + } else { + set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_expired); + _debug("lru c=%x", cookie->debug_id); + __fscache_withdraw_cookie(cookie); + } - trace_fscache_relinquish(cookie, retire); + fscache_put_cookie(cookie, fscache_cookie_put_lru); +} - /* No further netfs-accessing operations on this cookie permitted */ - if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) - BUG(); +static void fscache_cookie_lru_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie; + unsigned long unused_at; - __fscache_disable_cookie(cookie, aux_data, retire); + spin_lock(&fscache_cookie_lru_lock); - /* Clear pointers back to the netfs */ - cookie->netfs_data = NULL; - cookie->def = NULL; - BUG_ON(!radix_tree_empty(&cookie->stores)); + while (!list_empty(&fscache_cookie_lru)) { + cookie = list_first_entry(&fscache_cookie_lru, + struct fscache_cookie, commit_link); + unused_at = cookie->unused_at + fscache_lru_cookie_timeout; + if (time_before(jiffies, unused_at)) { + timer_reduce(&fscache_cookie_lru_timer, unused_at); + break; + } - if (cookie->parent) { - ASSERTCMP(refcount_read(&cookie->parent->ref), >, 0); - ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0); - atomic_dec(&cookie->parent->n_children); + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + spin_unlock(&fscache_cookie_lru_lock); + fscache_cookie_lru_do_one(cookie); + spin_lock(&fscache_cookie_lru_lock); } - /* Dispose of the netfs's link to the cookie */ - fscache_cookie_put(cookie, fscache_cookie_put_relinquish); + spin_unlock(&fscache_cookie_lru_lock); +} - _leave(""); +static void fscache_cookie_lru_timed_out(struct timer_list *timer) +{ + queue_work(fscache_wq, &fscache_cookie_lru_work); +} + +static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) +{ + bool need_put = false; + + if (!list_empty(&cookie->commit_link)) { + spin_lock(&fscache_cookie_lru_lock); + if (!list_empty(&cookie->commit_link)) { + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_dropped); + need_put = true; + } + spin_unlock(&fscache_cookie_lru_lock); + if (need_put) + fscache_put_cookie(cookie, fscache_cookie_put_lru); + } } -EXPORT_SYMBOL(__fscache_relinquish_cookie); /* * Remove a cookie from the hash table. @@ -851,43 +908,91 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie) hlist_bl_lock(h); hlist_bl_del(&cookie->hash_link); + clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); hlist_bl_unlock(h); + fscache_stat(&fscache_n_relinquishes_dropped); } +static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) +{ + fscache_cookie_drop_from_lru(cookie); + __fscache_withdraw_cookie(cookie); +} + +/** + * fscache_withdraw_cookie - Mark a cookie for withdrawal + * @cookie: The cookie to be withdrawn. + * + * Allow the cache backend to withdraw the backing for a cookie for its own + * reasons, even if that cookie is in active use. + */ +void fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); +} +EXPORT_SYMBOL(fscache_withdraw_cookie); + /* - * Drop a reference to a cookie. + * Allow the netfs to release a cookie back to the cache. + * - the object will be marked as recyclable on disk if retire is true */ -void fscache_cookie_put(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) +void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) { - struct fscache_cookie *parent; - int ref; + fscache_stat(&fscache_n_relinquishes); + if (retire) + fscache_stat(&fscache_n_relinquishes_retire); + + _enter("c=%08x{%d},%d", + cookie->debug_id, atomic_read(&cookie->n_active), retire); - _enter("%x", cookie->debug_id); + if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Cookie c=%x already relinquished\n", cookie->debug_id)) + return; - do { - unsigned int cookie_debug_id = cookie->debug_id; - bool zero = __refcount_dec_and_test(&cookie->ref, &ref); + if (retire) + set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); + trace_fscache_relinquish(cookie, retire); - trace_fscache_cookie(cookie_debug_id, ref - 1, where); - if (!zero) - return; + ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); + ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); + atomic_dec(&cookie->volume->n_cookies); - parent = cookie->parent; + if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); + } else { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); fscache_unhash_cookie(cookie); - fscache_free_cookie(cookie); + } + fscache_put_cookie(cookie, fscache_cookie_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_cookie); - cookie = parent; - where = fscache_cookie_put_parent; - } while (cookie); +/* + * Drop a reference to a cookie. + */ +void fscache_put_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + struct fscache_volume *volume = cookie->volume; + unsigned int cookie_debug_id = cookie->debug_id; + bool zero; + int ref; - _leave(""); + zero = __refcount_dec_and_test(&cookie->ref, &ref); + trace_fscache_cookie(cookie_debug_id, ref - 1, where); + if (zero) { + fscache_free_cookie(cookie); + fscache_put_volume(volume, fscache_volume_put_cookie); + } } +EXPORT_SYMBOL(fscache_put_cookie); /* * Get a reference to a cookie. */ -struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *cookie, +struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where) { int ref; @@ -896,85 +1001,73 @@ struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *cookie, trace_fscache_cookie(cookie->debug_id, ref + 1, where); return cookie; } +EXPORT_SYMBOL(fscache_get_cookie); /* - * check the consistency between the netfs inode and the backing cache - * - * NOTE: it only serves no-index type + * Ask the cache to effect invalidation of a cookie. */ -int __fscache_check_consistency(struct fscache_cookie *cookie, - const void *aux_data) +static void fscache_perform_invalidation(struct fscache_cookie *cookie) { - struct fscache_operation *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,", cookie); + if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) + fscache_caching_failed(cookie); + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); +} - ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); +/* + * Invalidate an object. + */ +void __fscache_invalidate(struct fscache_cookie *cookie, + const void *aux_data, loff_t new_size, + unsigned int flags) +{ + bool is_caching; - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; + _enter("c=%x", cookie->debug_id); - if (hlist_empty(&cookie->backing_objects)) - return 0; + fscache_stat(&fscache_n_invalidates); - op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY); - if (!op) - return -ENOMEM; + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to invalidate relinquished cookie\n")) + return; - fscache_operation_init(cookie, op, NULL, NULL, NULL); - op->flags = FSCACHE_OP_MYTHREAD | - (1 << FSCACHE_OP_WAITING) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency); + if ((flags & FSCACHE_INVAL_DIO_WRITE) && + test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) + return; spin_lock(&cookie->lock); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_update_aux(cookie, aux_data, &new_size); + cookie->inval_counter++; + trace_fscache_invalidate(cookie, new_size); - fscache_update_aux(cookie, aux_data); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto inconsistent; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) - goto inconsistent; - - op->debug_id = atomic_inc_return(&fscache_op_debug_id); + switch (cookie->state) { + case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ + default: + spin_unlock(&cookie->lock); + _leave(" [no %u]", cookie->state); + return; - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, op) < 0) - goto submit_failed; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + spin_unlock(&cookie->lock); + _leave(" [look %x]", cookie->inval_counter); + return; - /* the work queue now carries its own ref on the object */ - spin_unlock(&cookie->lock); + case FSCACHE_COOKIE_STATE_ACTIVE: + is_caching = fscache_begin_cookie_access( + cookie, fscache_access_invalidate_cookie); + if (is_caching) + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); - ret = fscache_wait_for_operation_activation(object, op, NULL, NULL); - if (ret == 0) { - /* ask the cache to honour the operation */ - ret = object->cache->ops->check_consistency(op); - fscache_op_complete(op, false); - } else if (ret == -ENOBUFS) { - ret = 0; + if (is_caching) + fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); + _leave(" [inv]"); + return; } - - fscache_put_operation(op); - _leave(" = %d", ret); - return ret; - -submit_failed: - wake_cookie = __fscache_unuse_cookie(cookie); -inconsistent: - spin_unlock(&cookie->lock); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - kfree(op); - _leave(" = -ESTALE"); - return -ESTALE; } -EXPORT_SYMBOL(__fscache_check_consistency); +EXPORT_SYMBOL(__fscache_invalidate); /* * Generate a list of extant cookies in /proc/fs/fscache/cookies @@ -983,44 +1076,27 @@ static int fscache_cookies_seq_show(struct seq_file *m, void *v) { struct fscache_cookie *cookie; unsigned int keylen = 0, auxlen = 0; - char _type[3], *type; u8 *p; if (v == &fscache_cookies) { seq_puts(m, - "COOKIE PARENT USAGE CHILD ACT TY FL DEF NETFS_DATA\n" - "======== ======== ===== ===== === == === ================ ==========\n" + "COOKIE VOLUME REF ACT ACC S FL DEF \n" + "======== ======== === === === = == ================\n" ); return 0; } cookie = list_entry(v, struct fscache_cookie, proc_link); - switch (cookie->type) { - case 0: - type = "IX"; - break; - case 1: - type = "DT"; - break; - default: - snprintf(_type, sizeof(_type), "%02u", - cookie->type); - type = _type; - break; - } - seq_printf(m, - "%08x %08x %5u %5u %3u %s %03lx %-16s %px", + "%08x %08x %3d %3d %3d %c %02lx", cookie->debug_id, - cookie->parent ? cookie->parent->debug_id : 0, + cookie->volume->debug_id, refcount_read(&cookie->ref), - atomic_read(&cookie->n_children), atomic_read(&cookie->n_active), - type, - cookie->flags, - cookie->def->name, - cookie->netfs_data); + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state], + cookie->flags); keylen = cookie->key_len; auxlen = cookie->aux_len; diff --git a/fs/fscache/fsdef.c b/fs/fscache/fsdef.c deleted file mode 100644 index 0402673c680e..000000000000 --- a/fs/fscache/fsdef.c +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Filesystem index definition - * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include <linux/module.h> -#include "internal.h" - -static -enum fscache_checkaux fscache_fsdef_netfs_check_aux(void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size); - -/* - * The root index is owned by FS-Cache itself. - * - * When a netfs requests caching facilities, FS-Cache will, if one doesn't - * already exist, create an entry in the root index with the key being the name - * of the netfs ("AFS" for example), and the auxiliary data holding the index - * structure version supplied by the netfs: - * - * FSDEF - * | - * +-----------+ - * | | - * NFS AFS - * [v=1] [v=1] - * - * If an entry with the appropriate name does already exist, the version is - * compared. If the version is different, the entire subtree from that entry - * will be discarded and a new entry created. - * - * The new entry will be an index, and a cookie referring to it will be passed - * to the netfs. This is then the root handle by which the netfs accesses the - * cache. It can create whatever objects it likes in that index, including - * further indices. - */ -static struct fscache_cookie_def fscache_fsdef_index_def = { - .name = ".FS-Cache", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -struct fscache_cookie fscache_fsdef_index = { - .debug_id = 1, - .ref = REFCOUNT_INIT(1), - .n_active = ATOMIC_INIT(1), - .lock = __SPIN_LOCK_UNLOCKED(fscache_fsdef_index.lock), - .backing_objects = HLIST_HEAD_INIT, - .def = &fscache_fsdef_index_def, - .flags = 1 << FSCACHE_COOKIE_ENABLED, - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; -EXPORT_SYMBOL(fscache_fsdef_index); - -/* - * Definition of an entry in the root index. Each entry is an index, keyed to - * a specific netfs and only applicable to a particular version of the index - * structure used by that netfs. - */ -struct fscache_cookie_def fscache_fsdef_netfs_def = { - .name = "FSDEF.netfs", - .type = FSCACHE_COOKIE_TYPE_INDEX, - .check_aux = fscache_fsdef_netfs_check_aux, -}; - -/* - * check that the index structure version number stored in the auxiliary data - * matches the one the netfs gave us - */ -static enum fscache_checkaux fscache_fsdef_netfs_check_aux( - void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size) -{ - struct fscache_netfs *netfs = cookie_netfs_data; - uint32_t version; - - _enter("{%s},,%hu", netfs->name, datalen); - - if (datalen != sizeof(version)) { - _leave(" = OBSOLETE [dl=%d v=%zu]", datalen, sizeof(version)); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - memcpy(&version, data, sizeof(version)); - if (version != netfs->version) { - _leave(" = OBSOLETE [ver=%x net=%x]", version, netfs->version); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - _leave(" = OKAY"); - return FSCACHE_CHECKAUX_OKAY; -} diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index c3e4804b8fcb..f121c21590dc 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -1,65 +1,69 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* Internal definitions for FS-Cache * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ -/* - * Lock order, in the order in which multiple locks should be obtained: - * - fscache_addremove_sem - * - cookie->lock - * - cookie->parent->lock - * - cache->object_list_lock - * - object->lock - * - object->parent->lock - * - cookie->stores_lock - * - fscache_thread_lock - * - */ - #ifdef pr_fmt #undef pr_fmt #endif #define pr_fmt(fmt) "FS-Cache: " fmt +#include <linux/slab.h> #include <linux/fscache-cache.h> #include <trace/events/fscache.h> #include <linux/sched.h> #include <linux/seq_file.h> -#define FSCACHE_MIN_THREADS 4 -#define FSCACHE_MAX_THREADS 32 - /* * cache.c */ -extern struct list_head fscache_cache_list; -extern struct rw_semaphore fscache_addremove_sem; +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_caches_seq_ops; +#endif +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); +void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); + +static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) +{ + return smp_load_acquire(&cache->state); +} + +static inline bool fscache_cache_is_live(const struct fscache_cache *cache) +{ + return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; +} -extern struct fscache_cache *fscache_select_cache_for_object( - struct fscache_cookie *); +static inline void fscache_set_cache_state(struct fscache_cache *cache, + enum fscache_cache_state new_state) +{ + smp_store_release(&cache->state, new_state); + +} + +static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, + enum fscache_cache_state old_state, + enum fscache_cache_state new_state) +{ + return try_cmpxchg_release(&cache->state, &old_state, new_state); +} /* * cookie.c */ extern struct kmem_cache *fscache_cookie_jar; extern const struct seq_operations fscache_cookies_seq_ops; +extern struct timer_list fscache_cookie_lru_timer; + +extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); +extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why); -extern void fscache_free_cookie(struct fscache_cookie *); -extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *, - const struct fscache_cookie_def *, - const void *, size_t, - const void *, size_t, - void *, loff_t); -extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *); -extern struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *, - enum fscache_cookie_trace); -extern void fscache_cookie_put(struct fscache_cookie *, - enum fscache_cookie_trace); - -static inline void fscache_cookie_see(struct fscache_cookie *cookie, +static inline void fscache_see_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where) { trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), @@ -67,60 +71,22 @@ static inline void fscache_cookie_see(struct fscache_cookie *cookie, } /* - * fsdef.c + * io.c */ -extern struct fscache_cookie fscache_fsdef_index; -extern struct fscache_cookie_def fscache_fsdef_netfs_def; - -/* - * main.c - */ -extern unsigned fscache_defer_lookup; -extern unsigned fscache_defer_create; -extern unsigned fscache_debug; -extern struct kobject *fscache_root; -extern struct workqueue_struct *fscache_object_wq; -extern struct workqueue_struct *fscache_op_wq; -DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); - -extern unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n); - -static inline bool fscache_object_congested(void) +static inline void fscache_end_operation(struct netfs_cache_resources *cres) { - return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq); + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + + if (ops) + ops->end_operation(cres); } /* - * object.c + * main.c */ -extern void fscache_enqueue_object(struct fscache_object *); +extern unsigned fscache_debug; -/* - * operation.c - */ -extern int fscache_submit_exclusive_op(struct fscache_object *, - struct fscache_operation *); -extern int fscache_submit_op(struct fscache_object *, - struct fscache_operation *); -extern int fscache_cancel_op(struct fscache_operation *, bool); -extern void fscache_cancel_all_ops(struct fscache_object *); -extern void fscache_abort_object(struct fscache_object *); -extern void fscache_start_operations(struct fscache_object *); -extern void fscache_operation_gc(struct work_struct *); - -/* - * page.c - */ -extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *); -extern int fscache_wait_for_operation_activation(struct fscache_object *, - struct fscache_operation *, - atomic_t *, - atomic_t *); -extern void fscache_invalidate_writes(struct fscache_cookie *); -struct fscache_retrieval *fscache_alloc_retrieval(struct fscache_cookie *cookie, - struct address_space *mapping, - fscache_rw_complete_t end_io_func, - void *context); +extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); /* * proc.c @@ -137,125 +103,27 @@ extern void fscache_proc_cleanup(void); * stats.c */ #ifdef CONFIG_FSCACHE_STATS -extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS]; -extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS]; - -extern atomic_t fscache_n_op_pend; -extern atomic_t fscache_n_op_run; -extern atomic_t fscache_n_op_enqueue; -extern atomic_t fscache_n_op_deferred_release; -extern atomic_t fscache_n_op_initialised; -extern atomic_t fscache_n_op_release; -extern atomic_t fscache_n_op_gc; -extern atomic_t fscache_n_op_cancelled; -extern atomic_t fscache_n_op_rejected; - -extern atomic_t fscache_n_attr_changed; -extern atomic_t fscache_n_attr_changed_ok; -extern atomic_t fscache_n_attr_changed_nobufs; -extern atomic_t fscache_n_attr_changed_nomem; -extern atomic_t fscache_n_attr_changed_calls; - -extern atomic_t fscache_n_allocs; -extern atomic_t fscache_n_allocs_ok; -extern atomic_t fscache_n_allocs_wait; -extern atomic_t fscache_n_allocs_nobufs; -extern atomic_t fscache_n_allocs_intr; -extern atomic_t fscache_n_allocs_object_dead; -extern atomic_t fscache_n_alloc_ops; -extern atomic_t fscache_n_alloc_op_waits; - -extern atomic_t fscache_n_retrievals; -extern atomic_t fscache_n_retrievals_ok; -extern atomic_t fscache_n_retrievals_wait; -extern atomic_t fscache_n_retrievals_nodata; -extern atomic_t fscache_n_retrievals_nobufs; -extern atomic_t fscache_n_retrievals_intr; -extern atomic_t fscache_n_retrievals_nomem; -extern atomic_t fscache_n_retrievals_object_dead; -extern atomic_t fscache_n_retrieval_ops; -extern atomic_t fscache_n_retrieval_op_waits; - -extern atomic_t fscache_n_stores; -extern atomic_t fscache_n_stores_ok; -extern atomic_t fscache_n_stores_again; -extern atomic_t fscache_n_stores_nobufs; -extern atomic_t fscache_n_stores_oom; -extern atomic_t fscache_n_store_ops; -extern atomic_t fscache_n_store_calls; -extern atomic_t fscache_n_store_pages; -extern atomic_t fscache_n_store_radix_deletes; -extern atomic_t fscache_n_store_pages_over_limit; - -extern atomic_t fscache_n_store_vmscan_not_storing; -extern atomic_t fscache_n_store_vmscan_gone; -extern atomic_t fscache_n_store_vmscan_busy; -extern atomic_t fscache_n_store_vmscan_cancelled; -extern atomic_t fscache_n_store_vmscan_wait; - -extern atomic_t fscache_n_marks; -extern atomic_t fscache_n_uncaches; +extern atomic_t fscache_n_volumes; +extern atomic_t fscache_n_volumes_collision; +extern atomic_t fscache_n_volumes_nomem; +extern atomic_t fscache_n_cookies; +extern atomic_t fscache_n_cookies_lru; +extern atomic_t fscache_n_cookies_lru_expired; +extern atomic_t fscache_n_cookies_lru_removed; +extern atomic_t fscache_n_cookies_lru_dropped; extern atomic_t fscache_n_acquires; -extern atomic_t fscache_n_acquires_null; -extern atomic_t fscache_n_acquires_no_cache; extern atomic_t fscache_n_acquires_ok; -extern atomic_t fscache_n_acquires_nobufs; extern atomic_t fscache_n_acquires_oom; extern atomic_t fscache_n_invalidates; -extern atomic_t fscache_n_invalidates_run; - -extern atomic_t fscache_n_updates; -extern atomic_t fscache_n_updates_null; -extern atomic_t fscache_n_updates_run; extern atomic_t fscache_n_relinquishes; -extern atomic_t fscache_n_relinquishes_null; -extern atomic_t fscache_n_relinquishes_waitcrt; extern atomic_t fscache_n_relinquishes_retire; +extern atomic_t fscache_n_relinquishes_dropped; -extern atomic_t fscache_n_cookie_index; -extern atomic_t fscache_n_cookie_data; -extern atomic_t fscache_n_cookie_special; - -extern atomic_t fscache_n_object_alloc; -extern atomic_t fscache_n_object_no_alloc; -extern atomic_t fscache_n_object_lookups; -extern atomic_t fscache_n_object_lookups_negative; -extern atomic_t fscache_n_object_lookups_positive; -extern atomic_t fscache_n_object_lookups_timed_out; -extern atomic_t fscache_n_object_created; -extern atomic_t fscache_n_object_avail; -extern atomic_t fscache_n_object_dead; - -extern atomic_t fscache_n_checkaux_none; -extern atomic_t fscache_n_checkaux_okay; -extern atomic_t fscache_n_checkaux_update; -extern atomic_t fscache_n_checkaux_obsolete; - -extern atomic_t fscache_n_cop_alloc_object; -extern atomic_t fscache_n_cop_lookup_object; -extern atomic_t fscache_n_cop_lookup_complete; -extern atomic_t fscache_n_cop_grab_object; -extern atomic_t fscache_n_cop_invalidate_object; -extern atomic_t fscache_n_cop_update_object; -extern atomic_t fscache_n_cop_drop_object; -extern atomic_t fscache_n_cop_put_object; -extern atomic_t fscache_n_cop_sync_cache; -extern atomic_t fscache_n_cop_attr_changed; -extern atomic_t fscache_n_cop_read_or_alloc_page; -extern atomic_t fscache_n_cop_read_or_alloc_pages; -extern atomic_t fscache_n_cop_allocate_page; -extern atomic_t fscache_n_cop_allocate_pages; -extern atomic_t fscache_n_cop_write_page; -extern atomic_t fscache_n_cop_uncache_page; -extern atomic_t fscache_n_cop_dissociate_pages; - -extern atomic_t fscache_n_cache_no_space_reject; -extern atomic_t fscache_n_cache_stale_objects; -extern atomic_t fscache_n_cache_retired_objects; -extern atomic_t fscache_n_cache_culled_objects; +extern atomic_t fscache_n_resizes; +extern atomic_t fscache_n_resizes_null; static inline void fscache_stat(atomic_t *stat) { @@ -278,71 +146,26 @@ int fscache_stats_show(struct seq_file *m, void *v); #endif /* - * raise an event on an object - * - if the event is not masked for that object, then the object is - * queued for attention by the thread pool. - */ -static inline void fscache_raise_event(struct fscache_object *object, - unsigned event) -{ - BUG_ON(event >= NR_FSCACHE_OBJECT_EVENTS); -#if 0 - printk("*** fscache_raise_event(OBJ%d{%lx},%x)\n", - object->debug_id, object->event_mask, (1 << event)); -#endif - if (!test_and_set_bit(event, &object->events) && - test_bit(event, &object->event_mask)) - fscache_enqueue_object(object); -} - -/* - * get an extra reference to a netfs retrieval context + * volume.c */ -static inline -void *fscache_get_context(struct fscache_cookie *cookie, void *context) -{ - if (cookie->def->get_context) - cookie->def->get_context(cookie->netfs_data, context); - return context; -} +extern const struct seq_operations fscache_volumes_seq_ops; -/* - * release a reference to a netfs retrieval context - */ -static inline -void fscache_put_context(struct fscache_cookie *cookie, void *context) -{ - if (cookie->def->put_context) - cookie->def->put_context(cookie->netfs_data, context); -} +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why); +void fscache_create_volume(struct fscache_volume *volume, bool wait); -/* - * Update the auxiliary data on a cookie. - */ -static inline -void fscache_update_aux(struct fscache_cookie *cookie, const void *aux_data) -{ - void *p; - - if (!aux_data) - return; - if (cookie->aux_len <= sizeof(cookie->inline_aux)) - p = cookie->inline_aux; - else - p = cookie->aux; - - if (memcmp(p, aux_data, cookie->aux_len) != 0) { - memcpy(p, aux_data, cookie->aux_len); - set_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags); - } -} /*****************************************************************************/ /* * debug tracing */ #define dbgprintk(FMT, ...) \ - printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) + printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) @@ -395,7 +218,7 @@ do { \ #define FSCACHE_DEBUG_CACHE 0 #define FSCACHE_DEBUG_COOKIE 1 -#define FSCACHE_DEBUG_PAGE 2 +#define FSCACHE_DEBUG_OBJECT 2 #define FSCACHE_DEBUG_OPERATION 3 #define FSCACHE_POINT_ENTER 1 diff --git a/fs/fscache/io.c b/fs/fscache/io.c index 8ecc1141802f..7a769ea57720 100644 --- a/fs/fscache/io.c +++ b/fs/fscache/io.c @@ -4,113 +4,323 @@ * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ - -#define FSCACHE_DEBUG_LEVEL PAGE -#include <linux/module.h> -#define FSCACHE_USE_NEW_IO_API +#define FSCACHE_DEBUG_LEVEL OPERATION #include <linux/fscache-cache.h> +#include <linux/uio.h> +#include <linux/bvec.h> #include <linux/slab.h> -#include <linux/netfs.h> +#include <linux/uio.h> #include "internal.h" -/* - * Start a cache read operation. - * - we return: - * -ENOMEM - out of memory, some pages may be being read - * -ERESTARTSYS - interrupted, some pages may be being read - * -ENOBUFS - no backing object or space available in which to cache any - * pages not being read - * -ENODATA - no data available in the backing object for some or all of - * the pages - * 0 - dispatched a read on all pages +/** + * fscache_wait_for_operation - Wait for an object become accessible + * @cres: The cache resources for the operation being performed + * @want_state: The minimum state the object must be at + * + * See if the target cache object is at the specified minimum state of + * accessibility yet, and if not, wait for it. */ -int __fscache_begin_read_operation(struct netfs_read_request *rreq, - struct fscache_cookie *cookie) +bool fscache_wait_for_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) { - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; + struct fscache_cookie *cookie = fscache_cres_cookie(cres); + enum fscache_cookie_state state; - _enter("rr=%08x", rreq->debug_id); +again: + if (!fscache_cache_is_live(cookie->volume->cache)) { + _leave(" [broken]"); + return false; + } - fscache_stat(&fscache_n_retrievals); + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; + switch (state) { + case FSCACHE_COOKIE_STATE_CREATING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + fallthrough; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != state); + goto again; - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + default: + _leave(" [not live]"); + return false; } - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); +ready: + if (!cres->cache_priv2) + return cookie->volume->cache->ops->begin_operation(cres, want_state); + return true; +} +EXPORT_SYMBOL(fscache_wait_for_operation); + +/* + * Begin an I/O operation on the cache, waiting till we reach the right state. + * + * Attaches the resources required to the operation resources record. + */ +static int fscache_begin_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie, + enum fscache_want_state want_state, + enum fscache_access_trace why) +{ + enum fscache_cookie_state state; + long timeo; + bool once_only = false; - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; + cres->ops = NULL; + cres->cache_priv = cookie; + cres->cache_priv2 = NULL; + cres->debug_id = cookie->debug_id; + cres->inval_counter = cookie->inval_counter; - op = fscache_alloc_retrieval(cookie, NULL, NULL, NULL); - if (!op) - return -ENOMEM; - trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi); + if (!fscache_begin_cookie_access(cookie, why)) + return -ENOBUFS; +again: spin_lock(&cookie->lock); - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); + switch (state) { + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_CREATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", cookie->state); + goto not_live; + default: + goto not_live; + } - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; +ready: spin_unlock(&cookie->lock); + if (!cookie->volume->cache->ops->begin_operation(cres, want_state)) + goto failed; + return 0; - fscache_stat(&fscache_n_retrieval_ops); +wait_for_file_wrangling: + spin_unlock(&cookie->lock); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + atomic_read(&cookie->n_accesses), + fscache_access_io_wait); + timeo = wait_var_event_timeout(&cookie->state, + fscache_cookie_state(cookie) != state, 20 * HZ); + if (timeo <= 1 && !once_only) { + pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u", + __func__, fscache_cookie_state(cookie), state); + fscache_print_cookie(cookie, 'O'); + once_only = true; + } + goto again; - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - ret = object->cache->ops->begin_read_operation(rreq, op); - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: +not_live: spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); +failed: + cres->cache_priv = NULL; + cres->ops = NULL; + fscache_end_cookie_access(cookie, fscache_access_io_not_live); _leave(" = -ENOBUFS"); return -ENOBUFS; } + +int __fscache_begin_read_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_read); +} EXPORT_SYMBOL(__fscache_begin_read_operation); + +int __fscache_begin_write_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_write); +} +EXPORT_SYMBOL(__fscache_begin_write_operation); + +/** + * fscache_set_page_dirty - Mark page dirty and pin a cache object for writeback + * @page: The page being dirtied + * @cookie: The cookie referring to the cache object + * + * Set the dirty flag on a page and pin an in-use cache object in memory when + * dirtying a page so that writeback can later write to it. This is intended + * to be called from the filesystem's ->set_page_dirty() method. + * + * Returns 1 if PG_dirty was set on the page, 0 otherwise. + */ +int fscache_set_page_dirty(struct page *page, struct fscache_cookie *cookie) +{ + struct inode *inode = page->mapping->host; + bool need_use = false; + + _enter(""); + + if (!__set_page_dirty_nobuffers(page)) + return 0; + if (!fscache_cookie_valid(cookie)) + return 1; + + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + spin_lock(&inode->i_lock); + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + inode->i_state |= I_PINNING_FSCACHE_WB; + need_use = true; + } + spin_unlock(&inode->i_lock); + + if (need_use) + fscache_use_cookie(cookie, true); + } + return 1; +} +EXPORT_SYMBOL(fscache_set_page_dirty); + +struct fscache_write_request { + struct netfs_cache_resources cache_resources; + struct address_space *mapping; + loff_t start; + size_t len; + bool set_bits; + netfs_io_terminated_t term_func; + void *term_func_priv; +}; + +void __fscache_clear_page_bits(struct address_space *mapping, + loff_t start, size_t len) +{ + pgoff_t first = start / PAGE_SIZE; + pgoff_t last = (start + len - 1) / PAGE_SIZE; + struct page *page; + + if (len) { + XA_STATE(xas, &mapping->i_pages, first); + + rcu_read_lock(); + xas_for_each(&xas, page, last) { + end_page_fscache(page); + } + rcu_read_unlock(); + } +} +EXPORT_SYMBOL(__fscache_clear_page_bits); + +/* + * Deal with the completion of writing the data to the cache. + */ +static void fscache_wreq_done(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct fscache_write_request *wreq = priv; + + fscache_clear_page_bits(fscache_cres_cookie(&wreq->cache_resources), + wreq->mapping, wreq->start, wreq->len, + wreq->set_bits); + + if (wreq->term_func) + wreq->term_func(wreq->term_func_priv, transferred_or_error, + was_async); + fscache_end_operation(&wreq->cache_resources); + kfree(wreq); +} + +void __fscache_write_to_cache(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, loff_t i_size, + netfs_io_terminated_t term_func, + void *term_func_priv, + bool cond) +{ + struct fscache_write_request *wreq; + struct netfs_cache_resources *cres; + struct iov_iter iter; + int ret = -ENOBUFS; + + if (len == 0) + goto abandon; + + _enter("%llx,%zx", start, len); + + wreq = kzalloc(sizeof(struct fscache_write_request), GFP_NOFS); + if (!wreq) + goto abandon; + wreq->mapping = mapping; + wreq->start = start; + wreq->len = len; + wreq->set_bits = cond; + wreq->term_func = term_func; + wreq->term_func_priv = term_func_priv; + + cres = &wreq->cache_resources; + if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_write) < 0) + goto abandon_free; + + ret = cres->ops->prepare_write(cres, &start, &len, i_size, false); + if (ret < 0) + goto abandon_end; + + /* TODO: Consider clearing page bits now for space the write isn't + * covering. This is more complicated than it appears when THPs are + * taken into account. + */ + + iov_iter_xarray(&iter, WRITE, &mapping->i_pages, start, len); + fscache_write(cres, start, &iter, fscache_wreq_done, wreq); + return; + +abandon_end: + return fscache_wreq_done(wreq, ret, false); +abandon_free: + kfree(wreq); +abandon: + fscache_clear_page_bits(cookie, mapping, start, len, cond); + if (term_func) + term_func(term_func_priv, ret, false); +} +EXPORT_SYMBOL(__fscache_write_to_cache); + +/* + * Change the size of a backing object. + */ +void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +{ + struct netfs_cache_resources cres; + + trace_fscache_resize(cookie, new_size); + if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_resize) == 0) { + fscache_stat(&fscache_n_resizes); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + + /* We cannot defer a resize as we need to do it inside the + * netfs's inode lock so that we're serialised with respect to + * writes. + */ + cookie->volume->cache->ops->resize_cookie(&cres, new_size); + fscache_end_operation(&cres); + } else { + fscache_stat(&fscache_n_resizes_null); + } +} +EXPORT_SYMBOL(__fscache_resize_cookie); diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 4207f98e405f..dad85fd84f6f 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* General filesystem local caching manager * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ #define FSCACHE_DEBUG_LEVEL CACHE #include <linux/module.h> #include <linux/init.h> -#include <linux/sched.h> -#include <linux/completion.h> -#include <linux/slab.h> -#include <linux/seq_file.h> #define CREATE_TRACE_POINTS #include "internal.h" @@ -19,79 +15,18 @@ MODULE_DESCRIPTION("FS Cache Manager"); MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); -unsigned fscache_defer_lookup = 1; -module_param_named(defer_lookup, fscache_defer_lookup, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_defer_lookup, - "Defer cookie lookup to background thread"); - -unsigned fscache_defer_create = 1; -module_param_named(defer_create, fscache_defer_create, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_defer_create, - "Defer cookie creation to background thread"); - unsigned fscache_debug; module_param_named(debug, fscache_debug, uint, S_IWUSR | S_IRUGO); MODULE_PARM_DESC(fscache_debug, "FS-Cache debugging mask"); -struct kobject *fscache_root; -struct workqueue_struct *fscache_object_wq; -struct workqueue_struct *fscache_op_wq; - -DEFINE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); +EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); +EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); +EXPORT_TRACEPOINT_SYMBOL(fscache_access); -/* these values serve as lower bounds, will be adjusted in fscache_init() */ -static unsigned fscache_object_max_active = 4; -static unsigned fscache_op_max_active = 2; - -#ifdef CONFIG_SYSCTL -static struct ctl_table_header *fscache_sysctl_header; - -static int fscache_max_active_sysctl(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - struct workqueue_struct **wqp = table->extra1; - unsigned int *datap = table->data; - int ret; - - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret == 0) - workqueue_set_max_active(*wqp, *datap); - return ret; -} - -static struct ctl_table fscache_sysctls[] = { - { - .procname = "object_max_active", - .data = &fscache_object_max_active, - .maxlen = sizeof(unsigned), - .mode = 0644, - .proc_handler = fscache_max_active_sysctl, - .extra1 = &fscache_object_wq, - }, - { - .procname = "operation_max_active", - .data = &fscache_op_max_active, - .maxlen = sizeof(unsigned), - .mode = 0644, - .proc_handler = fscache_max_active_sysctl, - .extra1 = &fscache_op_wq, - }, - {} -}; - -static struct ctl_table fscache_sysctls_root[] = { - { - .procname = "fscache", - .mode = 0555, - .child = fscache_sysctls, - }, - {} -}; -#endif +struct workqueue_struct *fscache_wq; +EXPORT_SYMBOL(fscache_wq); /* * Mixing scores (in bits) for (7,20): @@ -118,15 +53,16 @@ static inline unsigned int fold_hash(unsigned long x, unsigned long y) /* * Generate a hash. This is derived from full_name_hash(), but we want to be * sure it is arch independent and that it doesn't change as bits of the - * computed hash value might appear on disk. The caller also guarantees that - * the hashed data will be a series of aligned 32-bit words. + * computed hash value might appear on disk. The caller must guarantee that + * the source data is a multiple of four bytes in size. */ -unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n) +unsigned int fscache_hash(unsigned int salt, const void *data, size_t len) { - unsigned int a, x = 0, y = salt; + const __le32 *p = data; + unsigned int a, x = 0, y = salt, n = len / sizeof(__le32); for (; n; n--) { - a = *data++; + a = le32_to_cpu(*p++); HASH_MIX(x, y, a); } return fold_hash(x, y); @@ -137,44 +73,16 @@ unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n) */ static int __init fscache_init(void) { - unsigned int nr_cpus = num_possible_cpus(); - unsigned int cpu; - int ret; - - fscache_object_max_active = - clamp_val(nr_cpus, - fscache_object_max_active, WQ_UNBOUND_MAX_ACTIVE); - - ret = -ENOMEM; - fscache_object_wq = alloc_workqueue("fscache_object", WQ_UNBOUND, - fscache_object_max_active); - if (!fscache_object_wq) - goto error_object_wq; - - fscache_op_max_active = - clamp_val(fscache_object_max_active / 2, - fscache_op_max_active, WQ_UNBOUND_MAX_ACTIVE); + int ret = -ENOMEM; - ret = -ENOMEM; - fscache_op_wq = alloc_workqueue("fscache_operation", WQ_UNBOUND, - fscache_op_max_active); - if (!fscache_op_wq) - goto error_op_wq; - - for_each_possible_cpu(cpu) - init_waitqueue_head(&per_cpu(fscache_object_cong_wait, cpu)); + fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!fscache_wq) + goto error_wq; ret = fscache_proc_init(); if (ret < 0) goto error_proc; -#ifdef CONFIG_SYSCTL - ret = -ENOMEM; - fscache_sysctl_header = register_sysctl_table(fscache_sysctls_root); - if (!fscache_sysctl_header) - goto error_sysctl; -#endif - fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", sizeof(struct fscache_cookie), 0, 0, NULL); @@ -184,26 +92,14 @@ static int __init fscache_init(void) goto error_cookie_jar; } - fscache_root = kobject_create_and_add("fscache", kernel_kobj); - if (!fscache_root) - goto error_kobj; - pr_notice("Loaded\n"); return 0; -error_kobj: - kmem_cache_destroy(fscache_cookie_jar); error_cookie_jar: -#ifdef CONFIG_SYSCTL - unregister_sysctl_table(fscache_sysctl_header); -error_sysctl: -#endif fscache_proc_cleanup(); error_proc: - destroy_workqueue(fscache_op_wq); -error_op_wq: - destroy_workqueue(fscache_object_wq); -error_object_wq: + destroy_workqueue(fscache_wq); +error_wq: return ret; } @@ -216,14 +112,9 @@ static void __exit fscache_exit(void) { _enter(""); - kobject_put(fscache_root); kmem_cache_destroy(fscache_cookie_jar); -#ifdef CONFIG_SYSCTL - unregister_sysctl_table(fscache_sysctl_header); -#endif fscache_proc_cleanup(); - destroy_workqueue(fscache_op_wq); - destroy_workqueue(fscache_object_wq); + destroy_workqueue(fscache_wq); pr_notice("Unloaded\n"); } diff --git a/fs/fscache/netfs.c b/fs/fscache/netfs.c deleted file mode 100644 index d6bdb7b5e723..000000000000 --- a/fs/fscache/netfs.c +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache netfs (client) registration - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include <linux/module.h> -#include <linux/slab.h> -#include "internal.h" - -/* - * register a network filesystem for caching - */ -int __fscache_register_netfs(struct fscache_netfs *netfs) -{ - struct fscache_cookie *candidate, *cookie; - - _enter("{%s}", netfs->name); - - /* allocate a cookie for the primary index */ - candidate = fscache_alloc_cookie(&fscache_fsdef_index, - &fscache_fsdef_netfs_def, - netfs->name, strlen(netfs->name), - &netfs->version, sizeof(netfs->version), - netfs, 0); - if (!candidate) { - _leave(" = -ENOMEM"); - return -ENOMEM; - } - - candidate->flags = 1 << FSCACHE_COOKIE_ENABLED; - - /* check the netfs type is not already present */ - cookie = fscache_hash_cookie(candidate); - if (!cookie) - goto already_registered; - if (cookie != candidate) { - trace_fscache_cookie(candidate->debug_id, 1, fscache_cookie_discard); - fscache_free_cookie(candidate); - } - - fscache_cookie_get(cookie->parent, fscache_cookie_get_register_netfs); - atomic_inc(&cookie->parent->n_children); - - netfs->primary_index = cookie; - - pr_notice("Netfs '%s' registered for caching\n", netfs->name); - trace_fscache_netfs(netfs); - _leave(" = 0"); - return 0; - -already_registered: - fscache_cookie_put(candidate, fscache_cookie_put_dup_netfs); - _leave(" = -EEXIST"); - return -EEXIST; -} -EXPORT_SYMBOL(__fscache_register_netfs); - -/* - * unregister a network filesystem from the cache - * - all cookies must have been released first - */ -void __fscache_unregister_netfs(struct fscache_netfs *netfs) -{ - _enter("{%s.%u}", netfs->name, netfs->version); - - fscache_relinquish_cookie(netfs->primary_index, NULL, false); - pr_notice("Netfs '%s' unregistered from caching\n", netfs->name); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_unregister_netfs); diff --git a/fs/fscache/object.c b/fs/fscache/object.c deleted file mode 100644 index 6a675652129b..000000000000 --- a/fs/fscache/object.c +++ /dev/null @@ -1,1125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache object state machine handler - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/object.rst for a description of the - * object state machine and the in-kernel representations. - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/prefetch.h> -#include "internal.h" - -static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *, int); -static const struct fscache_state *fscache_kill_dependents(struct fscache_object *, int); -static const struct fscache_state *fscache_drop_object(struct fscache_object *, int); -static const struct fscache_state *fscache_initialise_object(struct fscache_object *, int); -static const struct fscache_state *fscache_invalidate_object(struct fscache_object *, int); -static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *, int); -static const struct fscache_state *fscache_kill_object(struct fscache_object *, int); -static const struct fscache_state *fscache_lookup_failure(struct fscache_object *, int); -static const struct fscache_state *fscache_look_up_object(struct fscache_object *, int); -static const struct fscache_state *fscache_object_available(struct fscache_object *, int); -static const struct fscache_state *fscache_parent_ready(struct fscache_object *, int); -static const struct fscache_state *fscache_update_object(struct fscache_object *, int); -static const struct fscache_state *fscache_object_dead(struct fscache_object *, int); - -#define __STATE_NAME(n) fscache_osm_##n -#define STATE(n) (&__STATE_NAME(n)) - -/* - * Define a work state. Work states are execution states. No event processing - * is performed by them. The function attached to a work state returns a - * pointer indicating the next state to which the state machine should - * transition. Returning NO_TRANSIT repeats the current state, but goes back - * to the scheduler first. - */ -#define WORK_STATE(n, sn, f) \ - const struct fscache_state __STATE_NAME(n) = { \ - .name = #n, \ - .short_name = sn, \ - .work = f \ - } - -/* - * Returns from work states. - */ -#define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); }) - -#define NO_TRANSIT ((struct fscache_state *)NULL) - -/* - * Define a wait state. Wait states are event processing states. No execution - * is performed by them. Wait states are just tables of "if event X occurs, - * clear it and transition to state Y". The dispatcher returns to the - * scheduler if none of the events in which the wait state has an interest are - * currently pending. - */ -#define WAIT_STATE(n, sn, ...) \ - const struct fscache_state __STATE_NAME(n) = { \ - .name = #n, \ - .short_name = sn, \ - .work = NULL, \ - .transitions = { __VA_ARGS__, { 0, NULL } } \ - } - -#define TRANSIT_TO(state, emask) \ - { .events = (emask), .transit_to = STATE(state) } - -/* - * The object state machine. - */ -static WORK_STATE(INIT_OBJECT, "INIT", fscache_initialise_object); -static WORK_STATE(PARENT_READY, "PRDY", fscache_parent_ready); -static WORK_STATE(ABORT_INIT, "ABRT", fscache_abort_initialisation); -static WORK_STATE(LOOK_UP_OBJECT, "LOOK", fscache_look_up_object); -static WORK_STATE(OBJECT_AVAILABLE, "AVBL", fscache_object_available); -static WORK_STATE(JUMPSTART_DEPS, "JUMP", fscache_jumpstart_dependents); - -static WORK_STATE(INVALIDATE_OBJECT, "INVL", fscache_invalidate_object); -static WORK_STATE(UPDATE_OBJECT, "UPDT", fscache_update_object); - -static WORK_STATE(LOOKUP_FAILURE, "LCFL", fscache_lookup_failure); -static WORK_STATE(KILL_OBJECT, "KILL", fscache_kill_object); -static WORK_STATE(KILL_DEPENDENTS, "KDEP", fscache_kill_dependents); -static WORK_STATE(DROP_OBJECT, "DROP", fscache_drop_object); -static WORK_STATE(OBJECT_DEAD, "DEAD", fscache_object_dead); - -static WAIT_STATE(WAIT_FOR_INIT, "?INI", - TRANSIT_TO(INIT_OBJECT, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); - -static WAIT_STATE(WAIT_FOR_PARENT, "?PRN", - TRANSIT_TO(PARENT_READY, 1 << FSCACHE_OBJECT_EV_PARENT_READY)); - -static WAIT_STATE(WAIT_FOR_CMD, "?CMD", - TRANSIT_TO(INVALIDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_INVALIDATE), - TRANSIT_TO(UPDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_UPDATE), - TRANSIT_TO(JUMPSTART_DEPS, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); - -static WAIT_STATE(WAIT_FOR_CLEARANCE, "?CLR", - TRANSIT_TO(KILL_OBJECT, 1 << FSCACHE_OBJECT_EV_CLEARED)); - -/* - * Out-of-band event transition tables. These are for handling unexpected - * events, such as an I/O error. If an OOB event occurs, the state machine - * clears and disables the event and forces a transition to the nominated work - * state (acurrently executing work states will complete first). - * - * In such a situation, object->state remembers the state the machine should - * have been in/gone to and returning NO_TRANSIT returns to that. - */ -static const struct fscache_transition fscache_osm_init_oob[] = { - TRANSIT_TO(ABORT_INIT, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static const struct fscache_transition fscache_osm_lookup_oob[] = { - TRANSIT_TO(LOOKUP_FAILURE, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static const struct fscache_transition fscache_osm_run_oob[] = { - TRANSIT_TO(KILL_OBJECT, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static int fscache_get_object(struct fscache_object *, - enum fscache_obj_ref_trace); -static void fscache_put_object(struct fscache_object *, - enum fscache_obj_ref_trace); -static bool fscache_enqueue_dependents(struct fscache_object *, int); -static void fscache_dequeue_object(struct fscache_object *); -static void fscache_update_aux_data(struct fscache_object *); - -/* - * we need to notify the parent when an op completes that we had outstanding - * upon it - */ -static inline void fscache_done_parent_op(struct fscache_object *object) -{ - struct fscache_object *parent = object->parent; - - _enter("OBJ%x {OBJ%x,%x}", - object->debug_id, parent->debug_id, parent->n_ops); - - spin_lock_nested(&parent->lock, 1); - parent->n_obj_ops--; - parent->n_ops--; - if (parent->n_ops == 0) - fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); - spin_unlock(&parent->lock); -} - -/* - * Object state machine dispatcher. - */ -static void fscache_object_sm_dispatcher(struct fscache_object *object) -{ - const struct fscache_transition *t; - const struct fscache_state *state, *new_state; - unsigned long events, event_mask; - bool oob; - int event = -1; - - ASSERT(object != NULL); - - _enter("{OBJ%x,%s,%lx}", - object->debug_id, object->state->name, object->events); - - event_mask = object->event_mask; -restart: - object->event_mask = 0; /* Mask normal event handling */ - state = object->state; -restart_masked: - events = object->events; - - /* Handle any out-of-band events (typically an error) */ - if (events & object->oob_event_mask) { - _debug("{OBJ%x} oob %lx", - object->debug_id, events & object->oob_event_mask); - oob = true; - for (t = object->oob_table; t->events; t++) { - if (events & t->events) { - state = t->transit_to; - ASSERT(state->work != NULL); - event = fls(events & t->events) - 1; - __clear_bit(event, &object->oob_event_mask); - clear_bit(event, &object->events); - goto execute_work_state; - } - } - } - oob = false; - - /* Wait states are just transition tables */ - if (!state->work) { - if (events & event_mask) { - for (t = state->transitions; t->events; t++) { - if (events & t->events) { - new_state = t->transit_to; - event = fls(events & t->events) - 1; - trace_fscache_osm(object, state, - true, false, event); - clear_bit(event, &object->events); - _debug("{OBJ%x} ev %d: %s -> %s", - object->debug_id, event, - state->name, new_state->name); - object->state = state = new_state; - goto execute_work_state; - } - } - - /* The event mask didn't include all the tabled bits */ - BUG(); - } - /* Randomly woke up */ - goto unmask_events; - } - -execute_work_state: - _debug("{OBJ%x} exec %s", object->debug_id, state->name); - - trace_fscache_osm(object, state, false, oob, event); - new_state = state->work(object, event); - event = -1; - if (new_state == NO_TRANSIT) { - _debug("{OBJ%x} %s notrans", object->debug_id, state->name); - if (unlikely(state == STATE(OBJECT_DEAD))) { - _leave(" [dead]"); - return; - } - fscache_enqueue_object(object); - event_mask = object->oob_event_mask; - goto unmask_events; - } - - _debug("{OBJ%x} %s -> %s", - object->debug_id, state->name, new_state->name); - object->state = state = new_state; - - if (state->work) { - if (unlikely(state == STATE(OBJECT_DEAD))) { - _leave(" [dead]"); - return; - } - goto restart_masked; - } - - /* Transited to wait state */ - event_mask = object->oob_event_mask; - for (t = state->transitions; t->events; t++) - event_mask |= t->events; - -unmask_events: - object->event_mask = event_mask; - smp_mb(); - events = object->events; - if (events & event_mask) - goto restart; - _leave(" [msk %lx]", event_mask); -} - -/* - * execute an object - */ -static void fscache_object_work_func(struct work_struct *work) -{ - struct fscache_object *object = - container_of(work, struct fscache_object, work); - - _enter("{OBJ%x}", object->debug_id); - - fscache_object_sm_dispatcher(object); - fscache_put_object(object, fscache_obj_put_work); -} - -/** - * fscache_object_init - Initialise a cache object description - * @object: Object description - * @cookie: Cookie object will be attached to - * @cache: Cache in which backing object will be found - * - * Initialise a cache object description to its basic values. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_object_init(struct fscache_object *object, - struct fscache_cookie *cookie, - struct fscache_cache *cache) -{ - const struct fscache_transition *t; - - atomic_inc(&cache->object_count); - - object->state = STATE(WAIT_FOR_INIT); - object->oob_table = fscache_osm_init_oob; - object->flags = 1 << FSCACHE_OBJECT_IS_LIVE; - spin_lock_init(&object->lock); - INIT_LIST_HEAD(&object->cache_link); - INIT_HLIST_NODE(&object->cookie_link); - INIT_WORK(&object->work, fscache_object_work_func); - INIT_LIST_HEAD(&object->dependents); - INIT_LIST_HEAD(&object->dep_link); - INIT_LIST_HEAD(&object->pending_ops); - object->n_children = 0; - object->n_ops = object->n_in_progress = object->n_exclusive = 0; - object->events = 0; - object->store_limit = 0; - object->store_limit_l = 0; - object->cache = cache; - object->cookie = cookie; - fscache_cookie_get(cookie, fscache_cookie_get_attach_object); - object->parent = NULL; -#ifdef CONFIG_FSCACHE_OBJECT_LIST - RB_CLEAR_NODE(&object->objlist_link); -#endif - - object->oob_event_mask = 0; - for (t = object->oob_table; t->events; t++) - object->oob_event_mask |= t->events; - object->event_mask = object->oob_event_mask; - for (t = object->state->transitions; t->events; t++) - object->event_mask |= t->events; -} -EXPORT_SYMBOL(fscache_object_init); - -/* - * Mark the object as no longer being live, making sure that we synchronise - * against op submission. - */ -static inline void fscache_mark_object_dead(struct fscache_object *object) -{ - spin_lock(&object->lock); - clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags); - spin_unlock(&object->lock); -} - -/* - * Abort object initialisation before we start it. - */ -static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_event_mask = 0; - fscache_dequeue_object(object); - return transit_to(KILL_OBJECT); -} - -/* - * initialise an object - * - check the specified object's parent to see if we can make use of it - * immediately to do a creation - * - we may need to start the process of creating a parent and we need to wait - * for the parent's lookup and creation to complete if it's not there yet - */ -static const struct fscache_state *fscache_initialise_object(struct fscache_object *object, - int event) -{ - struct fscache_object *parent; - bool success; - - _enter("{OBJ%x},%d", object->debug_id, event); - - ASSERT(list_empty(&object->dep_link)); - - parent = object->parent; - if (!parent) { - _leave(" [no parent]"); - return transit_to(DROP_OBJECT); - } - - _debug("parent: %s of:%lx", parent->state->name, parent->flags); - - if (fscache_object_is_dying(parent)) { - _leave(" [bad parent]"); - return transit_to(DROP_OBJECT); - } - - if (fscache_object_is_available(parent)) { - _leave(" [ready]"); - return transit_to(PARENT_READY); - } - - _debug("wait"); - - spin_lock(&parent->lock); - fscache_stat(&fscache_n_cop_grab_object); - success = false; - if (fscache_object_is_live(parent) && - object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) { - list_add(&object->dep_link, &parent->dependents); - success = true; - } - fscache_stat_d(&fscache_n_cop_grab_object); - spin_unlock(&parent->lock); - if (!success) { - _leave(" [grab failed]"); - return transit_to(DROP_OBJECT); - } - - /* fscache_acquire_non_index_cookie() uses this - * to wake the chain up */ - fscache_raise_event(parent, FSCACHE_OBJECT_EV_NEW_CHILD); - _leave(" [wait]"); - return transit_to(WAIT_FOR_PARENT); -} - -/* - * Once the parent object is ready, we should kick off our lookup op. - */ -static const struct fscache_state *fscache_parent_ready(struct fscache_object *object, - int event) -{ - struct fscache_object *parent = object->parent; - - _enter("{OBJ%x},%d", object->debug_id, event); - - ASSERT(parent != NULL); - - spin_lock(&parent->lock); - parent->n_ops++; - parent->n_obj_ops++; - spin_unlock(&parent->lock); - - _leave(""); - return transit_to(LOOK_UP_OBJECT); -} - -/* - * look an object up in the cache from which it was allocated - * - we hold an "access lock" on the parent object, so the parent object cannot - * be withdrawn by either party till we've finished - */ -static const struct fscache_state *fscache_look_up_object(struct fscache_object *object, - int event) -{ - struct fscache_cookie *cookie = object->cookie; - struct fscache_object *parent = object->parent; - int ret; - - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_table = fscache_osm_lookup_oob; - - ASSERT(parent != NULL); - ASSERTCMP(parent->n_ops, >, 0); - ASSERTCMP(parent->n_obj_ops, >, 0); - - /* make sure the parent is still available */ - ASSERT(fscache_object_is_available(parent)); - - if (fscache_object_is_dying(parent) || - test_bit(FSCACHE_IOERROR, &object->cache->flags) || - !fscache_use_cookie(object)) { - _leave(" [unavailable]"); - return transit_to(LOOKUP_FAILURE); - } - - _debug("LOOKUP \"%s\" in \"%s\"", - cookie->def->name, object->cache->tag->name); - - fscache_stat(&fscache_n_object_lookups); - fscache_stat(&fscache_n_cop_lookup_object); - ret = object->cache->ops->lookup_object(object); - fscache_stat_d(&fscache_n_cop_lookup_object); - - fscache_unuse_cookie(object); - - if (ret == -ETIMEDOUT) { - /* probably stuck behind another object, so move this one to - * the back of the queue */ - fscache_stat(&fscache_n_object_lookups_timed_out); - _leave(" [timeout]"); - return NO_TRANSIT; - } - - if (ret < 0) { - _leave(" [error]"); - return transit_to(LOOKUP_FAILURE); - } - - _leave(" [ok]"); - return transit_to(OBJECT_AVAILABLE); -} - -/** - * fscache_object_lookup_negative - Note negative cookie lookup - * @object: Object pointing to cookie to mark - * - * Note negative lookup, permitting those waiting to read data from an already - * existing backing object to continue as there's no data for them to read. - */ -void fscache_object_lookup_negative(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x,%s}", object->debug_id, object->state->name); - - if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { - fscache_stat(&fscache_n_object_lookups_negative); - - /* Allow write requests to begin stacking up and read requests to begin - * returning ENODATA. - */ - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - } - _leave(""); -} -EXPORT_SYMBOL(fscache_object_lookup_negative); - -/** - * fscache_obtained_object - Note successful object lookup or creation - * @object: Object pointing to cookie to mark - * - * Note successful lookup and/or creation, permitting those waiting to write - * data to a backing object to continue. - * - * Note that after calling this, an object's cookie may be relinquished by the - * netfs, and so must be accessed with object lock held. - */ -void fscache_obtained_object(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x,%s}", object->debug_id, object->state->name); - - /* if we were still looking up, then we must have a positive lookup - * result, in which case there may be data available */ - if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { - fscache_stat(&fscache_n_object_lookups_positive); - - /* We do (presumably) have data */ - clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - /* Allow write requests to begin stacking up and read requests - * to begin shovelling data. - */ - clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - } else { - fscache_stat(&fscache_n_object_created); - } - - set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags); - _leave(""); -} -EXPORT_SYMBOL(fscache_obtained_object); - -/* - * handle an object that has just become available - */ -static const struct fscache_state *fscache_object_available(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_table = fscache_osm_run_oob; - - spin_lock(&object->lock); - - fscache_done_parent_op(object); - if (object->n_in_progress == 0) { - if (object->n_ops > 0) { - ASSERTCMP(object->n_ops, >=, object->n_obj_ops); - fscache_start_operations(object); - } else { - ASSERT(list_empty(&object->pending_ops)); - } - } - spin_unlock(&object->lock); - - fscache_stat(&fscache_n_cop_lookup_complete); - object->cache->ops->lookup_complete(object); - fscache_stat_d(&fscache_n_cop_lookup_complete); - - fscache_stat(&fscache_n_object_avail); - - _leave(""); - return transit_to(JUMPSTART_DEPS); -} - -/* - * Wake up this object's dependent objects now that we've become available. - */ -static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_PARENT_READY)) - return NO_TRANSIT; /* Not finished; requeue */ - return transit_to(WAIT_FOR_CMD); -} - -/* - * Handle lookup or creation failute. - */ -static const struct fscache_state *fscache_lookup_failure(struct fscache_object *object, - int event) -{ - struct fscache_cookie *cookie; - - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_event_mask = 0; - - fscache_stat(&fscache_n_cop_lookup_complete); - object->cache->ops->lookup_complete(object); - fscache_stat_d(&fscache_n_cop_lookup_complete); - - set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags); - - cookie = object->cookie; - set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - - fscache_done_parent_op(object); - return transit_to(KILL_OBJECT); -} - -/* - * Wait for completion of all active operations on this object and the death of - * all child objects of this object. - */ -static const struct fscache_state *fscache_kill_object(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x,%d,%d},%d", - object->debug_id, object->n_ops, object->n_children, event); - - fscache_mark_object_dead(object); - object->oob_event_mask = 0; - - if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) { - /* Reject any new read/write ops and abort any that are pending. */ - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_cancel_all_ops(object); - } - - if (list_empty(&object->dependents) && - object->n_ops == 0 && - object->n_children == 0) - return transit_to(DROP_OBJECT); - - if (object->n_in_progress == 0) { - spin_lock(&object->lock); - if (object->n_ops > 0 && object->n_in_progress == 0) - fscache_start_operations(object); - spin_unlock(&object->lock); - } - - if (!list_empty(&object->dependents)) - return transit_to(KILL_DEPENDENTS); - - return transit_to(WAIT_FOR_CLEARANCE); -} - -/* - * Kill dependent objects. - */ -static const struct fscache_state *fscache_kill_dependents(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_KILL)) - return NO_TRANSIT; /* Not finished */ - return transit_to(WAIT_FOR_CLEARANCE); -} - -/* - * Drop an object's attachments - */ -static const struct fscache_state *fscache_drop_object(struct fscache_object *object, - int event) -{ - struct fscache_object *parent = object->parent; - struct fscache_cookie *cookie = object->cookie; - struct fscache_cache *cache = object->cache; - bool awaken = false; - - _enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event); - - ASSERT(cookie != NULL); - ASSERT(!hlist_unhashed(&object->cookie_link)); - - if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) { - _debug("final update"); - fscache_update_aux_data(object); - } - - /* Make sure the cookie no longer points here and that the netfs isn't - * waiting for us. - */ - spin_lock(&cookie->lock); - hlist_del_init(&object->cookie_link); - if (hlist_empty(&cookie->backing_objects) && - test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - awaken = true; - spin_unlock(&cookie->lock); - - if (awaken) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - - - /* Prevent a race with our last child, which has to signal EV_CLEARED - * before dropping our spinlock. - */ - spin_lock(&object->lock); - spin_unlock(&object->lock); - - /* Discard from the cache's collection of objects */ - spin_lock(&cache->object_list_lock); - list_del_init(&object->cache_link); - spin_unlock(&cache->object_list_lock); - - fscache_stat(&fscache_n_cop_drop_object); - cache->ops->drop_object(object); - fscache_stat_d(&fscache_n_cop_drop_object); - - /* The parent object wants to know when all it dependents have gone */ - if (parent) { - _debug("release parent OBJ%x {%d}", - parent->debug_id, parent->n_children); - - spin_lock(&parent->lock); - parent->n_children--; - if (parent->n_children == 0) - fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); - spin_unlock(&parent->lock); - object->parent = NULL; - } - - /* this just shifts the object release to the work processor */ - fscache_put_object(object, fscache_obj_put_drop_obj); - fscache_stat(&fscache_n_object_dead); - - _leave(""); - return transit_to(OBJECT_DEAD); -} - -/* - * get a ref on an object - */ -static int fscache_get_object(struct fscache_object *object, - enum fscache_obj_ref_trace why) -{ - int ret; - - fscache_stat(&fscache_n_cop_grab_object); - ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN; - fscache_stat_d(&fscache_n_cop_grab_object); - return ret; -} - -/* - * Discard a ref on an object - */ -static void fscache_put_object(struct fscache_object *object, - enum fscache_obj_ref_trace why) -{ - fscache_stat(&fscache_n_cop_put_object); - object->cache->ops->put_object(object, why); - fscache_stat_d(&fscache_n_cop_put_object); -} - -/** - * fscache_object_destroy - Note that a cache object is about to be destroyed - * @object: The object to be destroyed - * - * Note the imminent destruction and deallocation of a cache object record. - */ -void fscache_object_destroy(struct fscache_object *object) -{ - /* We can get rid of the cookie now */ - fscache_cookie_put(object->cookie, fscache_cookie_put_object); - object->cookie = NULL; -} -EXPORT_SYMBOL(fscache_object_destroy); - -/* - * enqueue an object for metadata-type processing - */ -void fscache_enqueue_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - if (fscache_get_object(object, fscache_obj_get_queue) >= 0) { - wait_queue_head_t *cong_wq = - &get_cpu_var(fscache_object_cong_wait); - - if (queue_work(fscache_object_wq, &object->work)) { - if (fscache_object_congested()) - wake_up(cong_wq); - } else - fscache_put_object(object, fscache_obj_put_queue); - - put_cpu_var(fscache_object_cong_wait); - } -} - -/** - * fscache_object_sleep_till_congested - Sleep until object wq is congested - * @timeoutp: Scheduler sleep timeout - * - * Allow an object handler to sleep until the object workqueue is congested. - * - * The caller must set up a wake up event before calling this and must have set - * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own - * condition before calling this function as no test is made here. - * - * %true is returned if the object wq is congested, %false otherwise. - */ -bool fscache_object_sleep_till_congested(signed long *timeoutp) -{ - wait_queue_head_t *cong_wq = this_cpu_ptr(&fscache_object_cong_wait); - DEFINE_WAIT(wait); - - if (fscache_object_congested()) - return true; - - add_wait_queue_exclusive(cong_wq, &wait); - if (!fscache_object_congested()) - *timeoutp = schedule_timeout(*timeoutp); - finish_wait(cong_wq, &wait); - - return fscache_object_congested(); -} -EXPORT_SYMBOL_GPL(fscache_object_sleep_till_congested); - -/* - * Enqueue the dependents of an object for metadata-type processing. - * - * If we don't manage to finish the list before the scheduler wants to run - * again then return false immediately. We return true if the list was - * cleared. - */ -static bool fscache_enqueue_dependents(struct fscache_object *object, int event) -{ - struct fscache_object *dep; - bool ret = true; - - _enter("{OBJ%x}", object->debug_id); - - if (list_empty(&object->dependents)) - return true; - - spin_lock(&object->lock); - - while (!list_empty(&object->dependents)) { - dep = list_entry(object->dependents.next, - struct fscache_object, dep_link); - list_del_init(&dep->dep_link); - - fscache_raise_event(dep, event); - fscache_put_object(dep, fscache_obj_put_enq_dep); - - if (!list_empty(&object->dependents) && need_resched()) { - ret = false; - break; - } - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * remove an object from whatever queue it's waiting on - */ -static void fscache_dequeue_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - if (!list_empty(&object->dep_link)) { - spin_lock(&object->parent->lock); - list_del_init(&object->dep_link); - spin_unlock(&object->parent->lock); - } - - _leave(""); -} - -/** - * fscache_check_aux - Ask the netfs whether an object on disk is still valid - * @object: The object to ask about - * @data: The auxiliary data for the object - * @datalen: The size of the auxiliary data - * @object_size: The size of the object according to the server. - * - * This function consults the netfs about the coherency state of an object. - * The caller must be holding a ref on cookie->n_active (held by - * fscache_look_up_object() on behalf of the cache backend during object lookup - * and creation). - */ -enum fscache_checkaux fscache_check_aux(struct fscache_object *object, - const void *data, uint16_t datalen, - loff_t object_size) -{ - enum fscache_checkaux result; - - if (!object->cookie->def->check_aux) { - fscache_stat(&fscache_n_checkaux_none); - return FSCACHE_CHECKAUX_OKAY; - } - - result = object->cookie->def->check_aux(object->cookie->netfs_data, - data, datalen, object_size); - switch (result) { - /* entry okay as is */ - case FSCACHE_CHECKAUX_OKAY: - fscache_stat(&fscache_n_checkaux_okay); - break; - - /* entry requires update */ - case FSCACHE_CHECKAUX_NEEDS_UPDATE: - fscache_stat(&fscache_n_checkaux_update); - break; - - /* entry requires deletion */ - case FSCACHE_CHECKAUX_OBSOLETE: - fscache_stat(&fscache_n_checkaux_obsolete); - break; - - default: - BUG(); - } - - return result; -} -EXPORT_SYMBOL(fscache_check_aux); - -/* - * Asynchronously invalidate an object. - */ -static const struct fscache_state *_fscache_invalidate_object(struct fscache_object *object, - int event) -{ - struct fscache_operation *op; - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x},%d", object->debug_id, event); - - /* We're going to need the cookie. If the cookie is not available then - * retire the object instead. - */ - if (!fscache_use_cookie(object)) { - ASSERT(radix_tree_empty(&object->cookie->stores)); - set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); - _leave(" [no cookie]"); - return transit_to(KILL_OBJECT); - } - - /* Reject any new read/write ops and abort any that are pending. */ - fscache_invalidate_writes(cookie); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_cancel_all_ops(object); - - /* Now we have to wait for in-progress reads and writes */ - op = kzalloc(sizeof(*op), GFP_KERNEL); - if (!op) - goto nomem; - - fscache_operation_init(cookie, op, object->cache->ops->invalidate_object, - NULL, NULL); - op->flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_EXCLUSIVE) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_invalidate); - - spin_lock(&cookie->lock); - if (fscache_submit_exclusive_op(object, op) < 0) - goto submit_op_failed; - spin_unlock(&cookie->lock); - fscache_put_operation(op); - - /* Once we've completed the invalidation, we know there will be no data - * stored in the cache and thus we can reinstate the data-check-skip - * optimisation. - */ - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - - /* We can allow read and write requests to come in once again. They'll - * queue up behind our exclusive invalidation operation. - */ - if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - _leave(" [ok]"); - return transit_to(UPDATE_OBJECT); - -nomem: - fscache_mark_object_dead(object); - fscache_unuse_cookie(object); - _leave(" [ENOMEM]"); - return transit_to(KILL_OBJECT); - -submit_op_failed: - fscache_mark_object_dead(object); - spin_unlock(&cookie->lock); - fscache_unuse_cookie(object); - kfree(op); - _leave(" [EIO]"); - return transit_to(KILL_OBJECT); -} - -static const struct fscache_state *fscache_invalidate_object(struct fscache_object *object, - int event) -{ - const struct fscache_state *s; - - fscache_stat(&fscache_n_invalidates_run); - fscache_stat(&fscache_n_cop_invalidate_object); - s = _fscache_invalidate_object(object, event); - fscache_stat_d(&fscache_n_cop_invalidate_object); - return s; -} - -/* - * Update auxiliary data. - */ -static void fscache_update_aux_data(struct fscache_object *object) -{ - fscache_stat(&fscache_n_updates_run); - fscache_stat(&fscache_n_cop_update_object); - object->cache->ops->update_object(object); - fscache_stat_d(&fscache_n_cop_update_object); -} - -/* - * Asynchronously update an object. - */ -static const struct fscache_state *fscache_update_object(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - fscache_update_aux_data(object); - - _leave(""); - return transit_to(WAIT_FOR_CMD); -} - -/** - * fscache_object_retrying_stale - Note retrying stale object - * @object: The object that will be retried - * - * Note that an object lookup found an on-disk object that was adjudged to be - * stale and has been deleted. The lookup will be retried. - */ -void fscache_object_retrying_stale(struct fscache_object *object) -{ - fscache_stat(&fscache_n_cache_no_space_reject); -} -EXPORT_SYMBOL(fscache_object_retrying_stale); - -/** - * fscache_object_mark_killed - Note that an object was killed - * @object: The object that was culled - * @why: The reason the object was killed. - * - * Note that an object was killed. Returns true if the object was - * already marked killed, false if it wasn't. - */ -void fscache_object_mark_killed(struct fscache_object *object, - enum fscache_why_object_killed why) -{ - if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) { - pr_err("Error: Object already killed by cache [%s]\n", - object->cache->identifier); - return; - } - - switch (why) { - case FSCACHE_OBJECT_NO_SPACE: - fscache_stat(&fscache_n_cache_no_space_reject); - break; - case FSCACHE_OBJECT_IS_STALE: - fscache_stat(&fscache_n_cache_stale_objects); - break; - case FSCACHE_OBJECT_WAS_RETIRED: - fscache_stat(&fscache_n_cache_retired_objects); - break; - case FSCACHE_OBJECT_WAS_CULLED: - fscache_stat(&fscache_n_cache_culled_objects); - break; - } -} -EXPORT_SYMBOL(fscache_object_mark_killed); - -/* - * The object is dead. We can get here if an object gets queued by an event - * that would lead to its death (such as EV_KILL) when the dispatcher is - * already running (and so can be requeued) but hasn't yet cleared the event - * mask. - */ -static const struct fscache_state *fscache_object_dead(struct fscache_object *object, - int event) -{ - if (!test_and_set_bit(FSCACHE_OBJECT_RUN_AFTER_DEAD, - &object->flags)) - return NO_TRANSIT; - - WARN(true, "FS-Cache object redispatched after death"); - return NO_TRANSIT; -} diff --git a/fs/fscache/operation.c b/fs/fscache/operation.c deleted file mode 100644 index e002cdfaf3cc..000000000000 --- a/fs/fscache/operation.c +++ /dev/null @@ -1,633 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache worker operation management routines - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/operations.rst - */ - -#define FSCACHE_DEBUG_LEVEL OPERATION -#include <linux/module.h> -#include <linux/seq_file.h> -#include <linux/slab.h> -#include "internal.h" - -atomic_t fscache_op_debug_id; -EXPORT_SYMBOL(fscache_op_debug_id); - -static void fscache_operation_dummy_cancel(struct fscache_operation *op) -{ -} - -/** - * fscache_operation_init - Do basic initialisation of an operation - * @cookie: The cookie to operate on - * @op: The operation to initialise - * @processor: The function to perform the operation - * @cancel: A function to handle operation cancellation - * @release: The release function to assign - * - * Do basic initialisation of an operation. The caller must still set flags, - * object and processor if needed. - */ -void fscache_operation_init(struct fscache_cookie *cookie, - struct fscache_operation *op, - fscache_operation_processor_t processor, - fscache_operation_cancel_t cancel, - fscache_operation_release_t release) -{ - INIT_WORK(&op->work, fscache_op_work_func); - atomic_set(&op->usage, 1); - op->state = FSCACHE_OP_ST_INITIALISED; - op->debug_id = atomic_inc_return(&fscache_op_debug_id); - op->processor = processor; - op->cancel = cancel ?: fscache_operation_dummy_cancel; - op->release = release; - INIT_LIST_HEAD(&op->pend_link); - fscache_stat(&fscache_n_op_initialised); - trace_fscache_op(cookie, op, fscache_op_init); -} -EXPORT_SYMBOL(fscache_operation_init); - -/** - * fscache_enqueue_operation - Enqueue an operation for processing - * @op: The operation to enqueue - * - * Enqueue an operation for processing by the FS-Cache thread pool. - * - * This will get its own ref on the object. - */ -void fscache_enqueue_operation(struct fscache_operation *op) -{ - struct fscache_cookie *cookie = op->object->cookie; - - _enter("{OBJ%x OP%x,%u}", - op->object->debug_id, op->debug_id, atomic_read(&op->usage)); - - ASSERT(list_empty(&op->pend_link)); - ASSERT(op->processor != NULL); - ASSERT(fscache_object_is_available(op->object)); - ASSERTCMP(atomic_read(&op->usage), >, 0); - ASSERTIFCMP(op->state != FSCACHE_OP_ST_IN_PROGRESS, - op->state, ==, FSCACHE_OP_ST_CANCELLED); - - fscache_stat(&fscache_n_op_enqueue); - switch (op->flags & FSCACHE_OP_TYPE) { - case FSCACHE_OP_ASYNC: - trace_fscache_op(cookie, op, fscache_op_enqueue_async); - _debug("queue async"); - atomic_inc(&op->usage); - if (!queue_work(fscache_op_wq, &op->work)) - fscache_put_operation(op); - break; - case FSCACHE_OP_MYTHREAD: - trace_fscache_op(cookie, op, fscache_op_enqueue_mythread); - _debug("queue for caller's attention"); - break; - default: - pr_err("Unexpected op type %lx", op->flags); - BUG(); - break; - } -} -EXPORT_SYMBOL(fscache_enqueue_operation); - -/* - * start an op running - */ -static void fscache_run_op(struct fscache_object *object, - struct fscache_operation *op) -{ - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING); - - op->state = FSCACHE_OP_ST_IN_PROGRESS; - object->n_in_progress++; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - if (op->processor) - fscache_enqueue_operation(op); - else - trace_fscache_op(object->cookie, op, fscache_op_run); - fscache_stat(&fscache_n_op_run); -} - -/* - * report an unexpected submission - */ -static void fscache_report_unexpected_submission(struct fscache_object *object, - struct fscache_operation *op, - const struct fscache_state *ostate) -{ - static bool once_only; - struct fscache_operation *p; - unsigned n; - - if (once_only) - return; - once_only = true; - - kdebug("unexpected submission OP%x [OBJ%x %s]", - op->debug_id, object->debug_id, object->state->name); - kdebug("objstate=%s [%s]", object->state->name, ostate->name); - kdebug("objflags=%lx", object->flags); - kdebug("objevent=%lx [%lx]", object->events, object->event_mask); - kdebug("ops=%u inp=%u exc=%u", - object->n_ops, object->n_in_progress, object->n_exclusive); - - if (!list_empty(&object->pending_ops)) { - n = 0; - list_for_each_entry(p, &object->pending_ops, pend_link) { - ASSERTCMP(p->object, ==, object); - kdebug("%p %p", op->processor, op->release); - n++; - } - - kdebug("n=%u", n); - } - - dump_stack(); -} - -/* - * submit an exclusive operation for an object - * - other ops are excluded from running simultaneously with this one - * - this gets any extra refs it needs on an op - */ -int fscache_submit_exclusive_op(struct fscache_object *object, - struct fscache_operation *op) -{ - const struct fscache_state *ostate; - unsigned long flags; - int ret; - - _enter("{OBJ%x OP%x},", object->debug_id, op->debug_id); - - trace_fscache_op(object->cookie, op, fscache_op_submit_ex); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - ASSERTCMP(object->n_ops, >=, object->n_in_progress); - ASSERTCMP(object->n_ops, >=, object->n_exclusive); - ASSERT(list_empty(&op->pend_link)); - - ostate = object->state; - smp_rmb(); - - op->state = FSCACHE_OP_ST_PENDING; - flags = READ_ONCE(object->flags); - if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) { - fscache_stat(&fscache_n_op_rejected); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else if (unlikely(fscache_cache_is_broken(object))) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -EIO; - } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) { - op->object = object; - object->n_ops++; - object->n_exclusive++; /* reads and writes must wait */ - - if (object->n_in_progress > 0) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - } else if (!list_empty(&object->pending_ops)) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - fscache_start_operations(object); - } else { - ASSERTCMP(object->n_in_progress, ==, 0); - fscache_run_op(object, op); - } - - /* need to issue a new write op after this */ - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) { - op->object = object; - object->n_ops++; - object->n_exclusive++; /* reads and writes must wait */ - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else { - fscache_report_unexpected_submission(object, op, ostate); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * submit an operation for an object - * - objects may be submitted only in the following states: - * - during object creation (write ops may be submitted) - * - whilst the object is active - * - after an I/O error incurred in one of the two above states (op rejected) - * - this gets any extra refs it needs on an op - */ -int fscache_submit_op(struct fscache_object *object, - struct fscache_operation *op) -{ - const struct fscache_state *ostate; - unsigned long flags; - int ret; - - _enter("{OBJ%x OP%x},{%u}", - object->debug_id, op->debug_id, atomic_read(&op->usage)); - - trace_fscache_op(object->cookie, op, fscache_op_submit); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - ASSERTCMP(object->n_ops, >=, object->n_in_progress); - ASSERTCMP(object->n_ops, >=, object->n_exclusive); - ASSERT(list_empty(&op->pend_link)); - - ostate = object->state; - smp_rmb(); - - op->state = FSCACHE_OP_ST_PENDING; - flags = READ_ONCE(object->flags); - if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) { - fscache_stat(&fscache_n_op_rejected); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else if (unlikely(fscache_cache_is_broken(object))) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -EIO; - } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) { - op->object = object; - object->n_ops++; - - if (object->n_exclusive > 0) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - } else if (!list_empty(&object->pending_ops)) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - fscache_start_operations(object); - } else { - ASSERTCMP(object->n_exclusive, ==, 0); - fscache_run_op(object, op); - } - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) { - op->object = object; - object->n_ops++; - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else { - fscache_report_unexpected_submission(object, op, ostate); - ASSERT(!fscache_object_is_active(object)); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * queue an object for withdrawal on error, aborting all following asynchronous - * operations - */ -void fscache_abort_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - fscache_raise_event(object, FSCACHE_OBJECT_EV_ERROR); -} - -/* - * Jump start the operation processing on an object. The caller must hold - * object->lock. - */ -void fscache_start_operations(struct fscache_object *object) -{ - struct fscache_operation *op; - bool stop = false; - - while (!list_empty(&object->pending_ops) && !stop) { - op = list_entry(object->pending_ops.next, - struct fscache_operation, pend_link); - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) { - if (object->n_in_progress > 0) - break; - stop = true; - } - list_del_init(&op->pend_link); - fscache_run_op(object, op); - - /* the pending queue was holding a ref on the object */ - fscache_put_operation(op); - } - - ASSERTCMP(object->n_in_progress, <=, object->n_ops); - - _debug("woke %d ops on OBJ%x", - object->n_in_progress, object->debug_id); -} - -/* - * cancel an operation that's pending on an object - */ -int fscache_cancel_op(struct fscache_operation *op, - bool cancel_in_progress_op) -{ - struct fscache_object *object = op->object; - bool put = false; - int ret; - - _enter("OBJ%x OP%x}", op->object->debug_id, op->debug_id); - - trace_fscache_op(object->cookie, op, fscache_op_cancel); - - ASSERTCMP(op->state, >=, FSCACHE_OP_ST_PENDING); - ASSERTCMP(op->state, !=, FSCACHE_OP_ST_CANCELLED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - - ret = -EBUSY; - if (op->state == FSCACHE_OP_ST_PENDING) { - ASSERT(!list_empty(&op->pend_link)); - list_del_init(&op->pend_link); - put = true; - - fscache_stat(&fscache_n_op_cancelled); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - ret = 0; - } else if (op->state == FSCACHE_OP_ST_IN_PROGRESS && cancel_in_progress_op) { - ASSERTCMP(object->n_in_progress, >, 0); - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - object->n_in_progress--; - if (object->n_in_progress == 0) - fscache_start_operations(object); - - fscache_stat(&fscache_n_op_cancelled); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - ret = 0; - } - - if (put) - fscache_put_operation(op); - spin_unlock(&object->lock); - _leave(" = %d", ret); - return ret; -} - -/* - * Cancel all pending operations on an object - */ -void fscache_cancel_all_ops(struct fscache_object *object) -{ - struct fscache_operation *op; - - _enter("OBJ%x", object->debug_id); - - spin_lock(&object->lock); - - while (!list_empty(&object->pending_ops)) { - op = list_entry(object->pending_ops.next, - struct fscache_operation, pend_link); - fscache_stat(&fscache_n_op_cancelled); - list_del_init(&op->pend_link); - - trace_fscache_op(object->cookie, op, fscache_op_cancel_all); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - fscache_put_operation(op); - cond_resched_lock(&object->lock); - } - - spin_unlock(&object->lock); - _leave(""); -} - -/* - * Record the completion or cancellation of an in-progress operation. - */ -void fscache_op_complete(struct fscache_operation *op, bool cancelled) -{ - struct fscache_object *object = op->object; - - _enter("OBJ%x", object->debug_id); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_IN_PROGRESS); - ASSERTCMP(object->n_in_progress, >, 0); - ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags), - object->n_exclusive, >, 0); - ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags), - object->n_in_progress, ==, 1); - - spin_lock(&object->lock); - - if (!cancelled) { - trace_fscache_op(object->cookie, op, fscache_op_completed); - op->state = FSCACHE_OP_ST_COMPLETE; - } else { - op->cancel(op); - trace_fscache_op(object->cookie, op, fscache_op_cancelled); - op->state = FSCACHE_OP_ST_CANCELLED; - } - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - object->n_in_progress--; - if (object->n_in_progress == 0) - fscache_start_operations(object); - - spin_unlock(&object->lock); - _leave(""); -} -EXPORT_SYMBOL(fscache_op_complete); - -/* - * release an operation - * - queues pending ops if this is the last in-progress op - */ -void fscache_put_operation(struct fscache_operation *op) -{ - struct fscache_object *object; - struct fscache_cache *cache; - - _enter("{OBJ%x OP%x,%d}", - op->object ? op->object->debug_id : 0, - op->debug_id, atomic_read(&op->usage)); - - ASSERTCMP(atomic_read(&op->usage), >, 0); - - if (!atomic_dec_and_test(&op->usage)) - return; - - trace_fscache_op(op->object ? op->object->cookie : NULL, op, fscache_op_put); - - _debug("PUT OP"); - ASSERTIFCMP(op->state != FSCACHE_OP_ST_INITIALISED && - op->state != FSCACHE_OP_ST_COMPLETE, - op->state, ==, FSCACHE_OP_ST_CANCELLED); - - fscache_stat(&fscache_n_op_release); - - if (op->release) { - op->release(op); - op->release = NULL; - } - op->state = FSCACHE_OP_ST_DEAD; - - object = op->object; - if (likely(object)) { - if (test_bit(FSCACHE_OP_DEC_READ_CNT, &op->flags)) - atomic_dec(&object->n_reads); - if (test_bit(FSCACHE_OP_UNUSE_COOKIE, &op->flags)) - fscache_unuse_cookie(object); - - /* now... we may get called with the object spinlock held, so we - * complete the cleanup here only if we can immediately acquire the - * lock, and defer it otherwise */ - if (!spin_trylock(&object->lock)) { - _debug("defer put"); - fscache_stat(&fscache_n_op_deferred_release); - - cache = object->cache; - spin_lock(&cache->op_gc_list_lock); - list_add_tail(&op->pend_link, &cache->op_gc_list); - spin_unlock(&cache->op_gc_list_lock); - schedule_work(&cache->op_gc); - _leave(" [defer]"); - return; - } - - ASSERTCMP(object->n_ops, >, 0); - object->n_ops--; - if (object->n_ops == 0) - fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED); - - spin_unlock(&object->lock); - } - - kfree(op); - _leave(" [done]"); -} -EXPORT_SYMBOL(fscache_put_operation); - -/* - * garbage collect operations that have had their release deferred - */ -void fscache_operation_gc(struct work_struct *work) -{ - struct fscache_operation *op; - struct fscache_object *object; - struct fscache_cache *cache = - container_of(work, struct fscache_cache, op_gc); - int count = 0; - - _enter(""); - - do { - spin_lock(&cache->op_gc_list_lock); - if (list_empty(&cache->op_gc_list)) { - spin_unlock(&cache->op_gc_list_lock); - break; - } - - op = list_entry(cache->op_gc_list.next, - struct fscache_operation, pend_link); - list_del(&op->pend_link); - spin_unlock(&cache->op_gc_list_lock); - - object = op->object; - trace_fscache_op(object->cookie, op, fscache_op_gc); - - spin_lock(&object->lock); - - _debug("GC DEFERRED REL OBJ%x OP%x", - object->debug_id, op->debug_id); - fscache_stat(&fscache_n_op_gc); - - ASSERTCMP(atomic_read(&op->usage), ==, 0); - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_DEAD); - - ASSERTCMP(object->n_ops, >, 0); - object->n_ops--; - if (object->n_ops == 0) - fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED); - - spin_unlock(&object->lock); - kfree(op); - - } while (count++ < 20); - - if (!list_empty(&cache->op_gc_list)) - schedule_work(&cache->op_gc); - - _leave(""); -} - -/* - * execute an operation using fs_op_wq to provide processing context - - * the caller holds a ref to this object, so we don't need to hold one - */ -void fscache_op_work_func(struct work_struct *work) -{ - struct fscache_operation *op = - container_of(work, struct fscache_operation, work); - - _enter("{OBJ%x OP%x,%d}", - op->object->debug_id, op->debug_id, atomic_read(&op->usage)); - - trace_fscache_op(op->object->cookie, op, fscache_op_work); - - ASSERT(op->processor != NULL); - op->processor(op); - fscache_put_operation(op); - - _leave(""); -} diff --git a/fs/fscache/page.c b/fs/fscache/page.c deleted file mode 100644 index 27df94ef0e0b..000000000000 --- a/fs/fscache/page.c +++ /dev/null @@ -1,1242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Cache page management and data I/O routines - * - * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL PAGE -#include <linux/module.h> -#include <linux/fscache-cache.h> -#include <linux/buffer_head.h> -#include <linux/pagevec.h> -#include <linux/slab.h> -#include "internal.h" - -/* - * check to see if a page is being written to the cache - */ -bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page) -{ - void *val; - - rcu_read_lock(); - val = radix_tree_lookup(&cookie->stores, page->index); - rcu_read_unlock(); - trace_fscache_check_page(cookie, page, val, 0); - - return val != NULL; -} -EXPORT_SYMBOL(__fscache_check_page_write); - -/* - * wait for a page to finish being written to the cache - */ -void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page) -{ - wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); - - trace_fscache_page(cookie, page, fscache_page_write_wait); - - wait_event(*wq, !__fscache_check_page_write(cookie, page)); -} -EXPORT_SYMBOL(__fscache_wait_on_page_write); - -/* - * wait for a page to finish being written to the cache. Put a timeout here - * since we might be called recursively via parent fs. - */ -static -bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page) -{ - wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); - - return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page), - HZ); -} - -/* - * decide whether a page can be released, possibly by cancelling a store to it - * - we're allowed to sleep if __GFP_DIRECT_RECLAIM is flagged - */ -bool __fscache_maybe_release_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - struct page *xpage; - void *val; - - _enter("%p,%p,%x", cookie, page, gfp); - - trace_fscache_page(cookie, page, fscache_page_maybe_release); - -try_again: - rcu_read_lock(); - val = radix_tree_lookup(&cookie->stores, page->index); - if (!val) { - rcu_read_unlock(); - fscache_stat(&fscache_n_store_vmscan_not_storing); - __fscache_uncache_page(cookie, page); - return true; - } - - /* see if the page is actually undergoing storage - if so we can't get - * rid of it till the cache has finished with it */ - if (radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG)) { - rcu_read_unlock(); - goto page_busy; - } - - /* the page is pending storage, so we attempt to cancel the store and - * discard the store request so that the page can be reclaimed */ - spin_lock(&cookie->stores_lock); - rcu_read_unlock(); - - if (radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG)) { - /* the page started to undergo storage whilst we were looking, - * so now we can only wait or return */ - spin_unlock(&cookie->stores_lock); - goto page_busy; - } - - xpage = radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - spin_unlock(&cookie->stores_lock); - - if (xpage) { - fscache_stat(&fscache_n_store_vmscan_cancelled); - fscache_stat(&fscache_n_store_radix_deletes); - ASSERTCMP(xpage, ==, page); - } else { - fscache_stat(&fscache_n_store_vmscan_gone); - } - - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - if (xpage) - put_page(xpage); - __fscache_uncache_page(cookie, page); - return true; - -page_busy: - /* We will wait here if we're allowed to, but that could deadlock the - * allocator as the work threads writing to the cache may all end up - * sleeping on memory allocation, so we may need to impose a timeout - * too. */ - if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) { - fscache_stat(&fscache_n_store_vmscan_busy); - return false; - } - - fscache_stat(&fscache_n_store_vmscan_wait); - if (!release_page_wait_timeout(cookie, page)) - _debug("fscache writeout timeout page: %p{%lx}", - page, page->index); - - gfp &= ~__GFP_DIRECT_RECLAIM; - goto try_again; -} -EXPORT_SYMBOL(__fscache_maybe_release_page); - -/* - * note that a page has finished being written to the cache - */ -static void fscache_end_page_write(struct fscache_object *object, - struct page *page) -{ - struct fscache_cookie *cookie; - struct page *xpage = NULL, *val; - - spin_lock(&object->lock); - cookie = object->cookie; - if (cookie) { - /* delete the page from the tree if it is now no longer - * pending */ - spin_lock(&cookie->stores_lock); - radix_tree_tag_clear(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_clear_store); - if (!radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG)) { - fscache_stat(&fscache_n_store_radix_deletes); - xpage = radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - trace_fscache_page(cookie, page, fscache_page_write_end); - - val = radix_tree_lookup(&cookie->stores, page->index); - trace_fscache_check_page(cookie, page, val, 1); - } else { - trace_fscache_page(cookie, page, fscache_page_write_end_pend); - } - spin_unlock(&cookie->stores_lock); - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - } else { - trace_fscache_page(cookie, page, fscache_page_write_end_noc); - } - spin_unlock(&object->lock); - if (xpage) - put_page(xpage); -} - -/* - * actually apply the changed attributes to a cache object - */ -static void fscache_attr_changed_op(struct fscache_operation *op) -{ - struct fscache_object *object = op->object; - int ret; - - _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id); - - fscache_stat(&fscache_n_attr_changed_calls); - - if (fscache_object_is_active(object)) { - fscache_stat(&fscache_n_cop_attr_changed); - ret = object->cache->ops->attr_changed(object); - fscache_stat_d(&fscache_n_cop_attr_changed); - if (ret < 0) - fscache_abort_object(object); - fscache_op_complete(op, ret < 0); - } else { - fscache_op_complete(op, true); - } - - _leave(""); -} - -/* - * notification that the attributes on an object have changed - */ -int __fscache_attr_changed(struct fscache_cookie *cookie) -{ - struct fscache_operation *op; - struct fscache_object *object; - bool wake_cookie = false; - - _enter("%p", cookie); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - - fscache_stat(&fscache_n_attr_changed); - - op = kzalloc(sizeof(*op), GFP_KERNEL); - if (!op) { - fscache_stat(&fscache_n_attr_changed_nomem); - _leave(" = -ENOMEM"); - return -ENOMEM; - } - - fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed); - op->flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_EXCLUSIVE) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - if (fscache_submit_exclusive_op(object, op) < 0) - goto nobufs_dec; - spin_unlock(&cookie->lock); - fscache_stat(&fscache_n_attr_changed_ok); - fscache_put_operation(op); - _leave(" = 0"); - return 0; - -nobufs_dec: - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs: - spin_unlock(&cookie->lock); - fscache_put_operation(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_stat(&fscache_n_attr_changed_nobufs); - _leave(" = %d", -ENOBUFS); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_attr_changed); - -/* - * Handle cancellation of a pending retrieval op - */ -static void fscache_do_cancel_retrieval(struct fscache_operation *_op) -{ - struct fscache_retrieval *op = - container_of(_op, struct fscache_retrieval, op); - - atomic_set(&op->n_pages, 0); -} - -/* - * release a retrieval op reference - */ -static void fscache_release_retrieval_op(struct fscache_operation *_op) -{ - struct fscache_retrieval *op = - container_of(_op, struct fscache_retrieval, op); - - _enter("{OP%x}", op->op.debug_id); - - ASSERTIFCMP(op->op.state != FSCACHE_OP_ST_INITIALISED, - atomic_read(&op->n_pages), ==, 0); - - if (op->context) - fscache_put_context(op->cookie, op->context); - - _leave(""); -} - -/* - * allocate a retrieval op - */ -struct fscache_retrieval *fscache_alloc_retrieval( - struct fscache_cookie *cookie, - struct address_space *mapping, - fscache_rw_complete_t end_io_func, - void *context) -{ - struct fscache_retrieval *op; - - /* allocate a retrieval operation and attempt to submit it */ - op = kzalloc(sizeof(*op), GFP_NOIO); - if (!op) { - fscache_stat(&fscache_n_retrievals_nomem); - return NULL; - } - - fscache_operation_init(cookie, &op->op, NULL, - fscache_do_cancel_retrieval, - fscache_release_retrieval_op); - op->op.flags = FSCACHE_OP_MYTHREAD | - (1UL << FSCACHE_OP_WAITING) | - (1UL << FSCACHE_OP_UNUSE_COOKIE); - op->cookie = cookie; - op->mapping = mapping; - op->end_io_func = end_io_func; - op->context = context; - INIT_LIST_HEAD(&op->to_do); - - /* Pin the netfs read context in case we need to do the actual netfs - * read because we've encountered a cache read failure. - */ - if (context) - fscache_get_context(op->cookie, context); - return op; -} - -/* - * wait for a deferred lookup to complete - */ -int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie) -{ - _enter(""); - - if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) { - _leave(" = 0 [imm]"); - return 0; - } - - fscache_stat(&fscache_n_retrievals_wait); - - if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, - TASK_INTERRUPTIBLE) != 0) { - fscache_stat(&fscache_n_retrievals_intr); - _leave(" = -ERESTARTSYS"); - return -ERESTARTSYS; - } - - ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)); - - smp_rmb(); - _leave(" = 0 [dly]"); - return 0; -} - -/* - * wait for an object to become active (or dead) - */ -int fscache_wait_for_operation_activation(struct fscache_object *object, - struct fscache_operation *op, - atomic_t *stat_op_waits, - atomic_t *stat_object_dead) -{ - int ret; - - if (!test_bit(FSCACHE_OP_WAITING, &op->flags)) - goto check_if_dead; - - _debug(">>> WT"); - if (stat_op_waits) - fscache_stat(stat_op_waits); - if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING, - TASK_INTERRUPTIBLE) != 0) { - trace_fscache_op(object->cookie, op, fscache_op_signal); - ret = fscache_cancel_op(op, false); - if (ret == 0) - return -ERESTARTSYS; - - /* it's been removed from the pending queue by another party, - * so we should get to run shortly */ - wait_on_bit(&op->flags, FSCACHE_OP_WAITING, - TASK_UNINTERRUPTIBLE); - } - _debug("<<< GO"); - -check_if_dead: - if (op->state == FSCACHE_OP_ST_CANCELLED) { - if (stat_object_dead) - fscache_stat(stat_object_dead); - _leave(" = -ENOBUFS [cancelled]"); - return -ENOBUFS; - } - if (unlikely(fscache_object_is_dying(object) || - fscache_cache_is_broken(object))) { - enum fscache_operation_state state = op->state; - trace_fscache_op(object->cookie, op, fscache_op_signal); - fscache_cancel_op(op, true); - if (stat_object_dead) - fscache_stat(stat_object_dead); - _leave(" = -ENOBUFS [obj dead %d]", state); - return -ENOBUFS; - } - return 0; -} - -/* - * read a page from the cache or allocate a block in which to store it - * - we return: - * -ENOMEM - out of memory, nothing done - * -ERESTARTSYS - interrupted - * -ENOBUFS - no backing object available in which to cache the block - * -ENODATA - no data available in the backing object for this block - * 0 - dispatched a read - it'll call end_io_func() when finished - */ -int __fscache_read_or_alloc_page(struct fscache_cookie *cookie, - struct page *page, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%p,,,", cookie, page); - - fscache_stat(&fscache_n_retrievals); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, page->mapping, - end_io_func, context); - if (!op) { - _leave(" = -ENOMEM"); - return -ENOMEM; - } - atomic_set(&op->n_pages, 1); - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)); - - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); - - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_retrieval_ops); - - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { - fscache_stat(&fscache_n_cop_allocate_page); - ret = object->cache->ops->allocate_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_allocate_page); - if (ret == 0) - ret = -ENODATA; - } else { - fscache_stat(&fscache_n_cop_read_or_alloc_page); - ret = object->cache->ops->read_or_alloc_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_read_or_alloc_page); - } - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_put_retrieval(op); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_read_or_alloc_page); - -/* - * read a list of page from the cache or allocate a block in which to store - * them - * - we return: - * -ENOMEM - out of memory, some pages may be being read - * -ERESTARTSYS - interrupted, some pages may be being read - * -ENOBUFS - no backing object or space available in which to cache any - * pages not being read - * -ENODATA - no data available in the backing object for some or all of - * the pages - * 0 - dispatched a read on all pages - * - * end_io_func() will be called for each page read from the cache as it is - * finishes being read - * - * any pages for which a read is dispatched will be removed from pages and - * nr_pages - */ -int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,,%d,,,", cookie, *nr_pages); - - fscache_stat(&fscache_n_retrievals); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(*nr_pages, >, 0); - ASSERT(!list_empty(pages)); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context); - if (!op) - return -ENOMEM; - atomic_set(&op->n_pages, *nr_pages); - trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); - - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_retrieval_ops); - - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { - fscache_stat(&fscache_n_cop_allocate_pages); - ret = object->cache->ops->allocate_pages( - op, pages, nr_pages, gfp); - fscache_stat_d(&fscache_n_cop_allocate_pages); - } else { - fscache_stat(&fscache_n_cop_read_or_alloc_pages); - ret = object->cache->ops->read_or_alloc_pages( - op, pages, nr_pages, gfp); - fscache_stat_d(&fscache_n_cop_read_or_alloc_pages); - } - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_read_or_alloc_pages); - -/* - * allocate a block in the cache on which to store a page - * - we return: - * -ENOMEM - out of memory, nothing done - * -ERESTARTSYS - interrupted - * -ENOBUFS - no backing object available in which to cache the block - * 0 - block allocated - */ -int __fscache_alloc_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%p,,,", cookie, page); - - fscache_stat(&fscache_n_allocs); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL); - if (!op) - return -ENOMEM; - atomic_set(&op->n_pages, 1); - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_alloc_ops); - - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_alloc_op_waits), - __fscache_stat(&fscache_n_allocs_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - fscache_stat(&fscache_n_cop_allocate_page); - ret = object->cache->ops->allocate_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_allocate_page); - -error: - if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_allocs_intr); - else if (ret < 0) - fscache_stat(&fscache_n_allocs_nobufs); - else - fscache_stat(&fscache_n_allocs_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_allocs_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_alloc_page); - -/* - * Unmark pages allocate in the readahead code path (via: - * fscache_readpages_or_alloc) after delegating to the base filesystem - */ -void __fscache_readpages_cancel(struct fscache_cookie *cookie, - struct list_head *pages) -{ - struct page *page; - - list_for_each_entry(page, pages, lru) { - if (PageFsCache(page)) - __fscache_uncache_page(cookie, page); - } -} -EXPORT_SYMBOL(__fscache_readpages_cancel); - -/* - * release a write op reference - */ -static void fscache_release_write_op(struct fscache_operation *_op) -{ - _enter("{OP%x}", _op->debug_id); -} - -/* - * perform the background storage of a page into the cache - */ -static void fscache_write_op(struct fscache_operation *_op) -{ - struct fscache_storage *op = - container_of(_op, struct fscache_storage, op); - struct fscache_object *object = op->op.object; - struct fscache_cookie *cookie; - struct page *page; - unsigned n; - void *results[1]; - int ret; - - _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage)); - -again: - spin_lock(&object->lock); - cookie = object->cookie; - - if (!fscache_object_is_active(object)) { - /* If we get here, then the on-disk cache object likely no - * longer exists, so we should just cancel this write - * operation. - */ - spin_unlock(&object->lock); - fscache_op_complete(&op->op, true); - _leave(" [inactive]"); - return; - } - - if (!cookie) { - /* If we get here, then the cookie belonging to the object was - * detached, probably by the cookie being withdrawn due to - * memory pressure, which means that the pages we might write - * to the cache from no longer exist - therefore, we can just - * cancel this write operation. - */ - spin_unlock(&object->lock); - fscache_op_complete(&op->op, true); - _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}", - _op->flags, _op->state, object->state->short_name, - object->flags); - return; - } - - spin_lock(&cookie->stores_lock); - - fscache_stat(&fscache_n_store_calls); - - /* find a page to store */ - results[0] = NULL; - page = NULL; - n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit); - if (n != 1) - goto superseded; - page = results[0]; - _debug("gang %d [%lx]", n, page->index); - - radix_tree_tag_set(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG); - radix_tree_tag_clear(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_pend2store); - - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - - if (page->index >= op->store_limit) - goto discard_page; - - fscache_stat(&fscache_n_store_pages); - fscache_stat(&fscache_n_cop_write_page); - ret = object->cache->ops->write_page(op, page); - fscache_stat_d(&fscache_n_cop_write_page); - trace_fscache_wrote_page(cookie, page, &op->op, ret); - fscache_end_page_write(object, page); - if (ret < 0) { - fscache_abort_object(object); - fscache_op_complete(&op->op, true); - } else { - fscache_enqueue_operation(&op->op); - } - - _leave(""); - return; - -discard_page: - fscache_stat(&fscache_n_store_pages_over_limit); - trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS); - fscache_end_page_write(object, page); - goto again; - -superseded: - /* this writer is going away and there aren't any more things to - * write */ - _debug("cease"); - spin_unlock(&cookie->stores_lock); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - spin_unlock(&object->lock); - fscache_op_complete(&op->op, false); - _leave(""); -} - -/* - * Clear the pages pending writing for invalidation - */ -void fscache_invalidate_writes(struct fscache_cookie *cookie) -{ - struct page *page; - void *results[16]; - int n, i; - - _enter(""); - - for (;;) { - spin_lock(&cookie->stores_lock); - n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, - ARRAY_SIZE(results), - FSCACHE_COOKIE_PENDING_TAG); - if (n == 0) { - spin_unlock(&cookie->stores_lock); - break; - } - - for (i = n - 1; i >= 0; i--) { - page = results[i]; - radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - trace_fscache_page(cookie, page, fscache_page_inval); - } - - spin_unlock(&cookie->stores_lock); - - for (i = n - 1; i >= 0; i--) - put_page(results[i]); - } - - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - - _leave(""); -} - -/* - * request a page be stored in the cache - * - returns: - * -ENOMEM - out of memory, nothing done - * -ENOBUFS - no backing object available in which to cache the page - * 0 - dispatched a write - it'll call end_io_func() when finished - * - * if the cookie still has a backing object at this point, that object can be - * in one of a few states with respect to storage processing: - * - * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is - * set) - * - * (a) no writes yet - * - * (b) writes deferred till post-creation (mark page for writing and - * return immediately) - * - * (2) negative lookup, object created, initial fill being made from netfs - * - * (a) fill point not yet reached this page (mark page for writing and - * return) - * - * (b) fill point passed this page (queue op to store this page) - * - * (3) object extant (queue op to store this page) - * - * any other state is invalid - */ -int __fscache_write_page(struct fscache_cookie *cookie, - struct page *page, - loff_t object_size, - gfp_t gfp) -{ - struct fscache_storage *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%x,", cookie, (u32) page->flags); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERT(PageFsCache(page)); - - fscache_stat(&fscache_n_stores); - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY); - if (!op) - goto nomem; - - fscache_operation_init(cookie, &op->op, fscache_write_op, NULL, - fscache_release_write_op); - op->op.flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_WAITING) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - - ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM); - if (ret < 0) - goto nomem_free; - - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one); - - ret = -ENOBUFS; - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) - goto nobufs; - - trace_fscache_page(cookie, page, fscache_page_write); - - /* add the page to the pending-storage radix tree on the backing - * object */ - spin_lock(&object->lock); - - if (object->store_limit_l != object_size) - fscache_set_store_limit(object, object_size); - - spin_lock(&cookie->stores_lock); - - _debug("store limit %llx", (unsigned long long) object->store_limit); - - ret = radix_tree_insert(&cookie->stores, page->index, page); - if (ret < 0) { - if (ret == -EEXIST) - goto already_queued; - _debug("insert failed %d", ret); - goto nobufs_unlock_obj; - } - - trace_fscache_page(cookie, page, fscache_page_radix_insert); - radix_tree_tag_set(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_set_pend); - get_page(page); - - /* we only want one writer at a time, but we do need to queue new - * writers after exclusive ops */ - if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags)) - goto already_pending; - - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - - op->op.debug_id = atomic_inc_return(&fscache_op_debug_id); - op->store_limit = object->store_limit; - - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, &op->op) < 0) - goto submit_failed; - - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_stat(&fscache_n_store_ops); - fscache_stat(&fscache_n_stores_ok); - - /* the work queue now carries its own ref on the object */ - fscache_put_operation(&op->op); - _leave(" = 0"); - return 0; - -already_queued: - fscache_stat(&fscache_n_stores_again); -already_pending: - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_put_operation(&op->op); - fscache_stat(&fscache_n_stores_ok); - _leave(" = 0"); - return 0; - -submit_failed: - spin_lock(&cookie->stores_lock); - radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - spin_unlock(&cookie->stores_lock); - wake_cookie = __fscache_unuse_cookie(cookie); - put_page(page); - ret = -ENOBUFS; - goto nobufs; - -nobufs_unlock_obj: - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); -nobufs: - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_put_operation(&op->op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_stat(&fscache_n_stores_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; - -nomem_free: - fscache_put_operation(&op->op); -nomem: - fscache_stat(&fscache_n_stores_oom); - _leave(" = -ENOMEM"); - return -ENOMEM; -} -EXPORT_SYMBOL(__fscache_write_page); - -/* - * remove a page from the cache - */ -void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page) -{ - struct fscache_object *object; - - _enter(",%p", page); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - fscache_stat(&fscache_n_uncaches); - - /* cache withdrawal may beat us to it */ - if (!PageFsCache(page)) - goto done; - - trace_fscache_page(cookie, page, fscache_page_uncache); - - /* get the object */ - spin_lock(&cookie->lock); - - if (hlist_empty(&cookie->backing_objects)) { - ClearPageFsCache(page); - goto done_unlock; - } - - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - /* there might now be stuff on disk we could read */ - clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - - /* only invoke the cache backend if we managed to mark the page - * uncached here; this deals with synchronisation vs withdrawal */ - if (TestClearPageFsCache(page) && - object->cache->ops->uncache_page) { - /* the cache backend releases the cookie lock */ - fscache_stat(&fscache_n_cop_uncache_page); - object->cache->ops->uncache_page(object, page); - fscache_stat_d(&fscache_n_cop_uncache_page); - goto done; - } - -done_unlock: - spin_unlock(&cookie->lock); -done: - _leave(""); -} -EXPORT_SYMBOL(__fscache_uncache_page); - -/** - * fscache_mark_page_cached - Mark a page as being cached - * @op: The retrieval op pages are being marked for - * @page: The page to be marked - * - * Mark a netfs page as being cached. After this is called, the netfs - * must call fscache_uncache_page() to remove the mark. - */ -void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page) -{ - struct fscache_cookie *cookie = op->op.object->cookie; - -#ifdef CONFIG_FSCACHE_STATS - atomic_inc(&fscache_n_marks); -#endif - - trace_fscache_page(cookie, page, fscache_page_cached); - - _debug("- mark %p{%lx}", page, page->index); - if (TestSetPageFsCache(page)) { - static bool once_only; - if (!once_only) { - once_only = true; - pr_warn("Cookie type %s marked page %lx multiple times\n", - cookie->def->name, page->index); - } - } - - if (cookie->def->mark_page_cached) - cookie->def->mark_page_cached(cookie->netfs_data, - op->mapping, page); -} -EXPORT_SYMBOL(fscache_mark_page_cached); - -/** - * fscache_mark_pages_cached - Mark pages as being cached - * @op: The retrieval op pages are being marked for - * @pagevec: The pages to be marked - * - * Mark a bunch of netfs pages as being cached. After this is called, - * the netfs must call fscache_uncache_page() to remove the mark. - */ -void fscache_mark_pages_cached(struct fscache_retrieval *op, - struct pagevec *pagevec) -{ - unsigned long loop; - - for (loop = 0; loop < pagevec->nr; loop++) - fscache_mark_page_cached(op, pagevec->pages[loop]); - - pagevec_reinit(pagevec); -} -EXPORT_SYMBOL(fscache_mark_pages_cached); - -/* - * Uncache all the pages in an inode that are marked PG_fscache, assuming them - * to be associated with the given cookie. - */ -void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie, - struct inode *inode) -{ - struct address_space *mapping = inode->i_mapping; - struct pagevec pvec; - pgoff_t next; - int i; - - _enter("%p,%p", cookie, inode); - - if (!mapping || mapping->nrpages == 0) { - _leave(" [no pages]"); - return; - } - - pagevec_init(&pvec); - next = 0; - do { - if (!pagevec_lookup(&pvec, mapping, &next)) - break; - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; - if (PageFsCache(page)) { - __fscache_wait_on_page_write(cookie, page); - __fscache_uncache_page(cookie, page); - } - } - pagevec_release(&pvec); - cond_resched(); - } while (next); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_uncache_all_inode_pages); diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c index 061df8f61ffc..dc3b0e9c8cce 100644 --- a/fs/fscache/proc.c +++ b/fs/fscache/proc.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* FS-Cache statistics viewing interface * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ -#define FSCACHE_DEBUG_LEVEL OPERATION +#define FSCACHE_DEBUG_LEVEL CACHE #include <linux/module.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> @@ -16,42 +16,32 @@ */ int __init fscache_proc_init(void) { - _enter(""); - if (!proc_mkdir("fs/fscache", NULL)) goto error_dir; + if (!proc_create_seq("fs/fscache/caches", S_IFREG | 0444, NULL, + &fscache_caches_seq_ops)) + goto error; + + if (!proc_create_seq("fs/fscache/volumes", S_IFREG | 0444, NULL, + &fscache_volumes_seq_ops)) + goto error; + if (!proc_create_seq("fs/fscache/cookies", S_IFREG | 0444, NULL, &fscache_cookies_seq_ops)) - goto error_cookies; + goto error; #ifdef CONFIG_FSCACHE_STATS if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, - fscache_stats_show)) - goto error_stats; + fscache_stats_show)) + goto error; #endif -#ifdef CONFIG_FSCACHE_OBJECT_LIST - if (!proc_create("fs/fscache/objects", S_IFREG | 0444, NULL, - &fscache_objlist_proc_ops)) - goto error_objects; -#endif - - _leave(" = 0"); return 0; -#ifdef CONFIG_FSCACHE_OBJECT_LIST -error_objects: -#endif -#ifdef CONFIG_FSCACHE_STATS - remove_proc_entry("fs/fscache/stats", NULL); -error_stats: -#endif - remove_proc_entry("fs/fscache/cookies", NULL); -error_cookies: +error: remove_proc_entry("fs/fscache", NULL); error_dir: - _leave(" = -ENOMEM"); return -ENOMEM; } @@ -60,12 +50,5 @@ error_dir: */ void fscache_proc_cleanup(void) { -#ifdef CONFIG_FSCACHE_OBJECT_LIST - remove_proc_entry("fs/fscache/objects", NULL); -#endif -#ifdef CONFIG_FSCACHE_STATS - remove_proc_entry("fs/fscache/stats", NULL); -#endif - remove_proc_entry("fs/fscache/cookies", NULL); - remove_proc_entry("fs/fscache", NULL); + remove_proc_subtree("fs/fscache", NULL); } diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index a7c3ed89a3e0..fc94e5e79f1c 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -1,12 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* FS-Cache statistics * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ -#define FSCACHE_DEBUG_LEVEL THREAD -#include <linux/module.h> +#define FSCACHE_DEBUG_LEVEL CACHE #include <linux/proc_fs.h> #include <linux/seq_file.h> #include "internal.h" @@ -14,122 +13,41 @@ /* * operation counters */ -atomic_t fscache_n_op_pend; -atomic_t fscache_n_op_run; -atomic_t fscache_n_op_enqueue; -atomic_t fscache_n_op_deferred_release; -atomic_t fscache_n_op_initialised; -atomic_t fscache_n_op_release; -atomic_t fscache_n_op_gc; -atomic_t fscache_n_op_cancelled; -atomic_t fscache_n_op_rejected; - -atomic_t fscache_n_attr_changed; -atomic_t fscache_n_attr_changed_ok; -atomic_t fscache_n_attr_changed_nobufs; -atomic_t fscache_n_attr_changed_nomem; -atomic_t fscache_n_attr_changed_calls; - -atomic_t fscache_n_allocs; -atomic_t fscache_n_allocs_ok; -atomic_t fscache_n_allocs_wait; -atomic_t fscache_n_allocs_nobufs; -atomic_t fscache_n_allocs_intr; -atomic_t fscache_n_allocs_object_dead; -atomic_t fscache_n_alloc_ops; -atomic_t fscache_n_alloc_op_waits; - -atomic_t fscache_n_retrievals; -atomic_t fscache_n_retrievals_ok; -atomic_t fscache_n_retrievals_wait; -atomic_t fscache_n_retrievals_nodata; -atomic_t fscache_n_retrievals_nobufs; -atomic_t fscache_n_retrievals_intr; -atomic_t fscache_n_retrievals_nomem; -atomic_t fscache_n_retrievals_object_dead; -atomic_t fscache_n_retrieval_ops; -atomic_t fscache_n_retrieval_op_waits; - -atomic_t fscache_n_stores; -atomic_t fscache_n_stores_ok; -atomic_t fscache_n_stores_again; -atomic_t fscache_n_stores_nobufs; -atomic_t fscache_n_stores_oom; -atomic_t fscache_n_store_ops; -atomic_t fscache_n_store_calls; -atomic_t fscache_n_store_pages; -atomic_t fscache_n_store_radix_deletes; -atomic_t fscache_n_store_pages_over_limit; - -atomic_t fscache_n_store_vmscan_not_storing; -atomic_t fscache_n_store_vmscan_gone; -atomic_t fscache_n_store_vmscan_busy; -atomic_t fscache_n_store_vmscan_cancelled; -atomic_t fscache_n_store_vmscan_wait; - -atomic_t fscache_n_marks; -atomic_t fscache_n_uncaches; +atomic_t fscache_n_volumes; +atomic_t fscache_n_volumes_collision; +atomic_t fscache_n_volumes_nomem; +atomic_t fscache_n_cookies; +atomic_t fscache_n_cookies_lru; +atomic_t fscache_n_cookies_lru_expired; +atomic_t fscache_n_cookies_lru_removed; +atomic_t fscache_n_cookies_lru_dropped; atomic_t fscache_n_acquires; -atomic_t fscache_n_acquires_null; -atomic_t fscache_n_acquires_no_cache; atomic_t fscache_n_acquires_ok; -atomic_t fscache_n_acquires_nobufs; atomic_t fscache_n_acquires_oom; atomic_t fscache_n_invalidates; -atomic_t fscache_n_invalidates_run; atomic_t fscache_n_updates; -atomic_t fscache_n_updates_null; -atomic_t fscache_n_updates_run; +EXPORT_SYMBOL(fscache_n_updates); atomic_t fscache_n_relinquishes; -atomic_t fscache_n_relinquishes_null; -atomic_t fscache_n_relinquishes_waitcrt; atomic_t fscache_n_relinquishes_retire; - -atomic_t fscache_n_cookie_index; -atomic_t fscache_n_cookie_data; -atomic_t fscache_n_cookie_special; - -atomic_t fscache_n_object_alloc; -atomic_t fscache_n_object_no_alloc; -atomic_t fscache_n_object_lookups; -atomic_t fscache_n_object_lookups_negative; -atomic_t fscache_n_object_lookups_positive; -atomic_t fscache_n_object_lookups_timed_out; -atomic_t fscache_n_object_created; -atomic_t fscache_n_object_avail; -atomic_t fscache_n_object_dead; - -atomic_t fscache_n_checkaux_none; -atomic_t fscache_n_checkaux_okay; -atomic_t fscache_n_checkaux_update; -atomic_t fscache_n_checkaux_obsolete; - -atomic_t fscache_n_cop_alloc_object; -atomic_t fscache_n_cop_lookup_object; -atomic_t fscache_n_cop_lookup_complete; -atomic_t fscache_n_cop_grab_object; -atomic_t fscache_n_cop_invalidate_object; -atomic_t fscache_n_cop_update_object; -atomic_t fscache_n_cop_drop_object; -atomic_t fscache_n_cop_put_object; -atomic_t fscache_n_cop_sync_cache; -atomic_t fscache_n_cop_attr_changed; -atomic_t fscache_n_cop_read_or_alloc_page; -atomic_t fscache_n_cop_read_or_alloc_pages; -atomic_t fscache_n_cop_allocate_page; -atomic_t fscache_n_cop_allocate_pages; -atomic_t fscache_n_cop_write_page; -atomic_t fscache_n_cop_uncache_page; -atomic_t fscache_n_cop_dissociate_pages; - -atomic_t fscache_n_cache_no_space_reject; -atomic_t fscache_n_cache_stale_objects; -atomic_t fscache_n_cache_retired_objects; -atomic_t fscache_n_cache_culled_objects; +atomic_t fscache_n_relinquishes_dropped; + +atomic_t fscache_n_resizes; +atomic_t fscache_n_resizes_null; + +atomic_t fscache_n_read; +EXPORT_SYMBOL(fscache_n_read); +atomic_t fscache_n_write; +EXPORT_SYMBOL(fscache_n_write); +atomic_t fscache_n_no_write_space; +EXPORT_SYMBOL(fscache_n_no_write_space); +atomic_t fscache_n_no_create_space; +EXPORT_SYMBOL(fscache_n_no_create_space); +atomic_t fscache_n_culled; +EXPORT_SYMBOL(fscache_n_culled); /* * display the general statistics @@ -137,147 +55,48 @@ atomic_t fscache_n_cache_culled_objects; int fscache_stats_show(struct seq_file *m, void *v) { seq_puts(m, "FS-Cache statistics\n"); - - seq_printf(m, "Cookies: idx=%u dat=%u spc=%u\n", - atomic_read(&fscache_n_cookie_index), - atomic_read(&fscache_n_cookie_data), - atomic_read(&fscache_n_cookie_special)); - - seq_printf(m, "Objects: alc=%u nal=%u avl=%u ded=%u\n", - atomic_read(&fscache_n_object_alloc), - atomic_read(&fscache_n_object_no_alloc), - atomic_read(&fscache_n_object_avail), - atomic_read(&fscache_n_object_dead)); - seq_printf(m, "ChkAux : non=%u ok=%u upd=%u obs=%u\n", - atomic_read(&fscache_n_checkaux_none), - atomic_read(&fscache_n_checkaux_okay), - atomic_read(&fscache_n_checkaux_update), - atomic_read(&fscache_n_checkaux_obsolete)); - - seq_printf(m, "Pages : mrk=%u unc=%u\n", - atomic_read(&fscache_n_marks), - atomic_read(&fscache_n_uncaches)); - - seq_printf(m, "Acquire: n=%u nul=%u noc=%u ok=%u nbf=%u" - " oom=%u\n", + seq_printf(m, "Cookies: n=%d v=%d vcol=%u voom=%u\n", + atomic_read(&fscache_n_cookies), + atomic_read(&fscache_n_volumes), + atomic_read(&fscache_n_volumes_collision), + atomic_read(&fscache_n_volumes_nomem) + ); + + seq_printf(m, "Acquire: n=%u ok=%u oom=%u\n", atomic_read(&fscache_n_acquires), - atomic_read(&fscache_n_acquires_null), - atomic_read(&fscache_n_acquires_no_cache), atomic_read(&fscache_n_acquires_ok), - atomic_read(&fscache_n_acquires_nobufs), atomic_read(&fscache_n_acquires_oom)); - seq_printf(m, "Lookups: n=%u neg=%u pos=%u crt=%u tmo=%u\n", - atomic_read(&fscache_n_object_lookups), - atomic_read(&fscache_n_object_lookups_negative), - atomic_read(&fscache_n_object_lookups_positive), - atomic_read(&fscache_n_object_created), - atomic_read(&fscache_n_object_lookups_timed_out)); + seq_printf(m, "LRU : n=%u exp=%u rmv=%u drp=%u at=%ld\n", + atomic_read(&fscache_n_cookies_lru), + atomic_read(&fscache_n_cookies_lru_expired), + atomic_read(&fscache_n_cookies_lru_removed), + atomic_read(&fscache_n_cookies_lru_dropped), + timer_pending(&fscache_cookie_lru_timer) ? + fscache_cookie_lru_timer.expires - jiffies : 0); - seq_printf(m, "Invals : n=%u run=%u\n", - atomic_read(&fscache_n_invalidates), - atomic_read(&fscache_n_invalidates_run)); + seq_printf(m, "Invals : n=%u\n", + atomic_read(&fscache_n_invalidates)); - seq_printf(m, "Updates: n=%u nul=%u run=%u\n", + seq_printf(m, "Updates: n=%u rsz=%u rsn=%u\n", atomic_read(&fscache_n_updates), - atomic_read(&fscache_n_updates_null), - atomic_read(&fscache_n_updates_run)); + atomic_read(&fscache_n_resizes), + atomic_read(&fscache_n_resizes_null)); - seq_printf(m, "Relinqs: n=%u nul=%u wcr=%u rtr=%u\n", + seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", atomic_read(&fscache_n_relinquishes), - atomic_read(&fscache_n_relinquishes_null), - atomic_read(&fscache_n_relinquishes_waitcrt), - atomic_read(&fscache_n_relinquishes_retire)); - - seq_printf(m, "AttrChg: n=%u ok=%u nbf=%u oom=%u run=%u\n", - atomic_read(&fscache_n_attr_changed), - atomic_read(&fscache_n_attr_changed_ok), - atomic_read(&fscache_n_attr_changed_nobufs), - atomic_read(&fscache_n_attr_changed_nomem), - atomic_read(&fscache_n_attr_changed_calls)); - - seq_printf(m, "Allocs : n=%u ok=%u wt=%u nbf=%u int=%u\n", - atomic_read(&fscache_n_allocs), - atomic_read(&fscache_n_allocs_ok), - atomic_read(&fscache_n_allocs_wait), - atomic_read(&fscache_n_allocs_nobufs), - atomic_read(&fscache_n_allocs_intr)); - seq_printf(m, "Allocs : ops=%u owt=%u abt=%u\n", - atomic_read(&fscache_n_alloc_ops), - atomic_read(&fscache_n_alloc_op_waits), - atomic_read(&fscache_n_allocs_object_dead)); - - seq_printf(m, "Retrvls: n=%u ok=%u wt=%u nod=%u nbf=%u" - " int=%u oom=%u\n", - atomic_read(&fscache_n_retrievals), - atomic_read(&fscache_n_retrievals_ok), - atomic_read(&fscache_n_retrievals_wait), - atomic_read(&fscache_n_retrievals_nodata), - atomic_read(&fscache_n_retrievals_nobufs), - atomic_read(&fscache_n_retrievals_intr), - atomic_read(&fscache_n_retrievals_nomem)); - seq_printf(m, "Retrvls: ops=%u owt=%u abt=%u\n", - atomic_read(&fscache_n_retrieval_ops), - atomic_read(&fscache_n_retrieval_op_waits), - atomic_read(&fscache_n_retrievals_object_dead)); - - seq_printf(m, "Stores : n=%u ok=%u agn=%u nbf=%u oom=%u\n", - atomic_read(&fscache_n_stores), - atomic_read(&fscache_n_stores_ok), - atomic_read(&fscache_n_stores_again), - atomic_read(&fscache_n_stores_nobufs), - atomic_read(&fscache_n_stores_oom)); - seq_printf(m, "Stores : ops=%u run=%u pgs=%u rxd=%u olm=%u\n", - atomic_read(&fscache_n_store_ops), - atomic_read(&fscache_n_store_calls), - atomic_read(&fscache_n_store_pages), - atomic_read(&fscache_n_store_radix_deletes), - atomic_read(&fscache_n_store_pages_over_limit)); + atomic_read(&fscache_n_relinquishes_retire), + atomic_read(&fscache_n_relinquishes_dropped)); - seq_printf(m, "VmScan : nos=%u gon=%u bsy=%u can=%u wt=%u\n", - atomic_read(&fscache_n_store_vmscan_not_storing), - atomic_read(&fscache_n_store_vmscan_gone), - atomic_read(&fscache_n_store_vmscan_busy), - atomic_read(&fscache_n_store_vmscan_cancelled), - atomic_read(&fscache_n_store_vmscan_wait)); + seq_printf(m, "NoSpace: nwr=%u ncr=%u cull=%u\n", + atomic_read(&fscache_n_no_write_space), + atomic_read(&fscache_n_no_create_space), + atomic_read(&fscache_n_culled)); - seq_printf(m, "Ops : pend=%u run=%u enq=%u can=%u rej=%u\n", - atomic_read(&fscache_n_op_pend), - atomic_read(&fscache_n_op_run), - atomic_read(&fscache_n_op_enqueue), - atomic_read(&fscache_n_op_cancelled), - atomic_read(&fscache_n_op_rejected)); - seq_printf(m, "Ops : ini=%u dfr=%u rel=%u gc=%u\n", - atomic_read(&fscache_n_op_initialised), - atomic_read(&fscache_n_op_deferred_release), - atomic_read(&fscache_n_op_release), - atomic_read(&fscache_n_op_gc)); + seq_printf(m, "IO : rd=%u wr=%u\n", + atomic_read(&fscache_n_read), + atomic_read(&fscache_n_write)); - seq_printf(m, "CacheOp: alo=%d luo=%d luc=%d gro=%d\n", - atomic_read(&fscache_n_cop_alloc_object), - atomic_read(&fscache_n_cop_lookup_object), - atomic_read(&fscache_n_cop_lookup_complete), - atomic_read(&fscache_n_cop_grab_object)); - seq_printf(m, "CacheOp: inv=%d upo=%d dro=%d pto=%d atc=%d syn=%d\n", - atomic_read(&fscache_n_cop_invalidate_object), - atomic_read(&fscache_n_cop_update_object), - atomic_read(&fscache_n_cop_drop_object), - atomic_read(&fscache_n_cop_put_object), - atomic_read(&fscache_n_cop_attr_changed), - atomic_read(&fscache_n_cop_sync_cache)); - seq_printf(m, "CacheOp: rap=%d ras=%d alp=%d als=%d wrp=%d ucp=%d dsp=%d\n", - atomic_read(&fscache_n_cop_read_or_alloc_page), - atomic_read(&fscache_n_cop_read_or_alloc_pages), - atomic_read(&fscache_n_cop_allocate_page), - atomic_read(&fscache_n_cop_allocate_pages), - atomic_read(&fscache_n_cop_write_page), - atomic_read(&fscache_n_cop_uncache_page), - atomic_read(&fscache_n_cop_dissociate_pages)); - seq_printf(m, "CacheEv: nsp=%d stl=%d rtr=%d cul=%d\n", - atomic_read(&fscache_n_cache_no_space_reject), - atomic_read(&fscache_n_cache_stale_objects), - atomic_read(&fscache_n_cache_retired_objects), - atomic_read(&fscache_n_cache_culled_objects)); netfs_stats_show(m); return 0; } diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c new file mode 100644 index 000000000000..a57c6cbee858 --- /dev/null +++ b/fs/fscache/volume.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume-level cache cookie handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include <linux/export.h> +#include <linux/slab.h> +#include "internal.h" + +#define fscache_volume_hash_shift 10 +static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; +static atomic_t fscache_volume_debug_id; +static LIST_HEAD(fscache_volumes); + +static void fscache_create_volume_work(struct work_struct *work); + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref; + + __refcount_inc(&volume->ref, &ref); + trace_fscache_volume(volume->debug_id, ref + 1, where); + return volume; +} + +static void fscache_see_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref = refcount_read(&volume->ref); + + trace_fscache_volume(volume->debug_id, ref, where); +} + +/* + * Pin the cache behind a volume so that we can access it. + */ +static void __fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&volume->n_accesses); + smp_mb__after_atomic(); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); +} + +/** + * fscache_begin_volume_access - Pin a cache so a volume can be accessed + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing a volume and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the volume's n_accesses + * count and then recheck the cache liveness, ending the access if it + * ceased to be live. + * + * (3) When we end the access, we decrement the volume's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cache is caching, the volume's n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, the volume's n_accesses is decremented and we wait for it to + * become 0. + * + * The datafile @cookie and the @why indicator are merely provided for tracing + * purposes. + */ +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!fscache_cache_is_live(volume->cache)) + return false; + __fscache_begin_volume_access(volume, cookie, why); + if (!fscache_cache_is_live(volume->cache)) { + fscache_end_volume_access(volume, cookie, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_volume_access - Unpin a cache at the end of an access. + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache volume after we've accessed it. The datafile @cookie and the + * @why indicator are merely provided for tracing purposes. + */ +void fscache_end_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&volume->n_accesses); +} +EXPORT_SYMBOL(fscache_end_volume_access); + +static bool fscache_volume_same(const struct fscache_volume *a, + const struct fscache_volume *b) +{ + size_t klen; + + if (a->key_hash != b->key_hash || + a->cache != b->cache || + a->key[0] != b->key[0]) + return false; + + klen = round_up(a->key[0] + 1, sizeof(__le32)); + return memcmp(a->key, b->key, klen) == 0; +} + +static bool fscache_is_acquire_pending(struct fscache_volume *volume) +{ + return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); +} + +static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, + unsigned int collidee_debug_id) +{ + wait_var_event_timeout(&candidate->flags, + fscache_is_acquire_pending(candidate), 20 * HZ); + if (!fscache_is_acquire_pending(candidate)) { + pr_notice("Potential volume collision new=%08x old=%08x", + candidate->debug_id, collidee_debug_id); + fscache_stat(&fscache_n_volumes_collision); + wait_var_event(&candidate->flags, fscache_is_acquire_pending(candidate)); + } +} + +/* + * Attempt to insert the new volume into the hash. If there's a collision, we + * wait for the old volume to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_volume(struct fscache_volume *candidate) +{ + struct fscache_volume *cursor; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket, collidee_debug_id = 0; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(candidate, cursor)) { + if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) + goto collision; + fscache_see_volume(cursor, fscache_volume_get_hash_collision); + set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); + set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); + collidee_debug_id = cursor->debug_id; + break; + } + } + + hlist_bl_add_head(&candidate->hash_link, h); + hlist_bl_unlock(h); + + if (test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags)) + fscache_wait_on_volume_collision(candidate, collidee_debug_id); + return true; + +collision: + fscache_see_volume(cursor, fscache_volume_collision); + hlist_bl_unlock(h); + return false; +} + +/* + * Allocate and initialise a volume representation cookie. + */ +static struct fscache_volume *fscache_alloc_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + struct fscache_cache *cache; + size_t klen, hlen; + char *key; + + if (!coherency_data) + coherency_len = 0; + + cache = fscache_lookup_cache(cache_name, false); + if (IS_ERR(cache)) + return NULL; + + volume = kzalloc(struct_size(volume, coherency, coherency_len), + GFP_KERNEL); + if (!volume) + goto err_cache; + + volume->cache = cache; + volume->coherency_len = coherency_len; + if (coherency_data) + memcpy(volume->coherency, coherency_data, coherency_len); + INIT_LIST_HEAD(&volume->proc_link); + INIT_WORK(&volume->work, fscache_create_volume_work); + refcount_set(&volume->ref, 1); + spin_lock_init(&volume->lock); + + /* Stick the length on the front of the key and pad it out to make + * hashing easier. + */ + klen = strlen(volume_key); + hlen = round_up(1 + klen + 1, sizeof(__le32)); + key = kzalloc(hlen, GFP_KERNEL); + if (!key) + goto err_vol; + key[0] = klen; + memcpy(key + 1, volume_key, klen); + + volume->key = key; + volume->key_hash = fscache_hash(0, key, hlen); + + volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); + down_write(&fscache_addremove_sem); + atomic_inc(&cache->n_volumes); + list_add_tail(&volume->proc_link, &fscache_volumes); + fscache_see_volume(volume, fscache_volume_new_acquire); + fscache_stat(&fscache_n_volumes); + up_write(&fscache_addremove_sem); + _leave(" = v=%x", volume->debug_id); + return volume; + +err_vol: + kfree(volume); +err_cache: + fscache_put_cache(cache, fscache_cache_put_alloc_volume); + fscache_stat(&fscache_n_volumes_nomem); + return NULL; +} + +/* + * Create a volume's representation on disk. Have a volume ref and a cache + * access we have to release. + */ +static void fscache_create_volume_work(struct work_struct *work) +{ + const struct fscache_cache_ops *ops; + struct fscache_volume *volume = + container_of(work, struct fscache_volume, work); + + fscache_see_volume(volume, fscache_volume_see_create_work); + + ops = volume->cache->ops; + if (ops->acquire_volume) + ops->acquire_volume(volume); + fscache_end_cache_access(volume->cache, + fscache_access_acquire_volume_end); + + clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); + wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); + fscache_put_volume(volume, fscache_volume_put_create_work); +} + +/* + * Dispatch a worker thread to create a volume's representation on disk. + */ +void fscache_create_volume(struct fscache_volume *volume, bool wait) +{ + if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) + goto maybe_wait; + if (volume->cache_priv) + goto no_wait; /* We raced */ + if (!fscache_begin_cache_access(volume->cache, + fscache_access_acquire_volume)) + goto no_wait; + + fscache_get_volume(volume, fscache_volume_get_create_work); + if (!schedule_work(&volume->work)) + fscache_put_volume(volume, fscache_volume_put_create_work); + +maybe_wait: + if (wait) { + fscache_see_volume(volume, fscache_volume_wait_create_work); + wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, + TASK_UNINTERRUPTIBLE); + } + return; +no_wait: + clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); + wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); +} + +/* + * Acquire a volume representation cookie and link it to a (proposed) cache. + */ +struct fscache_volume *__fscache_acquire_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + + volume = fscache_alloc_volume(volume_key, cache_name, + coherency_data, coherency_len); + if (!volume) + return ERR_PTR(-ENOMEM); + + if (!fscache_hash_volume(volume)) { + fscache_put_volume(volume, fscache_volume_put_hash_collision); + return ERR_PTR(-EBUSY); + } + + fscache_create_volume(volume, false); + return volume; +} +EXPORT_SYMBOL(__fscache_acquire_volume); + +static void fscache_wake_pending_volume(struct fscache_volume *volume, + struct hlist_bl_head *h) +{ + struct fscache_volume *cursor; + struct hlist_bl_node *p; + + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(cursor, volume)) { + fscache_see_volume(cursor, fscache_volume_see_hash_wake); + clear_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &cursor->flags); + wake_up_bit(&cursor->flags, FSCACHE_VOLUME_ACQUIRE_PENDING); + return; + } + } +} + +/* + * Remove a volume cookie from the hash table. + */ +static void fscache_unhash_volume(struct fscache_volume *volume) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&volume->hash_link); + if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) + fscache_wake_pending_volume(volume, h); + hlist_bl_unlock(h); +} + +/* + * Drop a cache's volume attachments. + */ +static void fscache_free_volume(struct fscache_volume *volume) +{ + struct fscache_cache *cache = volume->cache; + + if (volume->cache_priv) { + __fscache_begin_volume_access(volume, NULL, + fscache_access_relinquish_volume); + if (volume->cache_priv) + cache->ops->free_volume(volume); + fscache_end_volume_access(volume, NULL, + fscache_access_relinquish_volume_end); + } + + down_write(&fscache_addremove_sem); + list_del_init(&volume->proc_link); + atomic_dec(&volume->cache->n_volumes); + up_write(&fscache_addremove_sem); + + if (!hlist_bl_unhashed(&volume->hash_link)) + fscache_unhash_volume(volume); + + trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); + kfree(volume->key); + kfree(volume); + fscache_stat_d(&fscache_n_volumes); + fscache_put_cache(cache, fscache_cache_put_volume); +} + +/* + * Drop a reference to a volume cookie. + */ +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + if (volume) { + unsigned int debug_id = volume->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&volume->ref, &ref); + trace_fscache_volume(debug_id, ref - 1, where); + if (zero) + fscache_free_volume(volume); + } +} + +/* + * Relinquish a volume representation cookie. + */ +void __fscache_relinquish_volume(struct fscache_volume *volume, + const void *coherency_data, + bool invalidate) +{ + if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) + return; + + if (invalidate) { + set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); + } else if (coherency_data) { + memcpy(volume->coherency, coherency_data, volume->coherency_len); + } + + fscache_put_volume(volume, fscache_volume_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_volume); + +/** + * fscache_withdraw_volume - Withdraw a volume from being cached + * @volume: Volume cookie + * + * Withdraw a cache volume from service, waiting for all accesses to complete + * before returning. + */ +void fscache_withdraw_volume(struct fscache_volume *volume) +{ + int n_accesses; + + _debug("withdraw V=%x", volume->debug_id); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, 0, + refcount_read(&volume->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&volume->n_accesses, + atomic_read(&volume->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_volume); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of volumes in /proc/fs/fscache/volumes + */ +static int fscache_volumes_seq_show(struct seq_file *m, void *v) +{ + struct fscache_volume *volume; + + if (v == &fscache_volumes) { + seq_puts(m, + "VOLUME REF nCOOK ACC FL CACHE KEY\n" + "======== ===== ===== === == =============== ================\n"); + return 0; + } + + volume = list_entry(v, struct fscache_volume, proc_link); + seq_printf(m, + "%08x %5d %5d %3d %02lx %-15.15s %s\n", + volume->debug_id, + refcount_read(&volume->ref), + atomic_read(&volume->n_cookies), + atomic_read(&volume->n_accesses), + volume->flags, + volume->cache->name ?: "-", + volume->key + 1); + return 0; +} + +static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(&fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_volumes, *_pos); +} + +static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_volumes, _pos); +} + +static void fscache_volumes_seq_stop(struct seq_file *m, void *v) + __releases(&fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_volumes_seq_ops = { + .start = fscache_volumes_seq_start, + .next = fscache_volumes_seq_next, + .stop = fscache_volumes_seq_stop, + .show = fscache_volumes_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/namei.c b/fs/namei.c index 1f9d2187c765..d81f04f8d818 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3958,7 +3958,8 @@ int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir, inode_lock(dentry->d_inode); error = -EBUSY; - if (is_local_mountpoint(dentry)) + if (is_local_mountpoint(dentry) || + (dentry->d_inode->i_flags & S_KERNEL_FILE)) goto out; error = security_inode_rmdir(dir, dentry); diff --git a/fs/netfs/read_helper.c b/fs/netfs/read_helper.c index 75c76cbb27cc..6169659857b3 100644 --- a/fs/netfs/read_helper.c +++ b/fs/netfs/read_helper.c @@ -170,7 +170,7 @@ static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error */ static void netfs_read_from_cache(struct netfs_read_request *rreq, struct netfs_read_subrequest *subreq, - bool seek_data) + enum netfs_read_from_hole read_hole) { struct netfs_cache_resources *cres = &rreq->cache_resources; struct iov_iter iter; @@ -180,7 +180,7 @@ static void netfs_read_from_cache(struct netfs_read_request *rreq, subreq->start + subreq->transferred, subreq->len - subreq->transferred); - cres->ops->read(cres, subreq->start, &iter, seek_data, + cres->ops->read(cres, subreq->start, &iter, read_hole, netfs_cache_read_terminated, subreq); } @@ -323,7 +323,7 @@ static void netfs_rreq_do_write_to_cache(struct netfs_read_request *rreq) } ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len, - rreq->i_size); + rreq->i_size, true); if (ret < 0) { trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write); trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip); @@ -461,7 +461,7 @@ static void netfs_rreq_short_read(struct netfs_read_request *rreq, netfs_get_read_subrequest(subreq); atomic_inc(&rreq->nr_rd_ops); if (subreq->source == NETFS_READ_FROM_CACHE) - netfs_read_from_cache(rreq, subreq, true); + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR); else netfs_read_from_server(rreq, subreq); } @@ -789,7 +789,7 @@ static bool netfs_rreq_submit_slice(struct netfs_read_request *rreq, netfs_read_from_server(rreq, subreq); break; case NETFS_READ_FROM_CACHE: - netfs_read_from_cache(rreq, subreq, false); + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE); break; default: BUG(); diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile index 22d11fdc6deb..5f6db37f461e 100644 --- a/fs/nfs/Makefile +++ b/fs/nfs/Makefile @@ -12,7 +12,7 @@ nfs-y := client.o dir.o file.o getroot.o inode.o super.o \ export.o sysfs.o fs_context.o nfs-$(CONFIG_ROOT_NFS) += nfsroot.o nfs-$(CONFIG_SYSCTL) += sysctl.o -nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o +nfs-$(CONFIG_NFS_FSCACHE) += fscache.o obj-$(CONFIG_NFS_V2) += nfsv2.o nfsv2-y := nfs2super.o proc.o nfs2xdr.o diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 1e4dc1ab9312..8d8b85b5a641 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -183,8 +183,6 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_net = get_net(cl_init->net); clp->cl_principal = "*"; - nfs_fscache_get_client_cookie(clp); - return clp; error_cleanup: @@ -238,8 +236,6 @@ static void pnfs_init_server(struct nfs_server *server) */ void nfs_free_client(struct nfs_client *clp) { - nfs_fscache_release_client_cookie(clp); - /* -EIO all pending I/O */ if (!IS_ERR(clp->cl_rpcclient)) rpc_shutdown_client(clp->cl_rpcclient); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 9cff8709c80a..eabfdab543c8 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -59,6 +59,7 @@ #include "internal.h" #include "iostat.h" #include "pnfs.h" +#include "fscache.h" #define NFSDBG_FACILITY NFSDBG_VFS @@ -959,6 +960,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter) } else { result = requested; } + nfs_fscache_invalidate(inode, FSCACHE_INVAL_DIO_WRITE); out_release: nfs_direct_req_release(dreq); out: diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 24e7dccce355..76d76acbc594 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -84,6 +84,7 @@ nfs_file_release(struct inode *inode, struct file *filp) nfs_inc_stats(inode, NFSIOS_VFSRELEASE); nfs_file_clear_open_context(filp); + nfs_fscache_release_file(inode, filp); return 0; } EXPORT_SYMBOL_GPL(nfs_file_release); @@ -415,8 +416,7 @@ static void nfs_invalidate_page(struct page *page, unsigned int offset, return; /* Cancel any unstarted writes on this page */ nfs_wb_page_cancel(page_file_mapping(page)->host, page); - - nfs_fscache_invalidate_page(page, page->mapping->host); + wait_on_page_fscache(page); } /* @@ -475,12 +475,11 @@ static void nfs_check_dirty_writeback(struct page *page, static int nfs_launder_page(struct page *page) { struct inode *inode = page_file_mapping(page)->host; - struct nfs_inode *nfsi = NFS_I(inode); dfprintk(PAGECACHE, "NFS: launder_page(%ld, %llu)\n", inode->i_ino, (long long)page_offset(page)); - nfs_fscache_wait_on_page_write(nfsi, page); + wait_on_page_fscache(page); return nfs_wb_page(inode, page); } @@ -555,7 +554,11 @@ static vm_fault_t nfs_vm_page_mkwrite(struct vm_fault *vmf) sb_start_pagefault(inode->i_sb); /* make sure the cache has finished storing the page */ - nfs_fscache_wait_on_page_write(NFS_I(inode), page); + if (PageFsCache(page) && + wait_on_page_fscache_killable(vmf->page) < 0) { + ret = VM_FAULT_RETRY; + goto out; + } wait_on_bit_action(&NFS_I(inode)->flags, NFS_INO_INVALIDATING, nfs_wait_bit_killable, TASK_KILLABLE); diff --git a/fs/nfs/fscache-index.c b/fs/nfs/fscache-index.c deleted file mode 100644 index 573b1da9342c..000000000000 --- a/fs/nfs/fscache-index.c +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* NFS FS-Cache index structure definition - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/nfs_fs.h> -#include <linux/nfs_fs_sb.h> -#include <linux/in6.h> -#include <linux/iversion.h> - -#include "internal.h" -#include "fscache.h" - -#define NFSDBG_FACILITY NFSDBG_FSCACHE - -/* - * Define the NFS filesystem for FS-Cache. Upon registration FS-Cache sticks - * the cookie for the top-level index object for NFS into here. The top-level - * index can than have other cache objects inserted into it. - */ -struct fscache_netfs nfs_fscache_netfs = { - .name = "nfs", - .version = 0, -}; - -/* - * Register NFS for caching - */ -int nfs_fscache_register(void) -{ - return fscache_register_netfs(&nfs_fscache_netfs); -} - -/* - * Unregister NFS for caching - */ -void nfs_fscache_unregister(void) -{ - fscache_unregister_netfs(&nfs_fscache_netfs); -} - -/* - * Define the server object for FS-Cache. This is used to describe a server - * object to fscache_acquire_cookie(). It is keyed by the NFS protocol and - * server address parameters. - */ -const struct fscache_cookie_def nfs_fscache_server_index_def = { - .name = "NFS.server", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -/* - * Define the superblock object for FS-Cache. This is used to describe a - * superblock object to fscache_acquire_cookie(). It is keyed by all the NFS - * parameters that might cause a separate superblock. - */ -const struct fscache_cookie_def nfs_fscache_super_index_def = { - .name = "NFS.super", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -/* - * Consult the netfs about the state of an object - * - This function can be absent if the index carries no state data - * - The netfs data from the cookie being used as the target is - * presented, as is the auxiliary data - */ -static -enum fscache_checkaux nfs_fscache_inode_check_aux(void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size) -{ - struct nfs_fscache_inode_auxdata auxdata; - struct nfs_inode *nfsi = cookie_netfs_data; - - if (datalen != sizeof(auxdata)) - return FSCACHE_CHECKAUX_OBSOLETE; - - memset(&auxdata, 0, sizeof(auxdata)); - auxdata.mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; - auxdata.mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; - auxdata.ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; - auxdata.ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; - - if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) - auxdata.change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); - - if (memcmp(data, &auxdata, datalen) != 0) - return FSCACHE_CHECKAUX_OBSOLETE; - - return FSCACHE_CHECKAUX_OKAY; -} - -/* - * Get an extra reference on a read context. - * - This function can be absent if the completion function doesn't require a - * context. - * - The read context is passed back to NFS in the event that a data read on the - * cache fails with EIO - in which case the server must be contacted to - * retrieve the data, which requires the read context for security. - */ -static void nfs_fh_get_context(void *cookie_netfs_data, void *context) -{ - get_nfs_open_context(context); -} - -/* - * Release an extra reference on a read context. - * - This function can be absent if the completion function doesn't require a - * context. - */ -static void nfs_fh_put_context(void *cookie_netfs_data, void *context) -{ - if (context) - put_nfs_open_context(context); -} - -/* - * Define the inode object for FS-Cache. This is used to describe an inode - * object to fscache_acquire_cookie(). It is keyed by the NFS file handle for - * an inode. - * - * Coherency is managed by comparing the copies of i_size, i_mtime and i_ctime - * held in the cache auxiliary data for the data storage object with those in - * the inode struct in memory. - */ -const struct fscache_cookie_def nfs_fscache_inode_object_def = { - .name = "NFS.fh", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .check_aux = nfs_fscache_inode_check_aux, - .get_context = nfs_fh_get_context, - .put_context = nfs_fh_put_context, -}; diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index d743629e05e1..cfe901650ab0 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -22,24 +22,18 @@ #define NFSDBG_FACILITY NFSDBG_FSCACHE -static struct rb_root nfs_fscache_keys = RB_ROOT; -static DEFINE_SPINLOCK(nfs_fscache_keys_lock); +#define NFS_MAX_KEY_LEN 1000 -/* - * Layout of the key for an NFS server cache object. - */ -struct nfs_server_key { - struct { - uint16_t nfsversion; /* NFS protocol version */ - uint32_t minorversion; /* NFSv4 minor version */ - uint16_t family; /* address family */ - __be16 port; /* IP port */ - } hdr; - union { - struct in_addr ipv4_addr; /* IPv4 address */ - struct in6_addr ipv6_addr; /* IPv6 address */ - }; -} __packed; +static bool nfs_append_int(char *key, int *_len, unsigned long long x) +{ + if (*_len > NFS_MAX_KEY_LEN) + return false; + if (x == 0) + key[(*_len)++] = ','; + else + *_len += sprintf(key + *_len, ",%llx", x); + return true; +} /* * Get the per-client index cookie for an NFS client if the appropriate mount @@ -47,160 +41,108 @@ struct nfs_server_key { * - We always try and get an index cookie for the client, but get filehandle * cookies on a per-superblock basis, depending on the mount flags */ -void nfs_fscache_get_client_cookie(struct nfs_client *clp) +static bool nfs_fscache_get_client_key(struct nfs_client *clp, + char *key, int *_len) { const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &clp->cl_addr; const struct sockaddr_in *sin = (struct sockaddr_in *) &clp->cl_addr; - struct nfs_server_key key; - uint16_t len = sizeof(key.hdr); - memset(&key, 0, sizeof(key)); - key.hdr.nfsversion = clp->rpc_ops->version; - key.hdr.minorversion = clp->cl_minorversion; - key.hdr.family = clp->cl_addr.ss_family; + *_len += snprintf(key + *_len, NFS_MAX_KEY_LEN - *_len, + ",%u.%u,%x", + clp->rpc_ops->version, + clp->cl_minorversion, + clp->cl_addr.ss_family); switch (clp->cl_addr.ss_family) { case AF_INET: - key.hdr.port = sin->sin_port; - key.ipv4_addr = sin->sin_addr; - len += sizeof(key.ipv4_addr); - break; + if (!nfs_append_int(key, _len, sin->sin_port) || + !nfs_append_int(key, _len, sin->sin_addr.s_addr)) + return false; + return true; case AF_INET6: - key.hdr.port = sin6->sin6_port; - key.ipv6_addr = sin6->sin6_addr; - len += sizeof(key.ipv6_addr); - break; + if (!nfs_append_int(key, _len, sin6->sin6_port) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[0]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[1]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[2]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[3])) + return false; + return true; default: printk(KERN_WARNING "NFS: Unknown network family '%d'\n", clp->cl_addr.ss_family); - clp->fscache = NULL; - return; + return false; } - - /* create a cache index for looking up filehandles */ - clp->fscache = fscache_acquire_cookie(nfs_fscache_netfs.primary_index, - &nfs_fscache_server_index_def, - &key, len, - NULL, 0, - clp, 0, true); - dfprintk(FSCACHE, "NFS: get client cookie (0x%p/0x%p)\n", - clp, clp->fscache); -} - -/* - * Dispose of a per-client cookie - */ -void nfs_fscache_release_client_cookie(struct nfs_client *clp) -{ - dfprintk(FSCACHE, "NFS: releasing client cookie (0x%p/0x%p)\n", - clp, clp->fscache); - - fscache_relinquish_cookie(clp->fscache, NULL, false); - clp->fscache = NULL; } /* - * Get the cache cookie for an NFS superblock. We have to handle - * uniquification here because the cache doesn't do it for us. + * Get the cache cookie for an NFS superblock. * * The default uniquifier is just an empty string, but it may be overridden * either by the 'fsc=xxx' option to mount, or by inheriting it from the parent * superblock across an automount point of some nature. */ -void nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int ulen) +int nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int ulen) { - struct nfs_fscache_key *key, *xkey; + struct fscache_volume *vcookie; struct nfs_server *nfss = NFS_SB(sb); - struct rb_node **p, *parent; - int diff; + unsigned int len = 3; + char *key; - nfss->fscache_key = NULL; - nfss->fscache = NULL; - if (!uniq) { - uniq = ""; - ulen = 1; + if (uniq) { + nfss->fscache_uniq = kmemdup_nul(uniq, ulen, GFP_KERNEL); + if (!nfss->fscache_uniq) + return -ENOMEM; } - key = kzalloc(sizeof(*key) + ulen, GFP_KERNEL); + key = kmalloc(NFS_MAX_KEY_LEN + 24, GFP_KERNEL); if (!key) - return; - - key->nfs_client = nfss->nfs_client; - key->key.super.s_flags = sb->s_flags & NFS_SB_MASK; - key->key.nfs_server.flags = nfss->flags; - key->key.nfs_server.rsize = nfss->rsize; - key->key.nfs_server.wsize = nfss->wsize; - key->key.nfs_server.acregmin = nfss->acregmin; - key->key.nfs_server.acregmax = nfss->acregmax; - key->key.nfs_server.acdirmin = nfss->acdirmin; - key->key.nfs_server.acdirmax = nfss->acdirmax; - key->key.nfs_server.fsid = nfss->fsid; - key->key.rpc_auth.au_flavor = nfss->client->cl_auth->au_flavor; - - key->key.uniq_len = ulen; - memcpy(key->key.uniquifier, uniq, ulen); - - spin_lock(&nfs_fscache_keys_lock); - p = &nfs_fscache_keys.rb_node; - parent = NULL; - while (*p) { - parent = *p; - xkey = rb_entry(parent, struct nfs_fscache_key, node); - - if (key->nfs_client < xkey->nfs_client) - goto go_left; - if (key->nfs_client > xkey->nfs_client) - goto go_right; - - diff = memcmp(&key->key, &xkey->key, sizeof(key->key)); - if (diff < 0) - goto go_left; - if (diff > 0) - goto go_right; - - if (key->key.uniq_len == 0) - goto non_unique; - diff = memcmp(key->key.uniquifier, - xkey->key.uniquifier, - key->key.uniq_len); - if (diff < 0) - goto go_left; - if (diff > 0) - goto go_right; - goto non_unique; - - go_left: - p = &(*p)->rb_left; - continue; - go_right: - p = &(*p)->rb_right; + return -ENOMEM; + + memcpy(key, "nfs", 3); + if (!nfs_fscache_get_client_key(nfss->nfs_client, key, &len) || + !nfs_append_int(key, &len, nfss->fsid.major) || + !nfs_append_int(key, &len, nfss->fsid.minor) || + !nfs_append_int(key, &len, sb->s_flags & NFS_SB_MASK) || + !nfs_append_int(key, &len, nfss->flags) || + !nfs_append_int(key, &len, nfss->rsize) || + !nfs_append_int(key, &len, nfss->wsize) || + !nfs_append_int(key, &len, nfss->acregmin) || + !nfs_append_int(key, &len, nfss->acregmax) || + !nfs_append_int(key, &len, nfss->acdirmin) || + !nfs_append_int(key, &len, nfss->acdirmax) || + !nfs_append_int(key, &len, nfss->client->cl_auth->au_flavor)) + goto out; + + if (ulen > 0) { + if (ulen > NFS_MAX_KEY_LEN - len) + goto out; + key[len++] = ','; + memcpy(key + len, uniq, ulen); + len += ulen; } - - rb_link_node(&key->node, parent, p); - rb_insert_color(&key->node, &nfs_fscache_keys); - spin_unlock(&nfs_fscache_keys_lock); - nfss->fscache_key = key; + key[len] = 0; /* create a cache index for looking up filehandles */ - nfss->fscache = fscache_acquire_cookie(nfss->nfs_client->fscache, - &nfs_fscache_super_index_def, - &key->key, - sizeof(key->key) + ulen, - NULL, 0, - nfss, 0, true); + vcookie = fscache_acquire_volume(key, + NULL, /* preferred_cache */ + NULL, 0 /* coherency_data */); dfprintk(FSCACHE, "NFS: get superblock cookie (0x%p/0x%p)\n", - nfss, nfss->fscache); - return; + nfss, vcookie); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(key); + return PTR_ERR(vcookie); + } + pr_err("NFS: Cache volume key already in use (%s)\n", key); + vcookie = NULL; + } + nfss->fscache = vcookie; -non_unique: - spin_unlock(&nfs_fscache_keys_lock); +out: kfree(key); - nfss->fscache_key = NULL; - nfss->fscache = NULL; - printk(KERN_WARNING "NFS:" - " Cache request denied due to non-unique superblock keys\n"); + return 0; } /* @@ -213,29 +155,9 @@ void nfs_fscache_release_super_cookie(struct super_block *sb) dfprintk(FSCACHE, "NFS: releasing superblock cookie (0x%p/0x%p)\n", nfss, nfss->fscache); - fscache_relinquish_cookie(nfss->fscache, NULL, false); + fscache_relinquish_volume(nfss->fscache, NULL, false); nfss->fscache = NULL; - - if (nfss->fscache_key) { - spin_lock(&nfs_fscache_keys_lock); - rb_erase(&nfss->fscache_key->node, &nfs_fscache_keys); - spin_unlock(&nfs_fscache_keys_lock); - kfree(nfss->fscache_key); - nfss->fscache_key = NULL; - } -} - -static void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, - struct nfs_inode *nfsi) -{ - memset(auxdata, 0, sizeof(*auxdata)); - auxdata->mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; - auxdata->mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; - auxdata->ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; - auxdata->ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; - - if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) - auxdata->change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); + kfree(nfss->fscache_uniq); } /* @@ -254,10 +176,12 @@ void nfs_fscache_init_inode(struct inode *inode) nfs_fscache_update_auxdata(&auxdata, nfsi); nfsi->fscache = fscache_acquire_cookie(NFS_SB(inode->i_sb)->fscache, - &nfs_fscache_inode_object_def, - nfsi->fh.data, nfsi->fh.size, - &auxdata, sizeof(auxdata), - nfsi, nfsi->vfs_inode.i_size, false); + 0, + nfsi->fh.data, /* index_key */ + nfsi->fh.size, + &auxdata, /* aux_data */ + sizeof(auxdata), + i_size_read(&nfsi->vfs_inode)); } /* @@ -265,24 +189,15 @@ void nfs_fscache_init_inode(struct inode *inode) */ void nfs_fscache_clear_inode(struct inode *inode) { - struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); dfprintk(FSCACHE, "NFS: clear cookie (0x%p/0x%p)\n", nfsi, cookie); - nfs_fscache_update_auxdata(&auxdata, nfsi); - fscache_relinquish_cookie(cookie, &auxdata, false); + fscache_relinquish_cookie(cookie, false); nfsi->fscache = NULL; } -static bool nfs_fscache_can_enable(void *data) -{ - struct inode *inode = data; - - return !inode_is_open_for_write(inode); -} - /* * Enable or disable caching for a file that is being opened as appropriate. * The cookie is allocated when the inode is initialised, but is not enabled at @@ -307,100 +222,104 @@ void nfs_fscache_open_file(struct inode *inode, struct file *filp) struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); + bool open_for_write = inode_is_open_for_write(inode); if (!fscache_cookie_valid(cookie)) return; - nfs_fscache_update_auxdata(&auxdata, nfsi); - - if (inode_is_open_for_write(inode)) { + fscache_use_cookie(cookie, open_for_write); + if (open_for_write) { dfprintk(FSCACHE, "NFS: nfsi 0x%p disabling cache\n", nfsi); - clear_bit(NFS_INO_FSCACHE, &nfsi->flags); - fscache_disable_cookie(cookie, &auxdata, true); - fscache_uncache_all_inode_pages(cookie, inode); - } else { - dfprintk(FSCACHE, "NFS: nfsi 0x%p enabling cache\n", nfsi); - fscache_enable_cookie(cookie, &auxdata, nfsi->vfs_inode.i_size, - nfs_fscache_can_enable, inode); - if (fscache_cookie_enabled(cookie)) - set_bit(NFS_INO_FSCACHE, &NFS_I(inode)->flags); + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_invalidate(cookie, &auxdata, i_size_read(inode), + FSCACHE_INVAL_DIO_WRITE); } } EXPORT_SYMBOL_GPL(nfs_fscache_open_file); -/* - * Release the caching state associated with a page, if the page isn't busy - * interacting with the cache. - * - Returns true (can release page) or false (page busy). - */ -int nfs_fscache_release_page(struct page *page, gfp_t gfp) +void nfs_fscache_release_file(struct inode *inode, struct file *filp) { - if (PageFsCache(page)) { - struct fscache_cookie *cookie = nfs_i_fscache(page->mapping->host); - - BUG_ON(!cookie); - dfprintk(FSCACHE, "NFS: fscache releasepage (0x%p/0x%p/0x%p)\n", - cookie, page, NFS_I(page->mapping->host)); - - if (!fscache_maybe_release_page(cookie, page, gfp)) - return 0; + struct nfs_fscache_inode_auxdata auxdata; + struct nfs_inode *nfsi = NFS_I(inode); + struct fscache_cookie *cookie = nfs_i_fscache(inode); - nfs_inc_fscache_stats(page->mapping->host, - NFSIOS_FSCACHE_PAGES_UNCACHED); + if (fscache_cookie_valid(cookie)) { + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_unuse_cookie(cookie, &auxdata, NULL); } +} - return 1; +static inline void fscache_end_operation(struct netfs_cache_resources *cres) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + + if (ops) + ops->end_operation(cres); } /* - * Release the caching state associated with a page if undergoing complete page - * invalidation. + * Fallback page reading interface. */ -void __nfs_fscache_invalidate_page(struct page *page, struct inode *inode) +static int fscache_fallback_read_page(struct inode *inode, struct page *page) { + struct netfs_cache_resources cres; struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct iov_iter iter; + struct bio_vec bvec[1]; + int ret; - BUG_ON(!cookie); - - dfprintk(FSCACHE, "NFS: fscache invalidatepage (0x%p/0x%p/0x%p)\n", - cookie, page, NFS_I(inode)); + memset(&cres, 0, sizeof(cres)); + bvec[0].bv_page = page; + bvec[0].bv_offset = 0; + bvec[0].bv_len = PAGE_SIZE; + iov_iter_bvec(&iter, READ, bvec, ARRAY_SIZE(bvec), PAGE_SIZE); - fscache_wait_on_page_write(cookie, page); + ret = fscache_begin_read_operation(&cres, cookie); + if (ret < 0) + return ret; - BUG_ON(!PageLocked(page)); - fscache_uncache_page(cookie, page); - nfs_inc_fscache_stats(page->mapping->host, - NFSIOS_FSCACHE_PAGES_UNCACHED); + ret = fscache_read(&cres, page_offset(page), &iter, NETFS_READ_HOLE_FAIL, + NULL, NULL); + fscache_end_operation(&cres); + return ret; } /* - * Handle completion of a page being read from the cache. - * - Called in process (keventd) context. + * Fallback page writing interface. */ -static void nfs_readpage_from_fscache_complete(struct page *page, - void *context, - int error) +static int fscache_fallback_write_page(struct inode *inode, struct page *page, + bool no_space_allocated_yet) { - dfprintk(FSCACHE, - "NFS: readpage_from_fscache_complete (0x%p/0x%p/%d)\n", - page, context, error); - - /* - * If the read completes with an error, mark the page with PG_checked, - * unlock the page, and let the VM reissue the readpage. - */ - if (!error) - SetPageUptodate(page); - else - SetPageChecked(page); - unlock_page(page); + struct netfs_cache_resources cres; + struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct iov_iter iter; + struct bio_vec bvec[1]; + loff_t start = page_offset(page); + size_t len = PAGE_SIZE; + int ret; + + memset(&cres, 0, sizeof(cres)); + bvec[0].bv_page = page; + bvec[0].bv_offset = 0; + bvec[0].bv_len = PAGE_SIZE; + iov_iter_bvec(&iter, WRITE, bvec, ARRAY_SIZE(bvec), PAGE_SIZE); + + ret = fscache_begin_write_operation(&cres, cookie); + if (ret < 0) + return ret; + + ret = cres.ops->prepare_write(&cres, &start, &len, i_size_read(inode), + no_space_allocated_yet); + if (ret == 0) + ret = fscache_write(&cres, page_offset(page), &iter, NULL, NULL); + fscache_end_operation(&cres); + return ret; } /* * Retrieve a page from fscache */ -int __nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, struct page *page) +int __nfs_readpage_from_fscache(struct inode *inode, struct page *page) { int ret; @@ -409,112 +328,49 @@ int __nfs_readpage_from_fscache(struct nfs_open_context *ctx, nfs_i_fscache(inode), page, page->index, page->flags, inode); if (PageChecked(page)) { + dfprintk(FSCACHE, "NFS: readpage_from_fscache: PageChecked\n"); ClearPageChecked(page); return 1; } - ret = fscache_read_or_alloc_page(nfs_i_fscache(inode), - page, - nfs_readpage_from_fscache_complete, - ctx, - GFP_KERNEL); - - switch (ret) { - case 0: /* read BIO submitted (page in fscache) */ - dfprintk(FSCACHE, - "NFS: readpage_from_fscache: BIO submitted\n"); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK); - return ret; - - case -ENOBUFS: /* inode not in cache */ - case -ENODATA: /* page not in cache */ + ret = fscache_fallback_read_page(inode, page); + if (ret < 0) { nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); dfprintk(FSCACHE, - "NFS: readpage_from_fscache %d\n", ret); - return 1; - - default: - dfprintk(FSCACHE, "NFS: readpage_from_fscache %d\n", ret); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); - } - return ret; -} - -/* - * Retrieve a set of pages from fscache - */ -int __nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - unsigned npages = *nr_pages; - int ret; - - dfprintk(FSCACHE, "NFS: nfs_getpages_from_fscache (0x%p/%u/0x%p)\n", - nfs_i_fscache(inode), npages, inode); - - ret = fscache_read_or_alloc_pages(nfs_i_fscache(inode), - mapping, pages, nr_pages, - nfs_readpage_from_fscache_complete, - ctx, - mapping_gfp_mask(mapping)); - if (*nr_pages < npages) - nfs_add_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK, - npages); - if (*nr_pages > 0) - nfs_add_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL, - *nr_pages); - - switch (ret) { - case 0: /* read submitted to the cache for all pages */ - BUG_ON(!list_empty(pages)); - BUG_ON(*nr_pages != 0); - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: submitted\n"); - + "NFS: readpage_from_fscache failed %d\n", ret); + SetPageChecked(page); return ret; - - case -ENOBUFS: /* some pages aren't cached and can't be */ - case -ENODATA: /* some pages aren't cached */ - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: no page: %d\n", ret); - return 1; - - default: - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: ret %d\n", ret); } - return ret; + /* Read completed synchronously */ + dfprintk(FSCACHE, "NFS: readpage_from_fscache: read successful\n"); + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK); + SetPageUptodate(page); + return 0; } /* - * Store a newly fetched page in fscache - * - PG_fscache must be set on the page + * Store a newly fetched page in fscache. We can be certain there's no page + * stored in the cache as yet otherwise we would've read it from there. */ -void __nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync) +void __nfs_readpage_to_fscache(struct inode *inode, struct page *page) { int ret; dfprintk(FSCACHE, - "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx)/%d)\n", - nfs_i_fscache(inode), page, page->index, page->flags, sync); + "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx))\n", + nfs_i_fscache(inode), page, page->index, page->flags); + + ret = fscache_fallback_write_page(inode, page, true); - ret = fscache_write_page(nfs_i_fscache(inode), page, - inode->i_size, GFP_KERNEL); dfprintk(FSCACHE, "NFS: readpage_to_fscache: p:%p(i:%lu f:%lx) ret %d\n", page, page->index, page->flags, ret); if (ret != 0) { - fscache_uncache_page(nfs_i_fscache(inode), page); - nfs_inc_fscache_stats(inode, - NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL); + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL); nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_UNCACHED); } else { - nfs_inc_fscache_stats(inode, - NFSIOS_FSCACHE_PAGES_WRITTEN_OK); + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_OK); } } diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index 6754c8607230..25a5c0f82392 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -8,51 +8,16 @@ #ifndef _NFS_FSCACHE_H #define _NFS_FSCACHE_H +#include <linux/swap.h> #include <linux/nfs_fs.h> #include <linux/nfs_mount.h> #include <linux/nfs4_mount.h> #include <linux/fscache.h> +#include <linux/iversion.h> #ifdef CONFIG_NFS_FSCACHE /* - * set of NFS FS-Cache objects that form a superblock key - */ -struct nfs_fscache_key { - struct rb_node node; - struct nfs_client *nfs_client; /* the server */ - - /* the elements of the unique key - as used by nfs_compare_super() and - * nfs_compare_mount_options() to distinguish superblocks */ - struct { - struct { - unsigned long s_flags; /* various flags - * (& NFS_MS_MASK) */ - } super; - - struct { - struct nfs_fsid fsid; - int flags; - unsigned int rsize; /* read size */ - unsigned int wsize; /* write size */ - unsigned int acregmin; /* attr cache timeouts */ - unsigned int acregmax; - unsigned int acdirmin; - unsigned int acdirmax; - } nfs_server; - - struct { - rpc_authflavor_t au_flavor; - } rpc_auth; - - /* uniquifier - can be used if nfs_server.flags includes - * NFS_MOUNT_UNSHARED */ - u8 uniq_len; - char uniquifier[0]; - } key; -}; - -/* * Definition of the auxiliary data attached to NFS inode storage objects * within the cache. * @@ -70,84 +35,42 @@ struct nfs_fscache_inode_auxdata { }; /* - * fscache-index.c - */ -extern struct fscache_netfs nfs_fscache_netfs; -extern const struct fscache_cookie_def nfs_fscache_server_index_def; -extern const struct fscache_cookie_def nfs_fscache_super_index_def; -extern const struct fscache_cookie_def nfs_fscache_inode_object_def; - -extern int nfs_fscache_register(void); -extern void nfs_fscache_unregister(void); - -/* * fscache.c */ -extern void nfs_fscache_get_client_cookie(struct nfs_client *); -extern void nfs_fscache_release_client_cookie(struct nfs_client *); - -extern void nfs_fscache_get_super_cookie(struct super_block *, const char *, int); +extern int nfs_fscache_get_super_cookie(struct super_block *, const char *, int); extern void nfs_fscache_release_super_cookie(struct super_block *); extern void nfs_fscache_init_inode(struct inode *); extern void nfs_fscache_clear_inode(struct inode *); extern void nfs_fscache_open_file(struct inode *, struct file *); +extern void nfs_fscache_release_file(struct inode *, struct file *); -extern void __nfs_fscache_invalidate_page(struct page *, struct inode *); -extern int nfs_fscache_release_page(struct page *, gfp_t); +extern int __nfs_readpage_from_fscache(struct inode *, struct page *); +extern void __nfs_read_completion_to_fscache(struct nfs_pgio_header *hdr, + unsigned long bytes); +extern void __nfs_readpage_to_fscache(struct inode *, struct page *); -extern int __nfs_readpage_from_fscache(struct nfs_open_context *, - struct inode *, struct page *); -extern int __nfs_readpages_from_fscache(struct nfs_open_context *, - struct inode *, struct address_space *, - struct list_head *, unsigned *); -extern void __nfs_readpage_to_fscache(struct inode *, struct page *, int); - -/* - * wait for a page to complete writing to the cache - */ -static inline void nfs_fscache_wait_on_page_write(struct nfs_inode *nfsi, - struct page *page) -{ - if (PageFsCache(page)) - fscache_wait_on_page_write(nfsi->fscache, page); -} - -/* - * release the caching state associated with a page if undergoing complete page - * invalidation - */ -static inline void nfs_fscache_invalidate_page(struct page *page, - struct inode *inode) +static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { - if (PageFsCache(page)) - __nfs_fscache_invalidate_page(page, inode); + if (PageFsCache(page)) { + if (current_is_kswapd() || !(gfp & __GFP_FS)) + return false; + wait_on_page_fscache(page); + fscache_note_page_release(nfs_i_fscache(page->mapping->host)); + nfs_inc_fscache_stats(page->mapping->host, + NFSIOS_FSCACHE_PAGES_UNCACHED); + } + return true; } /* * Retrieve a page from an inode data storage object. */ -static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, +static inline int nfs_readpage_from_fscache(struct inode *inode, struct page *page) { if (NFS_I(inode)->fscache) - return __nfs_readpage_from_fscache(ctx, inode, page); - return -ENOBUFS; -} - -/* - * Retrieve a set of pages from an inode data storage object. - */ -static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - if (NFS_I(inode)->fscache) - return __nfs_readpages_from_fscache(ctx, inode, mapping, pages, - nr_pages); + return __nfs_readpage_from_fscache(inode, page); return -ENOBUFS; } @@ -156,27 +79,38 @@ static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, * in the cache. */ static inline void nfs_readpage_to_fscache(struct inode *inode, - struct page *page, - int sync) + struct page *page) { - if (PageFsCache(page)) - __nfs_readpage_to_fscache(inode, page, sync); + if (NFS_I(inode)->fscache) + __nfs_readpage_to_fscache(inode, page); } -/* - * Invalidate the contents of fscache for this inode. This will not sleep. - */ -static inline void nfs_fscache_invalidate(struct inode *inode) +static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, + struct nfs_inode *nfsi) { - fscache_invalidate(NFS_I(inode)->fscache); + memset(auxdata, 0, sizeof(*auxdata)); + auxdata->mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; + auxdata->mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; + auxdata->ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; + auxdata->ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; + + if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) + auxdata->change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); } /* - * Wait for an object to finish being invalidated. + * Invalidate the contents of fscache for this inode. This will not sleep. */ -static inline void nfs_fscache_wait_on_invalidate(struct inode *inode) +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) { - fscache_wait_on_invalidate(NFS_I(inode)->fscache); + struct nfs_fscache_inode_auxdata auxdata; + struct nfs_inode *nfsi = NFS_I(inode); + + if (nfsi->fscache) { + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_invalidate(nfsi->fscache, &auxdata, + i_size_read(&nfsi->vfs_inode), flags); + } } /* @@ -190,48 +124,28 @@ static inline const char *nfs_server_fscache_state(struct nfs_server *server) } #else /* CONFIG_NFS_FSCACHE */ -static inline int nfs_fscache_register(void) { return 0; } -static inline void nfs_fscache_unregister(void) {} - -static inline void nfs_fscache_get_client_cookie(struct nfs_client *clp) {} -static inline void nfs_fscache_release_client_cookie(struct nfs_client *clp) {} - static inline void nfs_fscache_release_super_cookie(struct super_block *sb) {} static inline void nfs_fscache_init_inode(struct inode *inode) {} static inline void nfs_fscache_clear_inode(struct inode *inode) {} static inline void nfs_fscache_open_file(struct inode *inode, struct file *filp) {} +static inline void nfs_fscache_release_file(struct inode *inode, struct file *file) {} static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { return 1; /* True: may release page */ } -static inline void nfs_fscache_invalidate_page(struct page *page, - struct inode *inode) {} -static inline void nfs_fscache_wait_on_page_write(struct nfs_inode *nfsi, - struct page *page) {} - -static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, +static inline int nfs_readpage_from_fscache(struct inode *inode, struct page *page) { return -ENOBUFS; } -static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - return -ENOBUFS; -} static inline void nfs_readpage_to_fscache(struct inode *inode, - struct page *page, int sync) {} + struct page *page) {} -static inline void nfs_fscache_invalidate(struct inode *inode) {} -static inline void nfs_fscache_wait_on_invalidate(struct inode *inode) {} +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) {} static inline const char *nfs_server_fscache_state(struct nfs_server *server) { diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index fda530d5e764..a918c3a834b6 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -209,7 +209,7 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) if (!nfs_has_xattr_cache(nfsi)) flags &= ~NFS_INO_INVALID_XATTR; if (flags & NFS_INO_INVALID_DATA) - nfs_fscache_invalidate(inode); + nfs_fscache_invalidate(inode, 0); flags &= ~(NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED); nfsi->cache_validity |= flags; @@ -1289,6 +1289,7 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map { int ret; + nfs_fscache_invalidate(inode, 0); if (mapping->nrpages != 0) { if (S_ISREG(inode->i_mode)) { ret = nfs_sync_mapping(mapping); @@ -1300,7 +1301,6 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map return ret; } nfs_inc_stats(inode, NFSIOS_DATAINVALIDATE); - nfs_fscache_wait_on_invalidate(inode); dfprintk(PAGECACHE, "NFS: (%s/%Lu) data cache invalidated\n", inode->i_sb->s_id, @@ -2374,10 +2374,6 @@ static int __init init_nfs_fs(void) if (err < 0) goto out9; - err = nfs_fscache_register(); - if (err < 0) - goto out8; - err = nfsiod_start(); if (err) goto out7; @@ -2429,8 +2425,6 @@ out5: out6: nfsiod_stop(); out7: - nfs_fscache_unregister(); -out8: unregister_pernet_subsys(&nfs_net_ops); out9: nfs_sysfs_exit(); @@ -2445,7 +2439,6 @@ static void __exit exit_nfs_fs(void) nfs_destroy_readpagecache(); nfs_destroy_inodecache(); nfs_destroy_nfspagecache(); - nfs_fscache_unregister(); unregister_pernet_subsys(&nfs_net_ops); rpc_proc_unregister(&init_net, "nfs"); unregister_nfs_fs(); diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index b3aee261801e..317ce27bdc4b 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -42,7 +42,6 @@ { BIT(NFS_INO_ACL_LRU_SET), "ACL_LRU_SET" }, \ { BIT(NFS_INO_INVALIDATING), "INVALIDATING" }, \ { BIT(NFS_INO_FSCACHE), "FSCACHE" }, \ - { BIT(NFS_INO_FSCACHE_LOCK), "FSCACHE_LOCK" }, \ { BIT(NFS_INO_LAYOUTCOMMIT), "NEED_LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTCOMMITTING), "LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTSTATS), "LAYOUTSTATS" }, \ diff --git a/fs/nfs/read.c b/fs/nfs/read.c index d11af2a9299c..eb00229c1a50 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -123,7 +123,7 @@ static void nfs_readpage_release(struct nfs_page *req, int error) struct address_space *mapping = page_file_mapping(page); if (PageUptodate(page)) - nfs_readpage_to_fscache(inode, page, 0); + nfs_readpage_to_fscache(inode, page); else if (!PageError(page) && !PagePrivate(page)) generic_error_remove_page(mapping, page); unlock_page(page); @@ -305,6 +305,12 @@ readpage_async_filler(void *data, struct page *page) aligned_len = min_t(unsigned int, ALIGN(len, rsize), PAGE_SIZE); + if (!IS_SYNC(page->mapping->host)) { + error = nfs_readpage_from_fscache(page->mapping->host, page); + if (error == 0) + goto out_unlock; + } + new = nfs_create_request(desc->ctx, page, 0, aligned_len); if (IS_ERR(new)) goto out_error; @@ -320,6 +326,7 @@ readpage_async_filler(void *data, struct page *page) return 0; out_error: error = PTR_ERR(new); +out_unlock: unlock_page(page); out: return error; @@ -366,12 +373,6 @@ int nfs_readpage(struct file *file, struct page *page) desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); xchg(&desc.ctx->error, 0); - if (!IS_SYNC(inode)) { - ret = nfs_readpage_from_fscache(desc.ctx, inode, page); - if (ret == 0) - goto out_wait; - } - nfs_pageio_init_read(&desc.pgio, inode, false, &nfs_async_read_completion_ops); @@ -381,7 +382,6 @@ int nfs_readpage(struct file *file, struct page *page) nfs_pageio_complete_read(&desc.pgio); ret = desc.pgio.pg_error < 0 ? desc.pgio.pg_error : 0; -out_wait: if (!ret) { ret = wait_on_page_locked_killable(page); if (!PageUptodate(page) && !ret) @@ -419,14 +419,6 @@ int nfs_readpages(struct file *file, struct address_space *mapping, } else desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); - /* attempt to read as many of the pages as possible from the cache - * - this returns -ENOBUFS immediately if the cookie is negative - */ - ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping, - pages, &nr_pages); - if (ret == 0) - goto read_complete; /* all pages were read */ - nfs_pageio_init_read(&desc.pgio, inode, false, &nfs_async_read_completion_ops); @@ -434,7 +426,6 @@ int nfs_readpages(struct file *file, struct address_space *mapping, nfs_pageio_complete_read(&desc.pgio); -read_complete: put_nfs_open_context(desc.ctx); out: trace_nfs_aop_readahead_done(inode, nr_pages, ret); diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 3aced401735c..6ab5eeb000dc 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -1204,42 +1204,42 @@ static int nfs_compare_super(struct super_block *sb, struct fs_context *fc) } #ifdef CONFIG_NFS_FSCACHE -static void nfs_get_cache_cookie(struct super_block *sb, - struct nfs_fs_context *ctx) +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) { struct nfs_server *nfss = NFS_SB(sb); char *uniq = NULL; int ulen = 0; - nfss->fscache_key = NULL; nfss->fscache = NULL; if (!ctx) - return; + return 0; if (ctx->clone_data.sb) { struct nfs_server *mnt_s = NFS_SB(ctx->clone_data.sb); if (!(mnt_s->options & NFS_OPTION_FSCACHE)) - return; - if (mnt_s->fscache_key) { - uniq = mnt_s->fscache_key->key.uniquifier; - ulen = mnt_s->fscache_key->key.uniq_len; + return 0; + if (mnt_s->fscache_uniq) { + uniq = mnt_s->fscache_uniq; + ulen = strlen(uniq); } } else { if (!(ctx->options & NFS_OPTION_FSCACHE)) - return; + return 0; if (ctx->fscache_uniq) { uniq = ctx->fscache_uniq; ulen = strlen(ctx->fscache_uniq); } } - nfs_fscache_get_super_cookie(sb, uniq, ulen); + return nfs_fscache_get_super_cookie(sb, uniq, ulen); } #else -static void nfs_get_cache_cookie(struct super_block *sb, - struct nfs_fs_context *ctx) +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) { + return 0; } #endif @@ -1299,7 +1299,9 @@ int nfs_get_tree_common(struct fs_context *fc) s->s_blocksize_bits = bsize; s->s_blocksize = 1U << bsize; } - nfs_get_cache_cookie(s, ctx); + error = nfs_get_cache_cookie(s, ctx); + if (error < 0) + goto error_splat_super; } error = nfs_get_root(s, fc); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 9b7619ce17a7..987a187bd39a 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -294,6 +294,7 @@ static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int c nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); out: spin_unlock(&inode->i_lock); + nfs_fscache_invalidate(inode, 0); } /* A writeback failed: mark the page as bad, and invalidate the page cache */ @@ -2125,8 +2126,11 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage, if (PagePrivate(page)) return -EBUSY; - if (!nfs_fscache_release_page(page, GFP_KERNEL)) - return -EBUSY; + if (PageFsCache(page)) { + if (mode == MIGRATE_ASYNC) + return -EBUSY; + wait_on_page_fscache(page); + } return migrate_page(mapping, newpage, page, mode); } |