From b23fb0a60379a95e10c671f646b259ea2558421e Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Fri, 7 Jan 2011 17:49:35 +1100 Subject: fs: scale inode alias list Add a new lock, dcache_inode_lock, to protect the inode's i_dentry list from concurrent modification. d_alias is also protected by d_lock. Signed-off-by: Nick Piggin --- fs/dcache.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 8 deletions(-) (limited to 'fs/dcache.c') diff --git a/fs/dcache.c b/fs/dcache.c index a661247a20d5..de38680ee0ed 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -37,6 +37,8 @@ /* * Usage: + * dcache_inode_lock protects: + * - i_dentry, d_alias, d_inode * dcache_hash_lock protects: * - the dcache hash table, s_anon lists * dcache_lru_lock protects: @@ -49,12 +51,14 @@ * - d_unhashed() * - d_parent and d_subdirs * - childrens' d_child and d_parent + * - d_alias, d_inode * * Ordering: * dcache_lock - * dentry->d_lock - * dcache_lru_lock - * dcache_hash_lock + * dcache_inode_lock + * dentry->d_lock + * dcache_lru_lock + * dcache_hash_lock * * If there is an ancestor relationship: * dentry->d_parent->...->d_parent->d_lock @@ -70,11 +74,13 @@ int sysctl_vfs_cache_pressure __read_mostly = 100; EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); +__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock); static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock); static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock); __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); +EXPORT_SYMBOL(dcache_inode_lock); EXPORT_SYMBOL(dcache_lock); static struct kmem_cache *dentry_cache __read_mostly; @@ -154,6 +160,7 @@ static void d_free(struct dentry *dentry) */ static void dentry_iput(struct dentry * dentry) __releases(dentry->d_lock) + __releases(dcache_inode_lock) __releases(dcache_lock) { struct inode *inode = dentry->d_inode; @@ -161,6 +168,7 @@ static void dentry_iput(struct dentry * dentry) dentry->d_inode = NULL; list_del_init(&dentry->d_alias); spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); if (!inode->i_nlink) fsnotify_inoderemove(inode); @@ -170,6 +178,7 @@ static void dentry_iput(struct dentry * dentry) iput(inode); } else { spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); } } @@ -231,6 +240,7 @@ static void dentry_lru_move_tail(struct dentry *dentry) static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent) __releases(dentry->d_lock) __releases(parent->d_lock) + __releases(dcache_inode_lock) __releases(dcache_lock) { list_del(&dentry->d_u.d_child); @@ -332,13 +342,18 @@ repeat: * want to reduce dcache_lock anyway so this will * get improved. */ +drop1: spin_unlock(&dentry->d_lock); goto repeat; } - if (parent && !spin_trylock(&parent->d_lock)) { - spin_unlock(&dentry->d_lock); + if (!spin_trylock(&dcache_inode_lock)) { +drop2: spin_unlock(&dcache_lock); - goto repeat; + goto drop1; + } + if (parent && !spin_trylock(&parent->d_lock)) { + spin_unlock(&dcache_inode_lock); + goto drop2; } } dentry->d_count--; @@ -369,6 +384,7 @@ repeat: spin_unlock(&dentry->d_lock); if (parent) spin_unlock(&parent->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); return; @@ -558,7 +574,9 @@ struct dentry *d_find_alias(struct inode *inode) if (!list_empty(&inode->i_dentry)) { spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); de = __d_find_alias(inode, 0); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); } return de; @@ -574,18 +592,21 @@ void d_prune_aliases(struct inode *inode) struct dentry *dentry; restart: spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); list_for_each_entry(dentry, &inode->i_dentry, d_alias) { spin_lock(&dentry->d_lock); if (!dentry->d_count) { __dget_locked_dlock(dentry); __d_drop(dentry); spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); dput(dentry); goto restart; } spin_unlock(&dentry->d_lock); } + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); } EXPORT_SYMBOL(d_prune_aliases); @@ -601,6 +622,7 @@ EXPORT_SYMBOL(d_prune_aliases); static void prune_one_dentry(struct dentry *dentry, struct dentry *parent) __releases(dentry->d_lock) __releases(parent->d_lock) + __releases(dcache_inode_lock) __releases(dcache_lock) { __d_drop(dentry); @@ -612,6 +634,7 @@ static void prune_one_dentry(struct dentry *dentry, struct dentry *parent) */ while (dentry) { spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); again: spin_lock(&dentry->d_lock); if (IS_ROOT(dentry)) @@ -627,6 +650,7 @@ again: if (parent) spin_unlock(&parent->d_lock); spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); return; } @@ -676,8 +700,9 @@ relock: spin_unlock(&dcache_lru_lock); prune_one_dentry(dentry, parent); - /* dcache_lock and dentry->d_lock dropped */ + /* dcache_lock, dcache_inode_lock and dentry->d_lock dropped */ spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); spin_lock(&dcache_lru_lock); } } @@ -699,6 +724,7 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags) int cnt = *count; spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); relock: spin_lock(&dcache_lru_lock); while (!list_empty(&sb->s_dentry_lru)) { @@ -737,8 +763,8 @@ relock: if (!list_empty(&referenced)) list_splice(&referenced, &sb->s_dentry_lru); spin_unlock(&dcache_lru_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); - } /** @@ -832,12 +858,14 @@ void shrink_dcache_sb(struct super_block *sb) LIST_HEAD(tmp); spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); spin_lock(&dcache_lru_lock); while (!list_empty(&sb->s_dentry_lru)) { list_splice_init(&sb->s_dentry_lru, &tmp); shrink_dentry_list(&tmp); } spin_unlock(&dcache_lru_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); } EXPORT_SYMBOL(shrink_dcache_sb); @@ -1255,9 +1283,11 @@ EXPORT_SYMBOL(d_alloc_name); /* the caller must hold dcache_lock */ static void __d_instantiate(struct dentry *dentry, struct inode *inode) { + spin_lock(&dentry->d_lock); if (inode) list_add(&dentry->d_alias, &inode->i_dentry); dentry->d_inode = inode; + spin_unlock(&dentry->d_lock); fsnotify_d_instantiate(dentry, inode); } @@ -1280,7 +1310,9 @@ void d_instantiate(struct dentry *entry, struct inode * inode) { BUG_ON(!list_empty(&entry->d_alias)); spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); __d_instantiate(entry, inode); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); security_d_instantiate(entry, inode); } @@ -1341,7 +1373,9 @@ struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode) BUG_ON(!list_empty(&entry->d_alias)); spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); result = __d_instantiate_unique(entry, inode); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); if (!result) { @@ -1432,8 +1466,10 @@ struct dentry *d_obtain_alias(struct inode *inode) tmp->d_parent = tmp; /* make sure dput doesn't croak */ spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); res = __d_find_alias(inode, 0); if (res) { + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); dput(tmp); goto out_iput; @@ -1450,6 +1486,7 @@ struct dentry *d_obtain_alias(struct inode *inode) hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon); spin_unlock(&dcache_hash_lock); spin_unlock(&tmp->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); return tmp; @@ -1482,9 +1519,11 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) if (inode && S_ISDIR(inode->i_mode)) { spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); new = __d_find_alias(inode, 1); if (new) { BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED)); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); security_d_instantiate(new, inode); d_move(new, dentry); @@ -1492,6 +1531,7 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) } else { /* already taking dcache_lock, so d_add() by hand */ __d_instantiate(dentry, inode); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); security_d_instantiate(dentry, inode); d_rehash(dentry); @@ -1566,8 +1606,10 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, * already has a dentry. */ spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) { __d_instantiate(found, inode); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); security_d_instantiate(found, inode); return found; @@ -1579,6 +1621,7 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, */ new = list_entry(inode->i_dentry.next, struct dentry, d_alias); dget_locked(new); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); security_d_instantiate(found, inode); d_move(new, found); @@ -1797,6 +1840,7 @@ void d_delete(struct dentry * dentry) * Are we the only user? */ spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); spin_lock(&dentry->d_lock); isdir = S_ISDIR(dentry->d_inode->i_mode); if (dentry->d_count == 1) { @@ -1810,6 +1854,7 @@ void d_delete(struct dentry * dentry) __d_drop(dentry); spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); fsnotify_nameremove(dentry, isdir); @@ -2067,6 +2112,7 @@ struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) */ static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias) __releases(dcache_lock) + __releases(dcache_inode_lock) { struct mutex *m1 = NULL, *m2 = NULL; struct dentry *ret; @@ -2092,6 +2138,7 @@ out_unalias: d_move_locked(alias, dentry); ret = alias; out_err: + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); if (m2) mutex_unlock(m2); @@ -2153,6 +2200,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) BUG_ON(!d_unhashed(dentry)); spin_lock(&dcache_lock); + spin_lock(&dcache_inode_lock); if (!inode) { actual = dentry; @@ -2196,6 +2244,7 @@ found: _d_rehash(actual); spin_unlock(&dcache_hash_lock); spin_unlock(&actual->d_lock); + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); out_nolock: if (actual == dentry) { @@ -2207,6 +2256,7 @@ out_nolock: return actual; shouldnt_be_hashed: + spin_unlock(&dcache_inode_lock); spin_unlock(&dcache_lock); BUG(); } -- cgit v1.2.3-59-g8ed1b