diff options
Diffstat (limited to 'fs/btrfs/locking.c')
| -rw-r--r-- | fs/btrfs/locking.c | 157 |
1 files changed, 117 insertions, 40 deletions
diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index 82b84e4daad1..2f6c3c7851ed 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -12,10 +12,82 @@ #include "extent_io.h" #include "locking.h" -static void btrfs_assert_tree_read_locked(struct extent_buffer *eb); +#ifdef CONFIG_BTRFS_DEBUG +static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) +{ + WARN_ON(atomic_read(&eb->spinning_writers)); + atomic_inc(&eb->spinning_writers); +} + +static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) +{ + WARN_ON(atomic_read(&eb->spinning_writers) != 1); + atomic_dec(&eb->spinning_writers); +} + +static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) +{ + WARN_ON(atomic_read(&eb->spinning_writers)); +} + +static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) +{ + atomic_inc(&eb->spinning_readers); +} + +static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) +{ + WARN_ON(atomic_read(&eb->spinning_readers) == 0); + atomic_dec(&eb->spinning_readers); +} + +static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) +{ + atomic_inc(&eb->read_locks); +} + +static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) +{ + atomic_dec(&eb->read_locks); +} + +static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) +{ + BUG_ON(!atomic_read(&eb->read_locks)); +} + +static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) +{ + atomic_inc(&eb->write_locks); +} + +static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) +{ + atomic_dec(&eb->write_locks); +} + +void btrfs_assert_tree_locked(struct extent_buffer *eb) +{ + BUG_ON(!atomic_read(&eb->write_locks)); +} + +#else +static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { } +static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { } +static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { } +static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { } +static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { } +static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { } +static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { } +static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { } +void btrfs_assert_tree_locked(struct extent_buffer *eb) { } +static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { } +static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { } +#endif void btrfs_set_lock_blocking_read(struct extent_buffer *eb) { + trace_btrfs_set_lock_blocking_read(eb); /* * No lock is required. The lock owner may change if we have a read * lock, but it won't change to or away from us. If we have the write @@ -25,13 +97,13 @@ void btrfs_set_lock_blocking_read(struct extent_buffer *eb) return; btrfs_assert_tree_read_locked(eb); atomic_inc(&eb->blocking_readers); - WARN_ON(atomic_read(&eb->spinning_readers) == 0); - atomic_dec(&eb->spinning_readers); + btrfs_assert_spinning_readers_put(eb); read_unlock(&eb->lock); } void btrfs_set_lock_blocking_write(struct extent_buffer *eb) { + trace_btrfs_set_lock_blocking_write(eb); /* * No lock is required. The lock owner may change if we have a read * lock, but it won't change to or away from us. If we have the write @@ -40,8 +112,7 @@ void btrfs_set_lock_blocking_write(struct extent_buffer *eb) if (eb->lock_nested && current->pid == eb->lock_owner) return; if (atomic_read(&eb->blocking_writers) == 0) { - WARN_ON(atomic_read(&eb->spinning_writers) != 1); - atomic_dec(&eb->spinning_writers); + btrfs_assert_spinning_writers_put(eb); btrfs_assert_tree_locked(eb); atomic_inc(&eb->blocking_writers); write_unlock(&eb->lock); @@ -50,6 +121,7 @@ void btrfs_set_lock_blocking_write(struct extent_buffer *eb) void btrfs_clear_lock_blocking_read(struct extent_buffer *eb) { + trace_btrfs_clear_lock_blocking_read(eb); /* * No lock is required. The lock owner may change if we have a read * lock, but it won't change to or away from us. If we have the write @@ -59,7 +131,7 @@ void btrfs_clear_lock_blocking_read(struct extent_buffer *eb) return; BUG_ON(atomic_read(&eb->blocking_readers) == 0); read_lock(&eb->lock); - atomic_inc(&eb->spinning_readers); + btrfs_assert_spinning_readers_get(eb); /* atomic_dec_and_test implies a barrier */ if (atomic_dec_and_test(&eb->blocking_readers)) cond_wake_up_nomb(&eb->read_lock_wq); @@ -67,6 +139,7 @@ void btrfs_clear_lock_blocking_read(struct extent_buffer *eb) void btrfs_clear_lock_blocking_write(struct extent_buffer *eb) { + trace_btrfs_clear_lock_blocking_write(eb); /* * no lock is required. The lock owner may change if * we have a read lock, but it won't change to or away @@ -77,8 +150,7 @@ void btrfs_clear_lock_blocking_write(struct extent_buffer *eb) return; BUG_ON(atomic_read(&eb->blocking_writers) != 1); write_lock(&eb->lock); - WARN_ON(atomic_read(&eb->spinning_writers)); - atomic_inc(&eb->spinning_writers); + btrfs_assert_spinning_writers_get(eb); /* atomic_dec_and_test implies a barrier */ if (atomic_dec_and_test(&eb->blocking_writers)) cond_wake_up_nomb(&eb->write_lock_wq); @@ -90,6 +162,10 @@ void btrfs_clear_lock_blocking_write(struct extent_buffer *eb) */ void btrfs_tree_read_lock(struct extent_buffer *eb) { + u64 start_ns = 0; + + if (trace_btrfs_tree_read_lock_enabled()) + start_ns = ktime_get_ns(); again: BUG_ON(!atomic_read(&eb->blocking_writers) && current->pid == eb->lock_owner); @@ -104,8 +180,9 @@ again: * called on a partly (write-)locked tree. */ BUG_ON(eb->lock_nested); - eb->lock_nested = 1; + eb->lock_nested = true; read_unlock(&eb->lock); + trace_btrfs_tree_read_lock(eb, start_ns); return; } if (atomic_read(&eb->blocking_writers)) { @@ -114,8 +191,9 @@ again: atomic_read(&eb->blocking_writers) == 0); goto again; } - atomic_inc(&eb->read_locks); - atomic_inc(&eb->spinning_readers); + btrfs_assert_tree_read_locks_get(eb); + btrfs_assert_spinning_readers_get(eb); + trace_btrfs_tree_read_lock(eb, start_ns); } /* @@ -133,8 +211,9 @@ int btrfs_tree_read_lock_atomic(struct extent_buffer *eb) read_unlock(&eb->lock); return 0; } - atomic_inc(&eb->read_locks); - atomic_inc(&eb->spinning_readers); + btrfs_assert_tree_read_locks_get(eb); + btrfs_assert_spinning_readers_get(eb); + trace_btrfs_tree_read_lock_atomic(eb); return 1; } @@ -154,8 +233,9 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb) read_unlock(&eb->lock); return 0; } - atomic_inc(&eb->read_locks); - atomic_inc(&eb->spinning_readers); + btrfs_assert_tree_read_locks_get(eb); + btrfs_assert_spinning_readers_get(eb); + trace_btrfs_try_tree_read_lock(eb); return 1; } @@ -175,9 +255,10 @@ int btrfs_try_tree_write_lock(struct extent_buffer *eb) write_unlock(&eb->lock); return 0; } - atomic_inc(&eb->write_locks); - atomic_inc(&eb->spinning_writers); + btrfs_assert_tree_write_locks_get(eb); + btrfs_assert_spinning_writers_get(eb); eb->lock_owner = current->pid; + trace_btrfs_try_tree_write_lock(eb); return 1; } @@ -186,6 +267,7 @@ int btrfs_try_tree_write_lock(struct extent_buffer *eb) */ void btrfs_tree_read_unlock(struct extent_buffer *eb) { + trace_btrfs_tree_read_unlock(eb); /* * if we're nested, we have the write lock. No new locking * is needed as long as we are the lock owner. @@ -193,13 +275,12 @@ void btrfs_tree_read_unlock(struct extent_buffer *eb) * field only matters to the lock owner. */ if (eb->lock_nested && current->pid == eb->lock_owner) { - eb->lock_nested = 0; + eb->lock_nested = false; return; } btrfs_assert_tree_read_locked(eb); - WARN_ON(atomic_read(&eb->spinning_readers) == 0); - atomic_dec(&eb->spinning_readers); - atomic_dec(&eb->read_locks); + btrfs_assert_spinning_readers_put(eb); + btrfs_assert_tree_read_locks_put(eb); read_unlock(&eb->lock); } @@ -208,6 +289,7 @@ void btrfs_tree_read_unlock(struct extent_buffer *eb) */ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) { + trace_btrfs_tree_read_unlock_blocking(eb); /* * if we're nested, we have the write lock. No new locking * is needed as long as we are the lock owner. @@ -215,7 +297,7 @@ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) * field only matters to the lock owner. */ if (eb->lock_nested && current->pid == eb->lock_owner) { - eb->lock_nested = 0; + eb->lock_nested = false; return; } btrfs_assert_tree_read_locked(eb); @@ -223,7 +305,7 @@ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) /* atomic_dec_and_test implies a barrier */ if (atomic_dec_and_test(&eb->blocking_readers)) cond_wake_up_nomb(&eb->read_lock_wq); - atomic_dec(&eb->read_locks); + btrfs_assert_tree_read_locks_put(eb); } /* @@ -232,6 +314,11 @@ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) */ void btrfs_tree_lock(struct extent_buffer *eb) { + u64 start_ns = 0; + + if (trace_btrfs_tree_lock_enabled()) + start_ns = ktime_get_ns(); + WARN_ON(eb->lock_owner == current->pid); again: wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0); @@ -242,10 +329,10 @@ again: write_unlock(&eb->lock); goto again; } - WARN_ON(atomic_read(&eb->spinning_writers)); - atomic_inc(&eb->spinning_writers); - atomic_inc(&eb->write_locks); + btrfs_assert_spinning_writers_get(eb); + btrfs_assert_tree_write_locks_get(eb); eb->lock_owner = current->pid; + trace_btrfs_tree_lock(eb, start_ns); } /* @@ -258,28 +345,18 @@ void btrfs_tree_unlock(struct extent_buffer *eb) BUG_ON(blockers > 1); btrfs_assert_tree_locked(eb); + trace_btrfs_tree_unlock(eb); eb->lock_owner = 0; - atomic_dec(&eb->write_locks); + btrfs_assert_tree_write_locks_put(eb); if (blockers) { - WARN_ON(atomic_read(&eb->spinning_writers)); + btrfs_assert_no_spinning_writers(eb); atomic_dec(&eb->blocking_writers); /* Use the lighter barrier after atomic */ smp_mb__after_atomic(); cond_wake_up_nomb(&eb->write_lock_wq); } else { - WARN_ON(atomic_read(&eb->spinning_writers) != 1); - atomic_dec(&eb->spinning_writers); + btrfs_assert_spinning_writers_put(eb); write_unlock(&eb->lock); } } - -void btrfs_assert_tree_locked(struct extent_buffer *eb) -{ - BUG_ON(!atomic_read(&eb->write_locks)); -} - -static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) -{ - BUG_ON(!atomic_read(&eb->read_locks)); -} |