From e7186af7fb2609584a8bfb3da3c6ae09da5a5224 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 13 Jul 2023 09:26:05 -0400 Subject: tracing: Add back FORTIFY_SOURCE logic to kernel_stack event structure For backward compatibility, older tooling expects to see the kernel_stack event with a "caller" field that is a fixed size array of 8 addresses. The code now supports more than 8 with an added "size" field that states the real number of entries. But the "caller" field still just looks like a fixed size to user space. Since the tracing macros that create the user space format files also creates the structures that those files represent, the kernel_stack event structure had its "caller" field a fixed size of 8, but in reality, when it is allocated on the ring buffer, it can hold more if the stack trace is bigger that 8 functions. The copying of these entries was simply done with a memcpy(): size = nr_entries * sizeof(unsigned long); memcpy(entry->caller, fstack->calls, size); The FORTIFY_SOURCE logic noticed at runtime that when the nr_entries was larger than 8, that the memcpy() was writing more than what the structure stated it can hold and it complained about it. This is because the FORTIFY_SOURCE code is unaware that the amount allocated is actually enough to hold the size. It does not expect that a fixed size field will hold more than the fixed size. This was originally solved by hiding the caller assignment with some pointer arithmetic. ptr = ring_buffer_data(); entry = ptr; ptr += offsetof(typeof(*entry), caller); memcpy(ptr, fstack->calls, size); But it is considered bad form to hide from kernel hardening. Instead, make it work nicely with FORTIFY_SOURCE by adding a new __stack_array() macro that is specific for this one special use case. The macro will take 4 arguments: type, item, len, field (whereas the __array() macro takes just the first three). This macro will act just like the __array() macro when creating the code to deal with the format file that is exposed to user space. But for the kernel, it will turn the caller field into: type item[] __counted_by(field); or for this instance: unsigned long caller[] __counted_by(size); Now the kernel code can expose the assignment of the caller to the FORTIFY_SOURCE and everyone is happy! Link: https://lore.kernel.org/linux-trace-kernel/20230712105235.5fc441aa@gandalf.local.home/ Link: https://lore.kernel.org/linux-trace-kernel/20230713092605.2ddb9788@rorschach.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Sven Schnelle Suggested-by: Kees Cook Signed-off-by: Steven Rostedt (Google) Reviewed-by: Kees Cook --- kernel/trace/trace.c | 25 ++++--------------------- kernel/trace/trace.h | 10 ++++++++++ kernel/trace/trace_entries.h | 2 +- kernel/trace/trace_export.c | 9 +++++++++ 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index b8870078ef58..f6ed6b79d91f 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3119,7 +3119,6 @@ static void __ftrace_trace_stack(struct trace_buffer *buffer, struct ftrace_stack *fstack; struct stack_entry *entry; int stackidx; - void *ptr; /* * Add one, for this function and the call to save_stack_trace() @@ -3157,32 +3156,16 @@ static void __ftrace_trace_stack(struct trace_buffer *buffer, nr_entries = stack_trace_save(fstack->calls, size, skip); } - size = nr_entries * sizeof(unsigned long); event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, - (sizeof(*entry) - sizeof(entry->caller)) + size, + struct_size(entry, caller, nr_entries), trace_ctx); if (!event) goto out; - ptr = ring_buffer_event_data(event); - entry = ptr; - - /* - * For backward compatibility reasons, the entry->caller is an - * array of 8 slots to store the stack. This is also exported - * to user space. The amount allocated on the ring buffer actually - * holds enough for the stack specified by nr_entries. This will - * go into the location of entry->caller. Due to string fortifiers - * checking the size of the destination of memcpy() it triggers - * when it detects that size is greater than 8. To hide this from - * the fortifiers, we use "ptr" and pointer arithmetic to assign caller. - * - * The below is really just: - * memcpy(&entry->caller, fstack->calls, size); - */ - ptr += offsetof(typeof(*entry), caller); - memcpy(ptr, fstack->calls, size); + entry = ring_buffer_event_data(event); entry->size = nr_entries; + memcpy(&entry->caller, fstack->calls, + flex_array_size(entry, caller, nr_entries)); if (!call_filter_check_discard(call, entry, buffer, event)) __buffer_unlock_commit(buffer, event); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index e1edc2197fc8..ba7ababb8308 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -77,6 +77,16 @@ enum trace_type { #undef __array #define __array(type, item, size) type item[size]; +/* + * For backward compatibility, older user space expects to see the + * kernel_stack event with a fixed size caller field. But today the fix + * size is ignored by the kernel, and the real structure is dynamic. + * Expose to user space: "unsigned long caller[8];" but the real structure + * will be "unsigned long caller[] __counted_by(size)" + */ +#undef __stack_array +#define __stack_array(type, item, size, field) type item[] __counted_by(field); + #undef __array_desc #define __array_desc(type, container, item, size) diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h index 340b2fa98218..c47422b20908 100644 --- a/kernel/trace/trace_entries.h +++ b/kernel/trace/trace_entries.h @@ -190,7 +190,7 @@ FTRACE_ENTRY(kernel_stack, stack_entry, F_STRUCT( __field( int, size ) - __array( unsigned long, caller, FTRACE_STACK_ENTRIES ) + __stack_array( unsigned long, caller, FTRACE_STACK_ENTRIES, size) ), F_printk("\t=> %ps\n\t=> %ps\n\t=> %ps\n" diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index 58f3946081e2..1698fc22afa0 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c @@ -51,6 +51,9 @@ static int ftrace_event_register(struct trace_event_call *call, #undef __array #define __array(type, item, size) type item[size]; +#undef __stack_array +#define __stack_array(type, item, size, field) __array(type, item, size) + #undef __array_desc #define __array_desc(type, container, item, size) type item[size]; @@ -114,6 +117,9 @@ static void __always_unused ____ftrace_check_##name(void) \ is_signed_type(_type), .filter_type = FILTER_OTHER, \ .len = _len }, +#undef __stack_array +#define __stack_array(_type, _item, _len, _field) __array(_type, _item, _len) + #undef __array_desc #define __array_desc(_type, _container, _item, _len) __array(_type, _item, _len) @@ -149,6 +155,9 @@ static struct trace_event_fields ftrace_event_fields_##name[] = { \ #undef __array #define __array(type, item, len) +#undef __stack_array +#define __stack_array(type, item, len, field) + #undef __array_desc #define __array_desc(type, container, item, len) -- cgit v1.2.3-59-g8ed1b From 00a8478f8f5c0201004240746dd74d69d86fc0c4 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 14 Jul 2023 17:43:34 +0200 Subject: ring_buffer: Use try_cmpxchg instead of cmpxchg Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in ring_buffer.c. x86 CMPXCHG instruction returns success in ZF flag, so this change saves a compare after cmpxchg (and related move instruction in front of cmpxchg). No functional change intended. Link: https://lore.kernel.org/linux-trace-kernel/20230714154418.8884-1-ubizjak@gmail.com Cc: Steven Rostedt Cc: Masami Hiramatsu Signed-off-by: Uros Bizjak Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 52dea5dd5362..78502d4c7214 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -692,10 +692,7 @@ static void rb_time_set(rb_time_t *t, u64 val) static inline bool rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set) { - unsigned long ret; - - ret = local_cmpxchg(l, expect, set); - return ret == expect; + return local_try_cmpxchg(l, &expect, set); } static bool rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set) @@ -752,9 +749,7 @@ static void rb_time_set(rb_time_t *t, u64 val) static bool rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set) { - u64 val; - val = local64_cmpxchg(&t->time, expect, set); - return val == expect; + return local64_try_cmpxchg(&t->time, &expect, set); } #endif @@ -1494,14 +1489,11 @@ static bool rb_head_page_replace(struct buffer_page *old, { unsigned long *ptr = (unsigned long *)&old->list.prev->next; unsigned long val; - unsigned long ret; val = *ptr & ~RB_FLAG_MASK; val |= RB_PAGE_HEAD; - ret = cmpxchg(ptr, val, (unsigned long)&new->list); - - return ret == val; + return try_cmpxchg(ptr, &val, (unsigned long)&new->list); } /* @@ -3003,7 +2995,6 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, { unsigned long new_index, old_index; struct buffer_page *bpage; - unsigned long index; unsigned long addr; u64 write_stamp; u64 delta; @@ -3060,8 +3051,9 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, */ old_index += write_mask; new_index += write_mask; - index = local_cmpxchg(&bpage->write, old_index, new_index); - if (index == old_index) { + + /* caution: old_index gets updated on cmpxchg failure */ + if (local_try_cmpxchg(&bpage->write, &old_index, new_index)) { /* update counters */ local_sub(event_length, &cpu_buffer->entries_bytes); return true; -- cgit v1.2.3-59-g8ed1b From 9182b519b8c9733e921165f93e63cfffc18d1e0a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Sat, 15 Jul 2023 10:12:14 -0400 Subject: tracing: Remove unnecessary copying of tr->current_trace The iterator allocated a descriptor to copy the current_trace. This was done with the assumption that the function pointers might change. But this was a false assuption, as it does not change. There's no reason to make a copy of the current_trace and just use the pointer it points to. This removes needing to manage freeing the descriptor. Worse yet, there's locations that the iterator is used but does make a copy and just uses the pointer. This could cause the actual pointer to the trace descriptor to be freed and not the allocated copy. This is more of a clean up than a fix. Link: https://lkml.kernel.org/r/20230715141348.135792275@goodmis.org Cc: Mark Rutland Cc: Andrew Morton Fixes: d7350c3f45694 ("tracing/core: make the read callbacks reentrants") Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace.c | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index f6ed6b79d91f..5e665c356df7 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4189,15 +4189,9 @@ static void *s_start(struct seq_file *m, loff_t *pos) loff_t l = 0; int cpu; - /* - * copy the tracer to avoid using a global lock all around. - * iter->trace is a copy of current_trace, the pointer to the - * name may be used instead of a strcmp(), as iter->trace->name - * will point to the same string as current_trace->name. - */ mutex_lock(&trace_types_lock); - if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) - *iter->trace = *tr->current_trace; + if (unlikely(tr->current_trace != iter->trace)) + iter->trace = tr->current_trace; mutex_unlock(&trace_types_lock); #ifdef CONFIG_TRACER_MAX_TRACE @@ -4846,16 +4840,8 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) iter->fmt = NULL; iter->fmt_size = 0; - /* - * We make a copy of the current tracer to avoid concurrent - * changes on it while we are reading. - */ mutex_lock(&trace_types_lock); - iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); - if (!iter->trace) - goto fail; - - *iter->trace = *tr->current_trace; + iter->trace = tr->current_trace; if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) goto fail; @@ -4920,7 +4906,6 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) fail: mutex_unlock(&trace_types_lock); - kfree(iter->trace); kfree(iter->temp); kfree(iter->buffer_iter); release: @@ -5005,7 +4990,6 @@ static int tracing_release(struct inode *inode, struct file *file) free_cpumask_var(iter->started); kfree(iter->fmt); kfree(iter->temp); - kfree(iter->trace); kfree(iter->buffer_iter); seq_release_private(inode, file); -- cgit v1.2.3-59-g8ed1b From 6bba92881de9dde4c07cbb24b5691eb6eb9495f2 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Sat, 15 Jul 2023 10:12:15 -0400 Subject: tracing: Add free_trace_iter_content() helper function As the trace iterator is created and used by various interfaces, the clean up of it needs to be consistent. Create a free_trace_iter_content() helper function that frees the content of the iterator and use that to clean it up in all places that it is used. Link: https://lkml.kernel.org/r/20230715141348.341887497@goodmis.org Cc: Mark Rutland Cc: Andrew Morton Signed-off-by: Steven Rostedt (Google) Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 5e665c356df7..a3c4f9826872 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4799,6 +4799,25 @@ static const struct seq_operations tracer_seq_ops = { .show = s_show, }; +/* + * Note, as iter itself can be allocated and freed in different + * ways, this function is only used to free its content, and not + * the iterator itself. The only requirement to all the allocations + * is that it must zero all fields (kzalloc), as freeing works with + * ethier allocated content or NULL. + */ +static void free_trace_iter_content(struct trace_iterator *iter) +{ + /* The fmt is either NULL, allocated or points to static_fmt_buf */ + if (iter->fmt != static_fmt_buf) + kfree(iter->fmt); + + kfree(iter->temp); + kfree(iter->buffer_iter); + mutex_destroy(&iter->mutex); + free_cpumask_var(iter->started); +} + static struct trace_iterator * __tracing_open(struct inode *inode, struct file *file, bool snapshot) { @@ -4906,8 +4925,7 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) fail: mutex_unlock(&trace_types_lock); - kfree(iter->temp); - kfree(iter->buffer_iter); + free_trace_iter_content(iter); release: seq_release_private(inode, file); return ERR_PTR(-ENOMEM); @@ -4986,11 +5004,7 @@ static int tracing_release(struct inode *inode, struct file *file) mutex_unlock(&trace_types_lock); - mutex_destroy(&iter->mutex); - free_cpumask_var(iter->started); - kfree(iter->fmt); - kfree(iter->temp); - kfree(iter->buffer_iter); + free_trace_iter_content(iter); seq_release_private(inode, file); return 0; @@ -6747,10 +6761,7 @@ static int tracing_release_pipe(struct inode *inode, struct file *file) mutex_unlock(&trace_types_lock); - free_cpumask_var(iter->started); - kfree(iter->fmt); - kfree(iter->temp); - mutex_destroy(&iter->mutex); + free_trace_iter_content(iter); kfree(iter); trace_array_put(tr); -- cgit v1.2.3-59-g8ed1b From 6d98a0f2ac3c021d21be66fa34e992137cd25bcb Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Wed, 5 Jul 2023 08:27:05 +0800 Subject: tracing: Set actual size after ring buffer resize Currently we can resize trace ringbuffer by writing a value into file 'buffer_size_kb', then by reading the file, we get the value that is usually what we wrote. However, this value may be not actual size of trace ring buffer because of the round up when doing resize in kernel, and the actual size would be more useful. Link: https://lore.kernel.org/linux-trace-kernel/20230705002705.576633-1-zhengyejian1@huawei.com Cc: Signed-off-by: Zheng Yejian Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index a3c4f9826872..bc96567dc230 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6286,6 +6286,15 @@ static void set_buffer_entries(struct array_buffer *buf, unsigned long val) per_cpu_ptr(buf->data, cpu)->entries = val; } +static void update_buffer_entries(struct array_buffer *buf, int cpu) +{ + if (cpu == RING_BUFFER_ALL_CPUS) { + set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); + } else { + per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu); + } +} + #ifdef CONFIG_TRACER_MAX_TRACE /* resize @tr's buffer to the size of @size_tr's entries */ static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, @@ -6364,18 +6373,12 @@ static int __tracing_resize_ring_buffer(struct trace_array *tr, return ret; } - if (cpu == RING_BUFFER_ALL_CPUS) - set_buffer_entries(&tr->max_buffer, size); - else - per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; + update_buffer_entries(&tr->max_buffer, cpu); out: #endif /* CONFIG_TRACER_MAX_TRACE */ - if (cpu == RING_BUFFER_ALL_CPUS) - set_buffer_entries(&tr->array_buffer, size); - else - per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size; + update_buffer_entries(&tr->array_buffer, cpu); return ret; } -- cgit v1.2.3-59-g8ed1b From ee41106a12c76f38d0cf82ef17809fa62757151d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Fri, 28 Jul 2023 23:50:43 +0530 Subject: tracing: Require all trace events to have a TRACE_SYSTEM The creation of the trace event directory requires that a TRACE_SYSTEM is defined that the trace event directory is added within the system it was defined in. The code handled the case where a TRACE_SYSTEM was not added, and would then add the event at the events directory. But nothing should be doing this. This code also prevents the implementation of creating dynamic dentrys for the eventfs system. As this path has never been hit on correct code, remove it. If it does get hit, issues a WARN_ON_ONCE() and return ENODEV. Link: https://lkml.kernel.org/r/1690568452-46553-2-git-send-email-akaher@vmware.com Signed-off-by: Steven Rostedt (Google) Signed-off-by: Ajay Kaher Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 578f1f7d49a6..3ecc41f6acd9 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2420,14 +2420,15 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) /* * If the trace point header did not define TRACE_SYSTEM - * then the system would be called "TRACE_SYSTEM". + * then the system would be called "TRACE_SYSTEM". This should + * never happen. */ - if (strcmp(call->class->system, TRACE_SYSTEM) != 0) { - d_events = event_subsystem_dir(tr, call->class->system, file, parent); - if (!d_events) - return -ENOMEM; - } else - d_events = parent; + if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0)) + return -ENODEV; + + d_events = event_subsystem_dir(tr, call->class->system, file, parent); + if (!d_events) + return -ENOMEM; name = trace_event_name(call); file->dir = tracefs_create_dir(name, d_events); -- cgit v1.2.3-59-g8ed1b From ba37ff75e04be7df5fa19dcd86f81c984294a37b Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:44 +0530 Subject: eventfs: Implement tracefs_inode_cache Create a kmem cache of tracefs_inodes. To be more efficient, as there are lots of tracefs inodes, create its own cache. This also allows to see how many tracefs inodes have been created. Add helper functions: tracefs_alloc_inode() tracefs_free_inode() get_tracefs() Link: https://lkml.kernel.org/r/1690568452-46553-3-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/inode.c | 39 +++++++++++++++++++++++++++++++++++++++ fs/tracefs/internal.h | 15 +++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 fs/tracefs/internal.h diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 57ac8aa4a724..2508944cc4d8 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -21,13 +21,33 @@ #include #include #include +#include "internal.h" #define TRACEFS_DEFAULT_MODE 0700 +static struct kmem_cache *tracefs_inode_cachep __ro_after_init; static struct vfsmount *tracefs_mount; static int tracefs_mount_count; static bool tracefs_registered; +static struct inode *tracefs_alloc_inode(struct super_block *sb) +{ + struct tracefs_inode *ti; + + ti = kmem_cache_alloc(tracefs_inode_cachep, GFP_KERNEL); + if (!ti) + return NULL; + + ti->flags = 0; + + return &ti->vfs_inode; +} + +static void tracefs_free_inode(struct inode *inode) +{ + kmem_cache_free(tracefs_inode_cachep, get_tracefs(inode)); +} + static ssize_t default_read_file(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -346,6 +366,9 @@ static int tracefs_show_options(struct seq_file *m, struct dentry *root) } static const struct super_operations tracefs_super_operations = { + .alloc_inode = tracefs_alloc_inode, + .free_inode = tracefs_free_inode, + .drop_inode = generic_delete_inode, .statfs = simple_statfs, .remount_fs = tracefs_remount, .show_options = tracefs_show_options, @@ -628,10 +651,26 @@ bool tracefs_initialized(void) return tracefs_registered; } +static void init_once(void *foo) +{ + struct tracefs_inode *ti = (struct tracefs_inode *) foo; + + inode_init_once(&ti->vfs_inode); +} + static int __init tracefs_init(void) { int retval; + tracefs_inode_cachep = kmem_cache_create("tracefs_inode_cache", + sizeof(struct tracefs_inode), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_MEM_SPREAD| + SLAB_ACCOUNT), + init_once); + if (!tracefs_inode_cachep) + return -ENOMEM; + retval = sysfs_create_mount_point(kernel_kobj, "tracing"); if (retval) return -EINVAL; diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h new file mode 100644 index 000000000000..954ea005632b --- /dev/null +++ b/fs/tracefs/internal.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TRACEFS_INTERNAL_H +#define _TRACEFS_INTERNAL_H + +struct tracefs_inode { + unsigned long flags; + void *private; + struct inode vfs_inode; +}; + +static inline struct tracefs_inode *get_tracefs(const struct inode *inode) +{ + return container_of(inode, struct tracefs_inode, vfs_inode); +} +#endif /* _TRACEFS_INTERNAL_H */ -- cgit v1.2.3-59-g8ed1b From 2c6b6b1029d46a8760d6cba09b4e75cb1ac9b579 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:45 +0530 Subject: tracefs: Rename and export some tracefs functions Export a few tracefs functions that will be needed by the eventfs dynamic file system. Rename them to start with "tracefs_" to keep with the name space. start_creating -> tracefs_start_creating failed_creating -> tracefs_failed_creating end_creating -> tracefs_end_creating Link: https://lkml.kernel.org/r/1690568452-46553-4-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/inode.c | 20 ++++++++++---------- fs/tracefs/internal.h | 5 +++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 2508944cc4d8..4acc4b4dfd22 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -147,7 +147,7 @@ static const struct inode_operations tracefs_dir_inode_operations = { .rmdir = tracefs_syscall_rmdir, }; -static struct inode *tracefs_get_inode(struct super_block *sb) +struct inode *tracefs_get_inode(struct super_block *sb) { struct inode *inode = new_inode(sb); if (inode) { @@ -422,7 +422,7 @@ static struct file_system_type trace_fs_type = { }; MODULE_ALIAS_FS("tracefs"); -static struct dentry *start_creating(const char *name, struct dentry *parent) +struct dentry *tracefs_start_creating(const char *name, struct dentry *parent) { struct dentry *dentry; int error; @@ -460,7 +460,7 @@ static struct dentry *start_creating(const char *name, struct dentry *parent) return dentry; } -static struct dentry *failed_creating(struct dentry *dentry) +struct dentry *tracefs_failed_creating(struct dentry *dentry) { inode_unlock(d_inode(dentry->d_parent)); dput(dentry); @@ -468,7 +468,7 @@ static struct dentry *failed_creating(struct dentry *dentry) return NULL; } -static struct dentry *end_creating(struct dentry *dentry) +struct dentry *tracefs_end_creating(struct dentry *dentry) { inode_unlock(d_inode(dentry->d_parent)); return dentry; @@ -513,14 +513,14 @@ struct dentry *tracefs_create_file(const char *name, umode_t mode, if (!(mode & S_IFMT)) mode |= S_IFREG; BUG_ON(!S_ISREG(mode)); - dentry = start_creating(name, parent); + dentry = tracefs_start_creating(name, parent); if (IS_ERR(dentry)) return NULL; inode = tracefs_get_inode(dentry->d_sb); if (unlikely(!inode)) - return failed_creating(dentry); + return tracefs_failed_creating(dentry); inode->i_mode = mode; inode->i_fop = fops ? fops : &tracefs_file_operations; @@ -529,13 +529,13 @@ struct dentry *tracefs_create_file(const char *name, umode_t mode, inode->i_gid = d_inode(dentry->d_parent)->i_gid; d_instantiate(dentry, inode); fsnotify_create(d_inode(dentry->d_parent), dentry); - return end_creating(dentry); + return tracefs_end_creating(dentry); } static struct dentry *__create_dir(const char *name, struct dentry *parent, const struct inode_operations *ops) { - struct dentry *dentry = start_creating(name, parent); + struct dentry *dentry = tracefs_start_creating(name, parent); struct inode *inode; if (IS_ERR(dentry)) @@ -543,7 +543,7 @@ static struct dentry *__create_dir(const char *name, struct dentry *parent, inode = tracefs_get_inode(dentry->d_sb); if (unlikely(!inode)) - return failed_creating(dentry); + return tracefs_failed_creating(dentry); /* Do not set bits for OTH */ inode->i_mode = S_IFDIR | S_IRWXU | S_IRUSR| S_IRGRP | S_IXUSR | S_IXGRP; @@ -557,7 +557,7 @@ static struct dentry *__create_dir(const char *name, struct dentry *parent, d_instantiate(dentry, inode); inc_nlink(d_inode(dentry->d_parent)); fsnotify_mkdir(d_inode(dentry->d_parent), dentry); - return end_creating(dentry); + return tracefs_end_creating(dentry); } /** diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 954ea005632b..7dfb7ebc1c3f 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -12,4 +12,9 @@ static inline struct tracefs_inode *get_tracefs(const struct inode *inode) { return container_of(inode, struct tracefs_inode, vfs_inode); } + +struct dentry *tracefs_start_creating(const char *name, struct dentry *parent); +struct dentry *tracefs_end_creating(struct dentry *dentry); +struct dentry *tracefs_failed_creating(struct dentry *dentry); +struct inode *tracefs_get_inode(struct super_block *sb); #endif /* _TRACEFS_INTERNAL_H */ -- cgit v1.2.3-59-g8ed1b From c1504e5102384762bd06b068e9928b624a8a88c6 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:46 +0530 Subject: eventfs: Implement eventfs dir creation functions Add eventfs_file structure which will hold the properties of the eventfs files and directories. Add following functions to create the directories in eventfs: eventfs_create_events_dir() will create the top level "events" directory within the tracefs file system. eventfs_add_subsystem_dir() creates an eventfs_file descriptor with the given name of the subsystem. eventfs_add_dir() creates an eventfs_file descriptor with the given name of the directory and attached to a eventfs_file of a subsystem. Add tracefs_inode structure to hold the inodes, flags and pointers to private data used by eventfs. Link: https://lkml.kernel.org/r/1690568452-46553-5-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202305051619.9a469a9a-yujie.liu@intel.com Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/Makefile | 1 + fs/tracefs/event_inode.c | 211 +++++++++++++++++++++++++++++++++++++++++++++++ fs/tracefs/internal.h | 4 + include/linux/tracefs.h | 11 +++ 4 files changed, 227 insertions(+) create mode 100644 fs/tracefs/event_inode.c diff --git a/fs/tracefs/Makefile b/fs/tracefs/Makefile index 7c35a282b484..73c56da8e284 100644 --- a/fs/tracefs/Makefile +++ b/fs/tracefs/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only tracefs-objs := inode.o +tracefs-objs += event_inode.o obj-$(CONFIG_TRACING) += tracefs.o diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c new file mode 100644 index 000000000000..8f334b122e46 --- /dev/null +++ b/fs/tracefs/event_inode.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * event_inode.c - part of tracefs, a pseudo file system for activating tracing + * + * Copyright (C) 2020-23 VMware Inc, author: Steven Rostedt (VMware) + * Copyright (C) 2020-23 VMware Inc, author: Ajay Kaher + * + * eventfs is used to dynamically create inodes and dentries based on the + * meta data provided by the tracing system. + * + * eventfs stores the meta-data of files/dirs and holds off on creating + * inodes/dentries of the files. When accessed, the eventfs will create the + * inodes/dentries in a just-in-time (JIT) manner. The eventfs will clean up + * and delete the inodes/dentries when they are no longer referenced. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +struct eventfs_inode { + struct list_head e_top_files; +}; + +/** + * struct eventfs_file - hold the properties of the eventfs files and + * directories. + * @name: the name of the file or directory to create + * @list: file or directory to be added to parent directory + * @ei: list of files and directories within directory + * @fop: file_operations for file or directory + * @iop: inode_operations for file or directory + * @data: something that the caller will want to get to later on + * @mode: the permission that the file or directory should have + */ +struct eventfs_file { + const char *name; + struct list_head list; + struct eventfs_inode *ei; + const struct file_operations *fop; + const struct inode_operations *iop; + void *data; + umode_t mode; +}; + +static DEFINE_MUTEX(eventfs_mutex); + +static const struct inode_operations eventfs_root_dir_inode_operations = { +}; + +static const struct file_operations eventfs_file_operations = { +}; + +/** + * eventfs_prepare_ef - helper function to prepare eventfs_file + * @name: the name of the file/directory to create. + * @mode: the permission that the file should have. + * @fop: struct file_operations that should be used for this file/directory. + * @iop: struct inode_operations that should be used for this file/directory. + * @data: something that the caller will want to get to later on. The + * inode.i_private pointer will point to this value on the open() call. + * + * This function allocates and fills the eventfs_file structure. + */ +static struct eventfs_file *eventfs_prepare_ef(const char *name, umode_t mode, + const struct file_operations *fop, + const struct inode_operations *iop, + void *data) +{ + struct eventfs_file *ef; + + ef = kzalloc(sizeof(*ef), GFP_KERNEL); + if (!ef) + return ERR_PTR(-ENOMEM); + + ef->name = kstrdup(name, GFP_KERNEL); + if (!ef->name) { + kfree(ef); + return ERR_PTR(-ENOMEM); + } + + if (S_ISDIR(mode)) { + ef->ei = kzalloc(sizeof(*ef->ei), GFP_KERNEL); + if (!ef->ei) { + kfree(ef->name); + kfree(ef); + return ERR_PTR(-ENOMEM); + } + INIT_LIST_HEAD(&ef->ei->e_top_files); + } else { + ef->ei = NULL; + } + + ef->iop = iop; + ef->fop = fop; + ef->mode = mode; + ef->data = data; + return ef; +} + +/** + * eventfs_create_events_dir - create the trace event structure + * @name: the name of the directory to create. + * @parent: parent dentry for this file. This should be a directory dentry + * if set. If this parameter is NULL, then the directory will be + * created in the root of the tracefs filesystem. + * + * This function creates the top of the trace event directory. + */ +struct dentry *eventfs_create_events_dir(const char *name, + struct dentry *parent) +{ + struct dentry *dentry = tracefs_start_creating(name, parent); + struct eventfs_inode *ei; + struct tracefs_inode *ti; + struct inode *inode; + + if (IS_ERR(dentry)) + return dentry; + + ei = kzalloc(sizeof(*ei), GFP_KERNEL); + if (!ei) + return ERR_PTR(-ENOMEM); + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) { + kfree(ei); + tracefs_failed_creating(dentry); + return ERR_PTR(-ENOMEM); + } + + INIT_LIST_HEAD(&ei->e_top_files); + + ti = get_tracefs(inode); + ti->flags |= TRACEFS_EVENT_INODE; + ti->private = ei; + + inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + inode->i_op = &eventfs_root_dir_inode_operations; + inode->i_fop = &eventfs_file_operations; + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + d_instantiate(dentry, inode); + inc_nlink(dentry->d_parent->d_inode); + fsnotify_mkdir(dentry->d_parent->d_inode, dentry); + return tracefs_end_creating(dentry); +} + +/** + * eventfs_add_subsystem_dir - add eventfs subsystem_dir to list to create later + * @name: the name of the file to create. + * @parent: parent dentry for this dir. + * + * This function adds eventfs subsystem dir to list. + * And all these dirs are created on the fly when they are looked up, + * and the dentry and inodes will be removed when they are done. + */ +struct eventfs_file *eventfs_add_subsystem_dir(const char *name, + struct dentry *parent) +{ + struct tracefs_inode *ti_parent; + struct eventfs_inode *ei_parent; + struct eventfs_file *ef; + + if (!parent) + return ERR_PTR(-EINVAL); + + ti_parent = get_tracefs(parent->d_inode); + ei_parent = ti_parent->private; + + ef = eventfs_prepare_ef(name, S_IFDIR, NULL, NULL, NULL); + if (IS_ERR(ef)) + return ef; + + mutex_lock(&eventfs_mutex); + list_add_tail(&ef->list, &ei_parent->e_top_files); + mutex_unlock(&eventfs_mutex); + return ef; +} + +/** + * eventfs_add_dir - add eventfs dir to list to create later + * @name: the name of the file to create. + * @ef_parent: parent eventfs_file for this dir. + * + * This function adds eventfs dir to list. + * And all these dirs are created on the fly when they are looked up, + * and the dentry and inodes will be removed when they are done. + */ +struct eventfs_file *eventfs_add_dir(const char *name, + struct eventfs_file *ef_parent) +{ + struct eventfs_file *ef; + + if (!ef_parent) + return ERR_PTR(-EINVAL); + + ef = eventfs_prepare_ef(name, S_IFDIR, NULL, NULL, NULL); + if (IS_ERR(ef)) + return ef; + + mutex_lock(&eventfs_mutex); + list_add_tail(&ef->list, &ef_parent->ei->e_top_files); + mutex_unlock(&eventfs_mutex); + return ef; +} diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 7dfb7ebc1c3f..f0fd565d59ec 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -2,6 +2,10 @@ #ifndef _TRACEFS_INTERNAL_H #define _TRACEFS_INTERNAL_H +enum { + TRACEFS_EVENT_INODE = BIT(1), +}; + struct tracefs_inode { unsigned long flags; void *private; diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 99912445974c..432e5e6f7901 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -21,6 +21,17 @@ struct file_operations; #ifdef CONFIG_TRACING +struct eventfs_file; + +struct dentry *eventfs_create_events_dir(const char *name, + struct dentry *parent); + +struct eventfs_file *eventfs_add_subsystem_dir(const char *name, + struct dentry *parent); + +struct eventfs_file *eventfs_add_dir(const char *name, + struct eventfs_file *ef_parent); + struct dentry *tracefs_create_file(const char *name, umode_t mode, struct dentry *parent, void *data, const struct file_operations *fops); -- cgit v1.2.3-59-g8ed1b From 88f349b4a83aab6a0fabd4230a1b6796f26bc546 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:47 +0530 Subject: eventfs: Implement eventfs file add functions Add the following functions to add files to evenfs: eventfs_add_events_file() to add the data needed to create a specific file located at the top level events directory. The dentry/inode will be created when the events directory is scanned. eventfs_add_file() to add the data needed for files within the directories below the top level events directory. The dentry/inode of the file will be created when the directory that the file is in is scanned. Link: https://lkml.kernel.org/r/1690568452-46553-6-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202305051619.9a469a9a-yujie.liu@intel.com Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/tracefs.h | 8 +++++ 2 files changed, 94 insertions(+) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 8f334b122e46..9e4843be9dc9 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -209,3 +209,89 @@ struct eventfs_file *eventfs_add_dir(const char *name, mutex_unlock(&eventfs_mutex); return ef; } + +/** + * eventfs_add_events_file - add the data needed to create a file for later reference + * @name: the name of the file to create. + * @mode: the permission that the file should have. + * @parent: parent dentry for this file. + * @data: something that the caller will want to get to later on. + * @fop: struct file_operations that should be used for this file. + * + * This function is used to add the information needed to create a + * dentry/inode within the top level events directory. The file created + * will have the @mode permissions. The @data will be used to fill the + * inode.i_private when the open() call is done. The dentry and inodes are + * all created when they are referenced, and removed when they are no + * longer referenced. + */ +int eventfs_add_events_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fop) +{ + struct tracefs_inode *ti; + struct eventfs_inode *ei; + struct eventfs_file *ef; + + if (!parent) + return -EINVAL; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + + if (!parent->d_inode) + return -EINVAL; + + ti = get_tracefs(parent->d_inode); + if (!(ti->flags & TRACEFS_EVENT_INODE)) + return -EINVAL; + + ei = ti->private; + ef = eventfs_prepare_ef(name, mode, fop, NULL, data); + + if (IS_ERR(ef)) + return -ENOMEM; + + mutex_lock(&eventfs_mutex); + list_add_tail(&ef->list, &ei->e_top_files); + mutex_unlock(&eventfs_mutex); + return 0; +} + +/** + * eventfs_add_file - add eventfs file to list to create later + * @name: the name of the file to create. + * @mode: the permission that the file should have. + * @ef_parent: parent eventfs_file for this file. + * @data: something that the caller will want to get to later on. + * @fop: struct file_operations that should be used for this file. + * + * This function is used to add the information needed to create a + * file within a subdirectory of the events directory. The file created + * will have the @mode permissions. The @data will be used to fill the + * inode.i_private when the open() call is done. The dentry and inodes are + * all created when they are referenced, and removed when they are no + * longer referenced. + */ +int eventfs_add_file(const char *name, umode_t mode, + struct eventfs_file *ef_parent, + void *data, + const struct file_operations *fop) +{ + struct eventfs_file *ef; + + if (!ef_parent) + return -EINVAL; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + + ef = eventfs_prepare_ef(name, mode, fop, NULL, data); + if (IS_ERR(ef)) + return -ENOMEM; + + mutex_lock(&eventfs_mutex); + list_add_tail(&ef->list, &ef_parent->ei->e_top_files); + mutex_unlock(&eventfs_mutex); + return 0; +} diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 432e5e6f7901..54c9cbd0389b 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -32,6 +32,14 @@ struct eventfs_file *eventfs_add_subsystem_dir(const char *name, struct eventfs_file *eventfs_add_dir(const char *name, struct eventfs_file *ef_parent); +int eventfs_add_file(const char *name, umode_t mode, + struct eventfs_file *ef_parent, void *data, + const struct file_operations *fops); + +int eventfs_add_events_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fops); + struct dentry *tracefs_create_file(const char *name, umode_t mode, struct dentry *parent, void *data, const struct file_operations *fops); -- cgit v1.2.3-59-g8ed1b From 63940449555e799d387f316993ad824476c16953 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:48 +0530 Subject: eventfs: Implement eventfs lookup, read, open functions Add the inode_operations, file_operations, and helper functions to eventfs: dcache_dir_open_wrapper() eventfs_root_lookup() eventfs_release() eventfs_set_ef_status_free() eventfs_post_create_dir() The inode_operations and file_operations functions will be called from the VFS layer. create_file() and create_dir() are added as stub functions and will be filled in later. Link: https://lkml.kernel.org/r/1690568452-46553-7-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 304 +++++++++++++++++++++++++++++++++++++++++++++++ fs/tracefs/internal.h | 2 + 2 files changed, 306 insertions(+) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 9e4843be9dc9..24d645c61029 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -31,6 +31,8 @@ struct eventfs_inode { * struct eventfs_file - hold the properties of the eventfs files and * directories. * @name: the name of the file or directory to create + * @d_parent: holds parent's dentry + * @dentry: once accessed holds dentry * @list: file or directory to be added to parent directory * @ei: list of files and directories within directory * @fop: file_operations for file or directory @@ -40,22 +42,320 @@ struct eventfs_inode { */ struct eventfs_file { const char *name; + struct dentry *d_parent; + struct dentry *dentry; struct list_head list; struct eventfs_inode *ei; const struct file_operations *fop; const struct inode_operations *iop; + union { + struct list_head del_list; /* list of eventfs_file to delete */ + struct rcu_head rcu; /* eventfs_file to delete */ + unsigned long is_freed; /* Freed if one of the above is set */ + }; void *data; umode_t mode; }; static DEFINE_MUTEX(eventfs_mutex); +DEFINE_STATIC_SRCU(eventfs_srcu); + +static struct dentry *eventfs_root_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags); +static int dcache_dir_open_wrapper(struct inode *inode, struct file *file); +static int eventfs_release(struct inode *inode, struct file *file); static const struct inode_operations eventfs_root_dir_inode_operations = { + .lookup = eventfs_root_lookup, }; static const struct file_operations eventfs_file_operations = { + .open = dcache_dir_open_wrapper, + .read = generic_read_dir, + .iterate_shared = dcache_readdir, + .llseek = generic_file_llseek, + .release = eventfs_release, +}; + +/** + * create_file - create a file in the tracefs filesystem + * @name: the name of the file to create. + * @mode: the permission that the file should have. + * @parent: parent dentry for this file. + * @data: something that the caller will want to get to later on. + * @fop: struct file_operations that should be used for this file. + * + * This is the basic "create a file" function for tracefs. It allows for a + * wide range of flexibility in creating a file. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the tracefs_remove() function when the file is + * to be removed (no automatic cleanup happens if your module is unloaded, + * you are responsible here.) If an error occurs, %NULL will be returned. + * + * If tracefs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +static struct dentry *create_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fop) +{ + return NULL; }; +/** + * create_dir - create a dir in the tracefs filesystem + * @name: the name of the file to create. + * @parent: parent dentry for this file. + * @data: something that the caller will want to get to later on. + * + * This is the basic "create a dir" function for eventfs. It allows for a + * wide range of flexibility in creating a dir. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the tracefs_remove() function when the file is + * to be removed (no automatic cleanup happens if your module is unloaded, + * you are responsible here.) If an error occurs, %NULL will be returned. + * + * If tracefs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +static struct dentry *create_dir(const char *name, struct dentry *parent, void *data) +{ + return NULL; +} + +/** + * eventfs_set_ef_status_free - set the ef->status to free + * @dentry: dentry who's status to be freed + * + * eventfs_set_ef_status_free will be called if no more + * references remain + */ +void eventfs_set_ef_status_free(struct dentry *dentry) +{ + struct tracefs_inode *ti_parent; + struct eventfs_file *ef; + + mutex_lock(&eventfs_mutex); + ti_parent = get_tracefs(dentry->d_parent->d_inode); + if (!ti_parent || !(ti_parent->flags & TRACEFS_EVENT_INODE)) + goto out; + + ef = dentry->d_fsdata; + if (!ef) + goto out; + + dentry->d_fsdata = NULL; + ef->dentry = NULL; +out: + mutex_unlock(&eventfs_mutex); +} + +/** + * eventfs_post_create_dir - post create dir routine + * @ef: eventfs_file of recently created dir + * + * Map the meta-data of files within an eventfs dir to their parent dentry + */ +static void eventfs_post_create_dir(struct eventfs_file *ef) +{ + struct eventfs_file *ef_child; + struct tracefs_inode *ti; + + /* srcu lock already held */ + /* fill parent-child relation */ + list_for_each_entry_srcu(ef_child, &ef->ei->e_top_files, list, + srcu_read_lock_held(&eventfs_srcu)) { + ef_child->d_parent = ef->dentry; + } + + ti = get_tracefs(ef->dentry->d_inode); + ti->private = ef->ei; +} + +/** + * create_dentry - helper function to create dentry + * @ef: eventfs_file of file or directory to create + * @parent: parent dentry + * @lookup: true if called from lookup routine + * + * Used to create a dentry for file/dir, executes post dentry creation routine + */ +static struct dentry * +create_dentry(struct eventfs_file *ef, struct dentry *parent, bool lookup) +{ + bool invalidate = false; + struct dentry *dentry; + + mutex_lock(&eventfs_mutex); + if (ef->is_freed) { + mutex_unlock(&eventfs_mutex); + return NULL; + } + if (ef->dentry) { + dentry = ef->dentry; + /* On dir open, up the ref count */ + if (!lookup) + dget(dentry); + mutex_unlock(&eventfs_mutex); + return dentry; + } + mutex_unlock(&eventfs_mutex); + + if (!lookup) + inode_lock(parent->d_inode); + + if (ef->ei) + dentry = create_dir(ef->name, parent, ef->data); + else + dentry = create_file(ef->name, ef->mode, parent, + ef->data, ef->fop); + + if (!lookup) + inode_unlock(parent->d_inode); + + mutex_lock(&eventfs_mutex); + if (IS_ERR_OR_NULL(dentry)) { + /* If the ef was already updated get it */ + dentry = ef->dentry; + if (dentry && !lookup) + dget(dentry); + mutex_unlock(&eventfs_mutex); + return dentry; + } + + if (!ef->dentry && !ef->is_freed) { + ef->dentry = dentry; + if (ef->ei) + eventfs_post_create_dir(ef); + dentry->d_fsdata = ef; + } else { + /* A race here, should try again (unless freed) */ + invalidate = true; + } + mutex_unlock(&eventfs_mutex); + if (invalidate) + d_invalidate(dentry); + + if (lookup || invalidate) + dput(dentry); + + return invalidate ? NULL : dentry; +} + +static bool match_event_file(struct eventfs_file *ef, const char *name) +{ + bool ret; + + mutex_lock(&eventfs_mutex); + ret = !ef->is_freed && strcmp(ef->name, name) == 0; + mutex_unlock(&eventfs_mutex); + + return ret; +} + +/** + * eventfs_root_lookup - lookup routine to create file/dir + * @dir: in which a lookup is being done + * @dentry: file/dir dentry + * @flags: to pass as flags parameter to simple lookup + * + * Used to create a dynamic file/dir within @dir. Use the eventfs_inode + * list of meta data to find the information needed to create the file/dir. + */ +static struct dentry *eventfs_root_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct tracefs_inode *ti; + struct eventfs_inode *ei; + struct eventfs_file *ef; + struct dentry *ret = NULL; + int idx; + + ti = get_tracefs(dir); + if (!(ti->flags & TRACEFS_EVENT_INODE)) + return NULL; + + ei = ti->private; + idx = srcu_read_lock(&eventfs_srcu); + list_for_each_entry_srcu(ef, &ei->e_top_files, list, + srcu_read_lock_held(&eventfs_srcu)) { + if (!match_event_file(ef, dentry->d_name.name)) + continue; + ret = simple_lookup(dir, dentry, flags); + create_dentry(ef, ef->d_parent, true); + break; + } + srcu_read_unlock(&eventfs_srcu, idx); + return ret; +} + +/** + * eventfs_release - called to release eventfs file/dir + * @inode: inode to be released + * @file: file to be released (not used) + */ +static int eventfs_release(struct inode *inode, struct file *file) +{ + struct tracefs_inode *ti; + struct eventfs_inode *ei; + struct eventfs_file *ef; + struct dentry *dentry; + int idx; + + ti = get_tracefs(inode); + if (!(ti->flags & TRACEFS_EVENT_INODE)) + return -EINVAL; + + ei = ti->private; + idx = srcu_read_lock(&eventfs_srcu); + list_for_each_entry_srcu(ef, &ei->e_top_files, list, + srcu_read_lock_held(&eventfs_srcu)) { + mutex_lock(&eventfs_mutex); + dentry = ef->dentry; + mutex_unlock(&eventfs_mutex); + if (dentry) + dput(dentry); + } + srcu_read_unlock(&eventfs_srcu, idx); + return dcache_dir_close(inode, file); +} + +/** + * dcache_dir_open_wrapper - eventfs open wrapper + * @inode: not used + * @file: dir to be opened (to create its child) + * + * Used to dynamically create the file/dir within @file. @file is really a + * directory and all the files/dirs of the children within @file will be + * created. If any of the files/dirs have already been created, their + * reference count will be incremented. + */ +static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) +{ + struct tracefs_inode *ti; + struct eventfs_inode *ei; + struct eventfs_file *ef; + struct dentry *dentry = file_dentry(file); + struct inode *f_inode = file_inode(file); + int idx; + + ti = get_tracefs(f_inode); + if (!(ti->flags & TRACEFS_EVENT_INODE)) + return -EINVAL; + + ei = ti->private; + idx = srcu_read_lock(&eventfs_srcu); + list_for_each_entry_rcu(ef, &ei->e_top_files, list) { + create_dentry(ef, dentry, false); + } + srcu_read_unlock(&eventfs_srcu, idx); + return dcache_dir_open(inode, file); +} + /** * eventfs_prepare_ef - helper function to prepare eventfs_file * @name: the name of the file/directory to create. @@ -179,6 +479,7 @@ struct eventfs_file *eventfs_add_subsystem_dir(const char *name, mutex_lock(&eventfs_mutex); list_add_tail(&ef->list, &ei_parent->e_top_files); + ef->d_parent = parent; mutex_unlock(&eventfs_mutex); return ef; } @@ -206,6 +507,7 @@ struct eventfs_file *eventfs_add_dir(const char *name, mutex_lock(&eventfs_mutex); list_add_tail(&ef->list, &ef_parent->ei->e_top_files); + ef->d_parent = ef_parent->dentry; mutex_unlock(&eventfs_mutex); return ef; } @@ -254,6 +556,7 @@ int eventfs_add_events_file(const char *name, umode_t mode, mutex_lock(&eventfs_mutex); list_add_tail(&ef->list, &ei->e_top_files); + ef->d_parent = parent; mutex_unlock(&eventfs_mutex); return 0; } @@ -292,6 +595,7 @@ int eventfs_add_file(const char *name, umode_t mode, mutex_lock(&eventfs_mutex); list_add_tail(&ef->list, &ef_parent->ei->e_top_files); + ef->d_parent = ef_parent->dentry; mutex_unlock(&eventfs_mutex); return 0; } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index f0fd565d59ec..9bfad9d95a4a 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -21,4 +21,6 @@ struct dentry *tracefs_start_creating(const char *name, struct dentry *parent); struct dentry *tracefs_end_creating(struct dentry *dentry); struct dentry *tracefs_failed_creating(struct dentry *dentry); struct inode *tracefs_get_inode(struct super_block *sb); +void eventfs_set_ef_status_free(struct dentry *dentry); + #endif /* _TRACEFS_INTERNAL_H */ -- cgit v1.2.3-59-g8ed1b From a3760079177765b7f1782419f1c3e12facaf1e9d Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:49 +0530 Subject: eventfs: Implement functions to create files and dirs when accessed Add create_file() and create_dir() functions to create the files and directories respectively when they are accessed. The functions will be called from the lookup operation of the inode_operations or from the open function of file_operations. Link: https://lkml.kernel.org/r/1690568452-46553-8-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 61 +++++++++++++++++++++++++++++++++++++-- fs/tracefs/inode.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/tracefs/internal.h | 3 ++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 24d645c61029..5240bd2c81e7 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -101,7 +101,34 @@ static struct dentry *create_file(const char *name, umode_t mode, struct dentry *parent, void *data, const struct file_operations *fop) { - return NULL; + struct tracefs_inode *ti; + struct dentry *dentry; + struct inode *inode; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + + if (WARN_ON_ONCE(!S_ISREG(mode))) + return NULL; + + dentry = eventfs_start_creating(name, parent); + + if (IS_ERR(dentry)) + return dentry; + + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) + return eventfs_failed_creating(dentry); + + inode->i_mode = mode; + inode->i_fop = fop; + inode->i_private = data; + + ti = get_tracefs(inode); + ti->flags |= TRACEFS_EVENT_INODE; + d_instantiate(dentry, inode); + fsnotify_create(dentry->d_parent->d_inode, dentry); + return eventfs_end_creating(dentry); }; /** @@ -123,7 +150,31 @@ static struct dentry *create_file(const char *name, umode_t mode, */ static struct dentry *create_dir(const char *name, struct dentry *parent, void *data) { - return NULL; + struct tracefs_inode *ti; + struct dentry *dentry; + struct inode *inode; + + dentry = eventfs_start_creating(name, parent); + if (IS_ERR(dentry)) + return dentry; + + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) + return eventfs_failed_creating(dentry); + + inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + inode->i_op = &eventfs_root_dir_inode_operations; + inode->i_fop = &eventfs_file_operations; + inode->i_private = data; + + ti = get_tracefs(inode); + ti->flags |= TRACEFS_EVENT_INODE; + + inc_nlink(inode); + d_instantiate(dentry, inode); + inc_nlink(dentry->d_parent->d_inode); + fsnotify_mkdir(dentry->d_parent->d_inode, dentry); + return eventfs_end_creating(dentry); } /** @@ -234,6 +285,12 @@ create_dentry(struct eventfs_file *ef, struct dentry *parent, bool lookup) } else { /* A race here, should try again (unless freed) */ invalidate = true; + + /* + * Should never happen unless we get here due to being freed. + * Otherwise it means two dentries exist with the same name. + */ + WARN_ON_ONCE(!ef->is_freed); } mutex_unlock(&eventfs_mutex); if (invalidate) diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 4acc4b4dfd22..d9273066f25f 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -474,6 +474,80 @@ struct dentry *tracefs_end_creating(struct dentry *dentry) return dentry; } +/** + * eventfs_start_creating - start the process of creating a dentry + * @name: Name of the file created for the dentry + * @parent: The parent dentry where this dentry will be created + * + * This is a simple helper function for the dynamically created eventfs + * files. When the directory of the eventfs files are accessed, their + * dentries are created on the fly. This function is used to start that + * process. + */ +struct dentry *eventfs_start_creating(const char *name, struct dentry *parent) +{ + struct dentry *dentry; + int error; + + error = simple_pin_fs(&trace_fs_type, &tracefs_mount, + &tracefs_mount_count); + if (error) + return ERR_PTR(error); + + /* + * If the parent is not specified, we create it in the root. + * We need the root dentry to do this, which is in the super + * block. A pointer to that is in the struct vfsmount that we + * have around. + */ + if (!parent) + parent = tracefs_mount->mnt_root; + + if (unlikely(IS_DEADDIR(parent->d_inode))) + dentry = ERR_PTR(-ENOENT); + else + dentry = lookup_one_len(name, parent, strlen(name)); + + if (!IS_ERR(dentry) && dentry->d_inode) { + dput(dentry); + dentry = ERR_PTR(-EEXIST); + } + + if (IS_ERR(dentry)) + simple_release_fs(&tracefs_mount, &tracefs_mount_count); + + return dentry; +} + +/** + * eventfs_failed_creating - clean up a failed eventfs dentry creation + * @dentry: The dentry to clean up + * + * If after calling eventfs_start_creating(), a failure is detected, the + * resources created by eventfs_start_creating() needs to be cleaned up. In + * that case, this function should be called to perform that clean up. + */ +struct dentry *eventfs_failed_creating(struct dentry *dentry) +{ + dput(dentry); + simple_release_fs(&tracefs_mount, &tracefs_mount_count); + return NULL; +} + +/** + * eventfs_end_creating - Finish the process of creating a eventfs dentry + * @dentry: The dentry that has successfully been created. + * + * This function is currently just a place holder to match + * eventfs_start_creating(). In case any synchronization needs to be added, + * this function will be used to implement that without having to modify + * the callers of eventfs_start_creating(). + */ +struct dentry *eventfs_end_creating(struct dentry *dentry) +{ + return dentry; +} + /** * tracefs_create_file - create a file in the tracefs filesystem * @name: a pointer to a string containing the name of the file to create. diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 9bfad9d95a4a..69c2b1d87c46 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -21,6 +21,9 @@ struct dentry *tracefs_start_creating(const char *name, struct dentry *parent); struct dentry *tracefs_end_creating(struct dentry *dentry); struct dentry *tracefs_failed_creating(struct dentry *dentry); struct inode *tracefs_get_inode(struct super_block *sb); +struct dentry *eventfs_start_creating(const char *name, struct dentry *parent); +struct dentry *eventfs_failed_creating(struct dentry *dentry); +struct dentry *eventfs_end_creating(struct dentry *dentry); void eventfs_set_ef_status_free(struct dentry *dentry); #endif /* _TRACEFS_INTERNAL_H */ -- cgit v1.2.3-59-g8ed1b From 5bdcd5f5331a276a3ddf1fba8605986d0d15298a Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:50 +0530 Subject: eventfs: Implement removal of meta data from eventfs When events are removed from tracefs, the eventfs must be aware of this. The eventfs_remove() removes the meta data from eventfs so that it will no longer create the files associated with that event. When an instance is removed from tracefs, eventfs_remove_events_dir() will remove and clean up the entire "events" directory. The helper function eventfs_remove_rec() is used to clean up and free the associated data from eventfs for both of the added functions. SRCU is used to protect the lists of meta data stored in the eventfs. The eventfs_mutex is used to protect the content of the items in the list. As lookups may be happening as deletions of events are made, the freeing of dentry/inodes and relative information is done after the SRCU grace period has passed. Link: https://lkml.kernel.org/r/1690568452-46553-9-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202305030611.Kas747Ev-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 143 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/tracefs.h | 4 ++ 2 files changed, 147 insertions(+) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 5240bd2c81e7..da8d2e73cc47 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -198,6 +198,14 @@ void eventfs_set_ef_status_free(struct dentry *dentry) if (!ef) goto out; + /* + * If ef was freed, then the LSB bit is set for d_fsdata. + * But this should not happen, as it should still have a + * ref count that prevents it. Warn in case it does. + */ + if (WARN_ON_ONCE((unsigned long)ef & 1)) + goto out; + dentry->d_fsdata = NULL; ef->dentry = NULL; out: @@ -656,3 +664,138 @@ int eventfs_add_file(const char *name, umode_t mode, mutex_unlock(&eventfs_mutex); return 0; } + +static void free_ef(struct rcu_head *head) +{ + struct eventfs_file *ef = container_of(head, struct eventfs_file, rcu); + + kfree(ef->name); + kfree(ef->ei); + kfree(ef); +} + +/** + * eventfs_remove_rec - remove eventfs dir or file from list + * @ef: eventfs_file to be removed. + * @head: to create list of eventfs_file to be deleted + * @level: to check recursion depth + * + * The helper function eventfs_remove_rec() is used to clean up and free the + * associated data from eventfs for both of the added functions. + */ +static void eventfs_remove_rec(struct eventfs_file *ef, struct list_head *head, int level) +{ + struct eventfs_file *ef_child; + + if (!ef) + return; + /* + * Check recursion depth. It should never be greater than 3: + * 0 - events/ + * 1 - events/group/ + * 2 - events/group/event/ + * 3 - events/group/event/file + */ + if (WARN_ON_ONCE(level > 3)) + return; + + if (ef->ei) { + /* search for nested folders or files */ + list_for_each_entry_srcu(ef_child, &ef->ei->e_top_files, list, + lockdep_is_held(&eventfs_mutex)) { + eventfs_remove_rec(ef_child, head, level + 1); + } + } + + list_del_rcu(&ef->list); + list_add_tail(&ef->del_list, head); +} + +/** + * eventfs_remove - remove eventfs dir or file from list + * @ef: eventfs_file to be removed. + * + * This function acquire the eventfs_mutex lock and call eventfs_remove_rec() + */ +void eventfs_remove(struct eventfs_file *ef) +{ + struct eventfs_file *tmp; + LIST_HEAD(ef_del_list); + struct dentry *dentry_list = NULL; + struct dentry *dentry; + + if (!ef) + return; + + mutex_lock(&eventfs_mutex); + eventfs_remove_rec(ef, &ef_del_list, 0); + list_for_each_entry_safe(ef, tmp, &ef_del_list, del_list) { + if (ef->dentry) { + unsigned long ptr = (unsigned long)dentry_list; + + /* Keep the dentry from being freed yet */ + dget(ef->dentry); + + /* + * Paranoid: The dget() above should prevent the dentry + * from being freed and calling eventfs_set_ef_status_free(). + * But just in case, set the link list LSB pointer to 1 + * and have eventfs_set_ef_status_free() check that to + * make sure that if it does happen, it will not think + * the d_fsdata is an event_file. + * + * For this to work, no event_file should be allocated + * on a odd space, as the ef should always be allocated + * to be at least word aligned. Check for that too. + */ + WARN_ON_ONCE(ptr & 1); + + ef->dentry->d_fsdata = (void *)(ptr | 1); + dentry_list = ef->dentry; + ef->dentry = NULL; + } + call_srcu(&eventfs_srcu, &ef->rcu, free_ef); + } + mutex_unlock(&eventfs_mutex); + + while (dentry_list) { + unsigned long ptr; + + dentry = dentry_list; + ptr = (unsigned long)dentry->d_fsdata & ~1UL; + dentry_list = (struct dentry *)ptr; + dentry->d_fsdata = NULL; + d_invalidate(dentry); + mutex_lock(&eventfs_mutex); + /* dentry should now have at least a single reference */ + WARN_ONCE((int)d_count(dentry) < 1, + "dentry %p less than one reference (%d) after invalidate\n", + dentry, d_count(dentry)); + mutex_unlock(&eventfs_mutex); + dput(dentry); + } +} + +/** + * eventfs_remove_events_dir - remove eventfs dir or file from list + * @dentry: events's dentry to be removed. + * + * This function remove events main directory + */ +void eventfs_remove_events_dir(struct dentry *dentry) +{ + struct tracefs_inode *ti; + struct eventfs_inode *ei; + + if (!dentry || !dentry->d_inode) + return; + + ti = get_tracefs(dentry->d_inode); + if (!ti || !(ti->flags & TRACEFS_EVENT_INODE)) + return; + + ei = ti->private; + d_invalidate(dentry); + dput(dentry); + kfree(ei); +} diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 54c9cbd0389b..009072792fa3 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -40,6 +40,10 @@ int eventfs_add_events_file(const char *name, umode_t mode, struct dentry *parent, void *data, const struct file_operations *fops); +void eventfs_remove(struct eventfs_file *ef); + +void eventfs_remove_events_dir(struct dentry *dentry); + struct dentry *tracefs_create_file(const char *name, umode_t mode, struct dentry *parent, void *data, const struct file_operations *fops); -- cgit v1.2.3-59-g8ed1b From 27152bceea1df27ffebb12ac9cd9adbf2c4c3f35 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:51 +0530 Subject: eventfs: Move tracing/events to eventfs Up until now, /sys/kernel/tracing/events was no different than any other part of tracefs. The files and directories within the events directory was created when the tracefs was mounted, and also created for the instances in /sys/kernel/tracing/instances//events. Most of these files and directories will never be referenced. Since there are thousands of these files and directories they spend their time wasting precious memory resources. Move the "events" directory to the new eventfs. The eventfs will take the meta data of the events that they represent and store that. When the files in the events directory are referenced, the dentry and inodes to represent them are then created. When the files are no longer referenced, they are freed. This saves the precious memory resources that were wasted on these seldom referenced dentries and inodes. Running the following: ~# cat /proc/meminfo /proc/slabinfo > before.out ~# mkdir /sys/kernel/tracing/instances/foo ~# cat /proc/meminfo /proc/slabinfo > after.out to test the changes produces the following deltas: Before this change: Before after deltas for meminfo: MemFree: -32260 MemAvailable: -21496 KReclaimable: 21528 Slab: 22440 SReclaimable: 21528 SUnreclaim: 912 VmallocUsed: 16 Before after deltas for slabinfo: : [ * = ] tracefs_inode_cache: 14472 [* 1184 = 17134848] buffer_head: 24 [* 168 = 4032] hmem_inode_cache: 28 [* 1480 = 41440] dentry: 14450 [* 312 = 4508400] lsm_inode_cache: 14453 [* 32 = 462496] vma_lock: 11 [* 152 = 1672] vm_area_struct: 2 [* 184 = 368] trace_event_file: 1748 [* 88 = 153824] kmalloc-256: 1072 [* 256 = 274432] kmalloc-64: 2842 [* 64 = 181888] Total slab additions in size: 22,763,400 bytes With this change: Before after deltas for meminfo: MemFree: -12600 MemAvailable: -12580 Cached: 24 Active: 12 Inactive: 68 Inactive(anon): 48 Active(file): 12 Inactive(file): 20 Dirty: -4 AnonPages: 68 KReclaimable: 12 Slab: 1856 SReclaimable: 12 SUnreclaim: 1844 KernelStack: 16 PageTables: 36 VmallocUsed: 16 Before after deltas for slabinfo: : [ * = ] tracefs_inode_cache: 108 [* 1184 = 127872] buffer_head: 24 [* 168 = 4032] hmem_inode_cache: 18 [* 1480 = 26640] dentry: 127 [* 312 = 39624] lsm_inode_cache: 152 [* 32 = 4864] vma_lock: 67 [* 152 = 10184] vm_area_struct: -12 [* 184 = -2208] trace_event_file: 1764 [* 96 = 169344] kmalloc-96: 14322 [* 96 = 1374912] kmalloc-64: 2814 [* 64 = 180096] kmalloc-32: 1103 [* 32 = 35296] kmalloc-16: 2308 [* 16 = 36928] kmalloc-8: 12800 [* 8 = 102400] Total slab additions in size: 2,109,984 bytes Which is a savings of 20,653,416 bytes (20 MB) per tracing instance. Link: https://lkml.kernel.org/r/1690568452-46553-10-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/inode.c | 18 ++++++++++++ include/linux/trace_events.h | 1 + kernel/trace/trace.h | 2 +- kernel/trace/trace_events.c | 65 ++++++++++++++++++++++---------------------- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index d9273066f25f..bb6de89eb446 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -374,6 +374,23 @@ static const struct super_operations tracefs_super_operations = { .show_options = tracefs_show_options, }; +static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) +{ + struct tracefs_inode *ti; + + if (!dentry || !inode) + return; + + ti = get_tracefs(inode); + if (ti && ti->flags & TRACEFS_EVENT_INODE) + eventfs_set_ef_status_free(dentry); + iput(inode); +} + +static const struct dentry_operations tracefs_dentry_operations = { + .d_iput = tracefs_dentry_iput, +}; + static int trace_fill_super(struct super_block *sb, void *data, int silent) { static const struct tree_descr trace_files[] = {{""}}; @@ -396,6 +413,7 @@ static int trace_fill_super(struct super_block *sb, void *data, int silent) goto fail; sb->s_op = &tracefs_super_operations; + sb->s_d_op = &tracefs_dentry_operations; tracefs_apply_options(sb, false); diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 3930e676436c..c17623c78029 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -638,6 +638,7 @@ struct trace_event_file { struct list_head list; struct trace_event_call *event_call; struct event_filter __rcu *filter; + struct eventfs_file *ef; struct dentry *dir; struct trace_array *tr; struct trace_subsystem_dir *system; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index ba7ababb8308..5b1f9e24764a 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1334,7 +1334,7 @@ struct trace_subsystem_dir { struct list_head list; struct event_subsystem *subsystem; struct trace_array *tr; - struct dentry *entry; + struct eventfs_file *ef; int ref_count; int nr_events; }; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 3ecc41f6acd9..ed367d713be0 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -984,7 +984,7 @@ static void remove_subsystem(struct trace_subsystem_dir *dir) return; if (!--dir->nr_events) { - tracefs_remove(dir->entry); + eventfs_remove(dir->ef); list_del(&dir->list); __put_system_dir(dir); } @@ -1005,7 +1005,7 @@ static void remove_event_file_dir(struct trace_event_file *file) tracefs_remove(dir); } - + eventfs_remove(file->ef); list_del(&file->list); remove_subsystem(file->system); free_event_filter(file->filter); @@ -2291,13 +2291,13 @@ create_new_subsystem(const char *name) return NULL; } -static struct dentry * +static struct eventfs_file * event_subsystem_dir(struct trace_array *tr, const char *name, struct trace_event_file *file, struct dentry *parent) { struct event_subsystem *system, *iter; struct trace_subsystem_dir *dir; - struct dentry *entry; + int res; /* First see if we did not already create this dir */ list_for_each_entry(dir, &tr->systems, list) { @@ -2305,7 +2305,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name, if (strcmp(system->name, name) == 0) { dir->nr_events++; file->system = dir; - return dir->entry; + return dir->ef; } } @@ -2329,8 +2329,8 @@ event_subsystem_dir(struct trace_array *tr, const char *name, } else __get_system(system); - dir->entry = tracefs_create_dir(name, parent); - if (!dir->entry) { + dir->ef = eventfs_add_subsystem_dir(name, parent); + if (IS_ERR(dir->ef)) { pr_warn("Failed to create system directory %s\n", name); __put_system(system); goto out_free; @@ -2345,22 +2345,22 @@ event_subsystem_dir(struct trace_array *tr, const char *name, /* the ftrace system is special, do not create enable or filter files */ if (strcmp(name, "ftrace") != 0) { - entry = tracefs_create_file("filter", TRACE_MODE_WRITE, - dir->entry, dir, + res = eventfs_add_file("filter", TRACE_MODE_WRITE, + dir->ef, dir, &ftrace_subsystem_filter_fops); - if (!entry) { + if (res) { kfree(system->filter); system->filter = NULL; pr_warn("Could not create tracefs '%s/filter' entry\n", name); } - trace_create_file("enable", TRACE_MODE_WRITE, dir->entry, dir, + eventfs_add_file("enable", TRACE_MODE_WRITE, dir->ef, dir, &ftrace_system_enable_fops); } list_add(&dir->list, &tr->systems); - return dir->entry; + return dir->ef; out_free: kfree(dir); @@ -2413,8 +2413,8 @@ static int event_create_dir(struct dentry *parent, struct trace_event_file *file) { struct trace_event_call *call = file->event_call; + struct eventfs_file *ef_subsystem = NULL; struct trace_array *tr = file->tr; - struct dentry *d_events; const char *name; int ret; @@ -2426,24 +2426,24 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0)) return -ENODEV; - d_events = event_subsystem_dir(tr, call->class->system, file, parent); - if (!d_events) + ef_subsystem = event_subsystem_dir(tr, call->class->system, file, parent); + if (!ef_subsystem) return -ENOMEM; name = trace_event_name(call); - file->dir = tracefs_create_dir(name, d_events); - if (!file->dir) { + file->ef = eventfs_add_dir(name, ef_subsystem); + if (IS_ERR(file->ef)) { pr_warn("Could not create tracefs '%s' directory\n", name); return -1; } if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) - trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file, + eventfs_add_file("enable", TRACE_MODE_WRITE, file->ef, file, &ftrace_enable_fops); #ifdef CONFIG_PERF_EVENTS if (call->event.type && call->class->reg) - trace_create_file("id", TRACE_MODE_READ, file->dir, + eventfs_add_file("id", TRACE_MODE_READ, file->ef, (void *)(long)call->event.type, &ftrace_event_id_fops); #endif @@ -2459,27 +2459,27 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) * triggers or filters. */ if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) { - trace_create_file("filter", TRACE_MODE_WRITE, file->dir, + eventfs_add_file("filter", TRACE_MODE_WRITE, file->ef, file, &ftrace_event_filter_fops); - trace_create_file("trigger", TRACE_MODE_WRITE, file->dir, + eventfs_add_file("trigger", TRACE_MODE_WRITE, file->ef, file, &event_trigger_fops); } #ifdef CONFIG_HIST_TRIGGERS - trace_create_file("hist", TRACE_MODE_READ, file->dir, file, + eventfs_add_file("hist", TRACE_MODE_READ, file->ef, file, &event_hist_fops); #endif #ifdef CONFIG_HIST_TRIGGERS_DEBUG - trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file, + eventfs_add_file("hist_debug", TRACE_MODE_READ, file->ef, file, &event_hist_debug_fops); #endif - trace_create_file("format", TRACE_MODE_READ, file->dir, call, + eventfs_add_file("format", TRACE_MODE_READ, file->ef, call, &ftrace_event_format_fops); #ifdef CONFIG_TRACE_EVENT_INJECT if (call->event.type && call->class->reg) - trace_create_file("inject", 0200, file->dir, file, + eventfs_add_file("inject", 0200, file->ef, file, &event_inject_fops); #endif @@ -3632,21 +3632,22 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) { struct dentry *d_events; struct dentry *entry; + int error = 0; entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent, tr, &ftrace_set_event_fops); if (!entry) return -ENOMEM; - d_events = tracefs_create_dir("events", parent); - if (!d_events) { + d_events = eventfs_create_events_dir("events", parent); + if (IS_ERR(d_events)) { pr_warn("Could not create tracefs 'events' directory\n"); return -ENOMEM; } - entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events, + error = eventfs_add_events_file("enable", TRACE_MODE_WRITE, d_events, tr, &ftrace_tr_enable_fops); - if (!entry) + if (error) return -ENOMEM; /* There are not as crucial, just warn if they are not created */ @@ -3659,11 +3660,11 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) &ftrace_set_event_notrace_pid_fops); /* ring buffer internal formats */ - trace_create_file("header_page", TRACE_MODE_READ, d_events, + eventfs_add_events_file("header_page", TRACE_MODE_READ, d_events, ring_buffer_print_page_header, &ftrace_show_header_fops); - trace_create_file("header_event", TRACE_MODE_READ, d_events, + eventfs_add_events_file("header_event", TRACE_MODE_READ, d_events, ring_buffer_print_entry_header, &ftrace_show_header_fops); @@ -3751,7 +3752,7 @@ int event_trace_del_tracer(struct trace_array *tr) down_write(&trace_event_sem); __trace_remove_event_dirs(tr); - tracefs_remove(tr->event_dir); + eventfs_remove_events_dir(tr->event_dir); up_write(&trace_event_sem); tr->event_dir = NULL; -- cgit v1.2.3-59-g8ed1b From 7c1130ea5cae215588e6d51242b877fd405e6c32 Mon Sep 17 00:00:00 2001 From: Ajay Kaher Date: Fri, 28 Jul 2023 23:50:52 +0530 Subject: test: ftrace: Fix kprobe test for eventfs kprobe_args_char.tc, kprobe_args_string.tc has validation check for tracefs_create_dir, for eventfs it should be eventfs_create_dir. Link: https://lkml.kernel.org/r/1690568452-46553-11-git-send-email-akaher@vmware.com Signed-off-by: Ajay Kaher Co-developed-by: Steven Rostedt (VMware) Signed-off-by: Steven Rostedt (VMware) Tested-by: Ching-lin Yu Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc | 9 +++++++-- .../testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc index 285b4770efad..ff7499eb98d6 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc @@ -34,14 +34,19 @@ mips*) esac : "Test get argument (1)" -echo "p:testprobe tracefs_create_dir arg1=+0(${ARG1}):char" > kprobe_events +if grep -q eventfs_add_dir available_filter_functions; then + DIR_NAME="eventfs_add_dir" +else + DIR_NAME="tracefs_create_dir" +fi +echo "p:testprobe ${DIR_NAME} arg1=+0(${ARG1}):char" > kprobe_events echo 1 > events/kprobes/testprobe/enable echo "p:test $FUNCTION_FORK" >> kprobe_events grep -qe "testprobe.* arg1='t'" trace echo 0 > events/kprobes/testprobe/enable : "Test get argument (2)" -echo "p:testprobe tracefs_create_dir arg1=+0(${ARG1}):char arg2=+0(${ARG1}):char[4]" > kprobe_events +echo "p:testprobe ${DIR_NAME} arg1=+0(${ARG1}):char arg2=+0(${ARG1}):char[4]" > kprobe_events echo 1 > events/kprobes/testprobe/enable echo "p:test $FUNCTION_FORK" >> kprobe_events grep -qe "testprobe.* arg1='t' arg2={'t','e','s','t'}" trace diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index a4f8e7c53c1f..a202b2ea4baf 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -37,14 +37,19 @@ loongarch*) esac : "Test get argument (1)" -echo "p:testprobe tracefs_create_dir arg1=+0(${ARG1}):string" > kprobe_events +if grep -q eventfs_add_dir available_filter_functions; then + DIR_NAME="eventfs_add_dir" +else + DIR_NAME="tracefs_create_dir" +fi +echo "p:testprobe ${DIR_NAME} arg1=+0(${ARG1}):string" > kprobe_events echo 1 > events/kprobes/testprobe/enable echo "p:test $FUNCTION_FORK" >> kprobe_events grep -qe "testprobe.* arg1=\"test\"" trace echo 0 > events/kprobes/testprobe/enable : "Test get argument (2)" -echo "p:testprobe tracefs_create_dir arg1=+0(${ARG1}):string arg2=+0(${ARG1}):string" > kprobe_events +echo "p:testprobe ${DIR_NAME} arg1=+0(${ARG1}):string arg2=+0(${ARG1}):string" > kprobe_events echo 1 > events/kprobes/testprobe/enable echo "p:test $FUNCTION_FORK" >> kprobe_events grep -qe "testprobe.* arg1=\"test\" arg2=\"test\"" trace -- cgit v1.2.3-59-g8ed1b From cfb58e278c6a6b167e24f5820fce9f63bf802b30 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:47 +0100 Subject: tracing/filters: Dynamically allocate filter_pred.regex Every predicate allocation includes a MAX_FILTER_STR_VAL (256) char array in the regex field, even if the predicate function does not use the field. A later commit will introduce a dynamically allocated cpumask to struct filter_pred, which will require a dedicated freeing function. Bite the bullet and make filter_pred.regex dynamically allocated. While at it, reorder the fields of filter_pred to fill in the byte holes. The struct now fits on a single cacheline. No change in behaviour intended. The kfree()'s were patched via Coccinelle: @@ struct filter_pred *pred; @@ -kfree(pred); +free_predicate(pred); Link: https://lkml.kernel.org/r/20230707172155.70873-2-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 64 +++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 1dad64267878..91fc9990107f 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -70,15 +70,15 @@ enum filter_pred_fn { }; struct filter_pred { - enum filter_pred_fn fn_num; - u64 val; - u64 val2; - struct regex regex; + struct regex *regex; unsigned short *ops; struct ftrace_event_field *field; - int offset; + u64 val; + u64 val2; + enum filter_pred_fn fn_num; + int offset; int not; - int op; + int op; }; /* @@ -186,6 +186,14 @@ enum { PROCESS_OR = 4, }; +static void free_predicate(struct filter_pred *pred) +{ + if (pred) { + kfree(pred->regex); + kfree(pred); + } +} + /* * Without going into a formal proof, this explains the method that is used in * parsing the logical expressions. @@ -623,7 +631,7 @@ out_free: kfree(inverts); if (prog_stack) { for (i = 0; prog_stack[i].pred; i++) - kfree(prog_stack[i].pred); + free_predicate(prog_stack[i].pred); kfree(prog_stack); } return ERR_PTR(ret); @@ -750,7 +758,7 @@ static int filter_pred_string(struct filter_pred *pred, void *event) char *addr = (char *)(event + pred->offset); int cmp, match; - cmp = pred->regex.match(addr, &pred->regex, pred->regex.field_len); + cmp = pred->regex->match(addr, pred->regex, pred->regex->field_len); match = cmp ^ pred->not; @@ -763,7 +771,7 @@ static __always_inline int filter_pchar(struct filter_pred *pred, char *str) int len; len = strlen(str) + 1; /* including tailing '\0' */ - cmp = pred->regex.match(str, &pred->regex, len); + cmp = pred->regex->match(str, pred->regex, len); match = cmp ^ pred->not; @@ -813,7 +821,7 @@ static int filter_pred_strloc(struct filter_pred *pred, void *event) char *addr = (char *)(event + str_loc); int cmp, match; - cmp = pred->regex.match(addr, &pred->regex, str_len); + cmp = pred->regex->match(addr, pred->regex, str_len); match = cmp ^ pred->not; @@ -836,7 +844,7 @@ static int filter_pred_strrelloc(struct filter_pred *pred, void *event) char *addr = (char *)(&item[1]) + str_loc; int cmp, match; - cmp = pred->regex.match(addr, &pred->regex, str_len); + cmp = pred->regex->match(addr, pred->regex, str_len); match = cmp ^ pred->not; @@ -874,7 +882,7 @@ static int filter_pred_comm(struct filter_pred *pred, void *event) { int cmp; - cmp = pred->regex.match(current->comm, &pred->regex, + cmp = pred->regex->match(current->comm, pred->regex, TASK_COMM_LEN); return cmp ^ pred->not; } @@ -1004,7 +1012,7 @@ enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not) static void filter_build_regex(struct filter_pred *pred) { - struct regex *r = &pred->regex; + struct regex *r = pred->regex; char *search; enum regex_type type = MATCH_FULL; @@ -1169,7 +1177,7 @@ static void free_prog(struct event_filter *filter) return; for (i = 0; prog[i].pred; i++) - kfree(prog[i].pred); + free_predicate(prog[i].pred); kfree(prog); } @@ -1553,9 +1561,12 @@ static int parse_pred(const char *str, void *data, goto err_free; } - pred->regex.len = len; - strncpy(pred->regex.pattern, str + s, len); - pred->regex.pattern[len] = 0; + pred->regex = kzalloc(sizeof(*pred->regex), GFP_KERNEL); + if (!pred->regex) + goto err_mem; + pred->regex->len = len; + strncpy(pred->regex->pattern, str + s, len); + pred->regex->pattern[len] = 0; /* This is either a string, or an integer */ } else if (str[i] == '\'' || str[i] == '"') { @@ -1597,9 +1608,12 @@ static int parse_pred(const char *str, void *data, goto err_free; } - pred->regex.len = len; - strncpy(pred->regex.pattern, str + s, len); - pred->regex.pattern[len] = 0; + pred->regex = kzalloc(sizeof(*pred->regex), GFP_KERNEL); + if (!pred->regex) + goto err_mem; + pred->regex->len = len; + strncpy(pred->regex->pattern, str + s, len); + pred->regex->pattern[len] = 0; filter_build_regex(pred); @@ -1608,7 +1622,7 @@ static int parse_pred(const char *str, void *data, } else if (field->filter_type == FILTER_STATIC_STRING) { pred->fn_num = FILTER_PRED_FN_STRING; - pred->regex.field_len = field->size; + pred->regex->field_len = field->size; } else if (field->filter_type == FILTER_DYN_STRING) { pred->fn_num = FILTER_PRED_FN_STRLOC; @@ -1691,10 +1705,10 @@ static int parse_pred(const char *str, void *data, return i; err_free: - kfree(pred); + free_predicate(pred); return -EINVAL; err_mem: - kfree(pred); + free_predicate(pred); return -ENOMEM; } @@ -2287,8 +2301,8 @@ static int ftrace_function_set_filter_pred(struct filter_pred *pred, return ret; return __ftrace_function_set_filter(pred->op == OP_EQ, - pred->regex.pattern, - pred->regex.len, + pred->regex->pattern, + pred->regex->len, data); } -- cgit v1.2.3-59-g8ed1b From 39f7c41c908bc82947ce47cdd021520ff486c53f Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:48 +0100 Subject: tracing/filters: Enable filtering a cpumask field by another cpumask The recently introduced ipi_send_cpumask trace event contains a cpumask field, but it currently cannot be used in filter expressions. Make event filtering aware of cpumask fields, and allow these to be filtered by a user-provided cpumask. The user-provided cpumask is to be given in cpulist format and wrapped as: "CPUS{$cpulist}". The use of curly braces instead of parentheses is to prevent predicate_parse() from parsing the contents of CPUS{...} as a full-fledged predicate subexpression. This enables e.g.: $ trace-cmd record -e 'ipi_send_cpumask' -f 'cpumask & CPUS{2,4,6,8-32}' Link: https://lkml.kernel.org/r/20230707172155.70873-3-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_events.h | 1 + kernel/trace/trace_events_filter.c | 97 +++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index c17623c78029..1600aeb8e1a3 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -808,6 +808,7 @@ enum { FILTER_RDYN_STRING, FILTER_PTR_STRING, FILTER_TRACE_FN, + FILTER_CPUMASK, FILTER_COMM, FILTER_CPU, FILTER_STACKTRACE, diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 91fc9990107f..cb1863dfa280 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -64,6 +64,7 @@ enum filter_pred_fn { FILTER_PRED_FN_PCHAR_USER, FILTER_PRED_FN_PCHAR, FILTER_PRED_FN_CPU, + FILTER_PRED_FN_CPUMASK, FILTER_PRED_FN_FUNCTION, FILTER_PRED_FN_, FILTER_PRED_TEST_VISITED, @@ -71,6 +72,7 @@ enum filter_pred_fn { struct filter_pred { struct regex *regex; + struct cpumask *mask; unsigned short *ops; struct ftrace_event_field *field; u64 val; @@ -94,6 +96,8 @@ struct filter_pred { C(TOO_MANY_OPEN, "Too many '('"), \ C(TOO_MANY_CLOSE, "Too few '('"), \ C(MISSING_QUOTE, "Missing matching quote"), \ + C(MISSING_BRACE_OPEN, "Missing '{'"), \ + C(MISSING_BRACE_CLOSE, "Missing '}'"), \ C(OPERAND_TOO_LONG, "Operand too long"), \ C(EXPECT_STRING, "Expecting string field"), \ C(EXPECT_DIGIT, "Expecting numeric field"), \ @@ -103,6 +107,7 @@ struct filter_pred { C(BAD_SUBSYS_FILTER, "Couldn't find or set field in one of a subsystem's events"), \ C(TOO_MANY_PREDS, "Too many terms in predicate expression"), \ C(INVALID_FILTER, "Meaningless filter expression"), \ + C(INVALID_CPULIST, "Invalid cpulist"), \ C(IP_FIELD_ONLY, "Only 'ip' field is supported for function trace"), \ C(INVALID_VALUE, "Invalid value (did you forget quotes)?"), \ C(NO_FUNCTION, "Function not found"), \ @@ -190,6 +195,7 @@ static void free_predicate(struct filter_pred *pred) { if (pred) { kfree(pred->regex); + kfree(pred->mask); kfree(pred); } } @@ -877,6 +883,26 @@ static int filter_pred_cpu(struct filter_pred *pred, void *event) } } +/* Filter predicate for cpumask field vs user-provided cpumask */ +static int filter_pred_cpumask(struct filter_pred *pred, void *event) +{ + u32 item = *(u32 *)(event + pred->offset); + int loc = item & 0xffff; + const struct cpumask *mask = (event + loc); + const struct cpumask *cmp = pred->mask; + + switch (pred->op) { + case OP_EQ: + return cpumask_equal(mask, cmp); + case OP_NE: + return !cpumask_equal(mask, cmp); + case OP_BAND: + return cpumask_intersects(mask, cmp); + default: + return 0; + } +} + /* Filter predicate for COMM. */ static int filter_pred_comm(struct filter_pred *pred, void *event) { @@ -1244,8 +1270,12 @@ static void filter_free_subsystem_filters(struct trace_subsystem_dir *dir, int filter_assign_type(const char *type) { - if (strstr(type, "__data_loc") && strstr(type, "char")) - return FILTER_DYN_STRING; + if (strstr(type, "__data_loc")) { + if (strstr(type, "char")) + return FILTER_DYN_STRING; + if (strstr(type, "cpumask_t")) + return FILTER_CPUMASK; + } if (strstr(type, "__rel_loc") && strstr(type, "char")) return FILTER_RDYN_STRING; @@ -1357,6 +1387,8 @@ static int filter_pred_fn_call(struct filter_pred *pred, void *event) return filter_pred_pchar(pred, event); case FILTER_PRED_FN_CPU: return filter_pred_cpu(pred, event); + case FILTER_PRED_FN_CPUMASK: + return filter_pred_cpumask(pred, event); case FILTER_PRED_FN_FUNCTION: return filter_pred_function(pred, event); case FILTER_PRED_TEST_VISITED: @@ -1568,6 +1600,67 @@ static int parse_pred(const char *str, void *data, strncpy(pred->regex->pattern, str + s, len); pred->regex->pattern[len] = 0; + } else if (!strncmp(str + i, "CPUS", 4)) { + unsigned int maskstart; + char *tmp; + + switch (field->filter_type) { + case FILTER_CPUMASK: + break; + default: + parse_error(pe, FILT_ERR_ILLEGAL_FIELD_OP, pos + i); + goto err_free; + } + + switch (op) { + case OP_EQ: + case OP_NE: + case OP_BAND: + break; + default: + parse_error(pe, FILT_ERR_ILLEGAL_FIELD_OP, pos + i); + goto err_free; + } + + /* Skip CPUS */ + i += 4; + if (str[i++] != '{') { + parse_error(pe, FILT_ERR_MISSING_BRACE_OPEN, pos + i); + goto err_free; + } + maskstart = i; + + /* Walk the cpulist until closing } */ + for (; str[i] && str[i] != '}'; i++); + if (str[i] != '}') { + parse_error(pe, FILT_ERR_MISSING_BRACE_CLOSE, pos + i); + goto err_free; + } + + if (maskstart == i) { + parse_error(pe, FILT_ERR_INVALID_CPULIST, pos + i); + goto err_free; + } + + /* Copy the cpulist between { and } */ + tmp = kmalloc((i - maskstart) + 1, GFP_KERNEL); + strscpy(tmp, str + maskstart, (i - maskstart) + 1); + + pred->mask = kzalloc(cpumask_size(), GFP_KERNEL); + if (!pred->mask) + goto err_mem; + + /* Now parse it */ + if (cpulist_parse(tmp, pred->mask)) { + parse_error(pe, FILT_ERR_INVALID_CPULIST, pos + i); + goto err_free; + } + + /* Move along */ + i++; + if (field->filter_type == FILTER_CPUMASK) + pred->fn_num = FILTER_PRED_FN_CPUMASK; + /* This is either a string, or an integer */ } else if (str[i] == '\'' || str[i] == '"') { char q = str[i]; -- cgit v1.2.3-59-g8ed1b From 3cbec9d7b93d33082cfbbf20fab7ccadd29075ef Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:49 +0100 Subject: tracing/filters: Enable filtering a scalar field by a cpumask Several events use a scalar field to denote a CPU: o sched_wakeup.target_cpu o sched_migrate_task.orig_cpu,dest_cpu o sched_move_numa.src_cpu,dst_cpu o ipi_send_cpu.cpu o ... Filtering these currently requires using arithmetic comparison functions, which can be tedious when dealing with interleaved SMT or NUMA CPU ids. Allow these to be filtered by a user-provided cpumask, which enables e.g.: $ trace-cmd record -e 'sched_wakeup' -f 'target_cpu & CPUS{2,4,6,8-32}' Link: https://lkml.kernel.org/r/20230707172155.70873-4-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 92 +++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index cb1863dfa280..1e14f801685a 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -46,15 +46,19 @@ static const char * ops[] = { OPS }; enum filter_pred_fn { FILTER_PRED_FN_NOP, FILTER_PRED_FN_64, + FILTER_PRED_FN_64_CPUMASK, FILTER_PRED_FN_S64, FILTER_PRED_FN_U64, FILTER_PRED_FN_32, + FILTER_PRED_FN_32_CPUMASK, FILTER_PRED_FN_S32, FILTER_PRED_FN_U32, FILTER_PRED_FN_16, + FILTER_PRED_FN_16_CPUMASK, FILTER_PRED_FN_S16, FILTER_PRED_FN_U16, FILTER_PRED_FN_8, + FILTER_PRED_FN_8_CPUMASK, FILTER_PRED_FN_S8, FILTER_PRED_FN_U8, FILTER_PRED_FN_COMM, @@ -643,6 +647,39 @@ out_free: return ERR_PTR(ret); } +static inline int +do_filter_cpumask(int op, const struct cpumask *mask, const struct cpumask *cmp) +{ + switch (op) { + case OP_EQ: + return cpumask_equal(mask, cmp); + case OP_NE: + return !cpumask_equal(mask, cmp); + case OP_BAND: + return cpumask_intersects(mask, cmp); + default: + return 0; + } +} + +/* Optimisation of do_filter_cpumask() for scalar fields */ +static inline int +do_filter_scalar_cpumask(int op, unsigned int cpu, const struct cpumask *mask) +{ + switch (op) { + case OP_EQ: + return cpumask_test_cpu(cpu, mask) && + cpumask_nth(1, mask) >= nr_cpu_ids; + case OP_NE: + return !cpumask_test_cpu(cpu, mask) || + cpumask_nth(1, mask) < nr_cpu_ids; + case OP_BAND: + return cpumask_test_cpu(cpu, mask); + default: + return 0; + } +} + enum pred_cmp_types { PRED_CMP_TYPE_NOP, PRED_CMP_TYPE_LT, @@ -686,6 +723,18 @@ static int filter_pred_##type(struct filter_pred *pred, void *event) \ } \ } +#define DEFINE_CPUMASK_COMPARISON_PRED(size) \ +static int filter_pred_##size##_cpumask(struct filter_pred *pred, void *event) \ +{ \ + u##size *addr = (u##size *)(event + pred->offset); \ + unsigned int cpu = *addr; \ + \ + if (cpu >= nr_cpu_ids) \ + return 0; \ + \ + return do_filter_scalar_cpumask(pred->op, cpu, pred->mask); \ +} + #define DEFINE_EQUALITY_PRED(size) \ static int filter_pred_##size(struct filter_pred *pred, void *event) \ { \ @@ -707,6 +756,11 @@ DEFINE_COMPARISON_PRED(u16); DEFINE_COMPARISON_PRED(s8); DEFINE_COMPARISON_PRED(u8); +DEFINE_CPUMASK_COMPARISON_PRED(64); +DEFINE_CPUMASK_COMPARISON_PRED(32); +DEFINE_CPUMASK_COMPARISON_PRED(16); +DEFINE_CPUMASK_COMPARISON_PRED(8); + DEFINE_EQUALITY_PRED(64); DEFINE_EQUALITY_PRED(32); DEFINE_EQUALITY_PRED(16); @@ -891,16 +945,7 @@ static int filter_pred_cpumask(struct filter_pred *pred, void *event) const struct cpumask *mask = (event + loc); const struct cpumask *cmp = pred->mask; - switch (pred->op) { - case OP_EQ: - return cpumask_equal(mask, cmp); - case OP_NE: - return !cpumask_equal(mask, cmp); - case OP_BAND: - return cpumask_intersects(mask, cmp); - default: - return 0; - } + return do_filter_cpumask(pred->op, mask, cmp); } /* Filter predicate for COMM. */ @@ -1351,24 +1396,32 @@ static int filter_pred_fn_call(struct filter_pred *pred, void *event) switch (pred->fn_num) { case FILTER_PRED_FN_64: return filter_pred_64(pred, event); + case FILTER_PRED_FN_64_CPUMASK: + return filter_pred_64_cpumask(pred, event); case FILTER_PRED_FN_S64: return filter_pred_s64(pred, event); case FILTER_PRED_FN_U64: return filter_pred_u64(pred, event); case FILTER_PRED_FN_32: return filter_pred_32(pred, event); + case FILTER_PRED_FN_32_CPUMASK: + return filter_pred_32_cpumask(pred, event); case FILTER_PRED_FN_S32: return filter_pred_s32(pred, event); case FILTER_PRED_FN_U32: return filter_pred_u32(pred, event); case FILTER_PRED_FN_16: return filter_pred_16(pred, event); + case FILTER_PRED_FN_16_CPUMASK: + return filter_pred_16_cpumask(pred, event); case FILTER_PRED_FN_S16: return filter_pred_s16(pred, event); case FILTER_PRED_FN_U16: return filter_pred_u16(pred, event); case FILTER_PRED_FN_8: return filter_pred_8(pred, event); + case FILTER_PRED_FN_8_CPUMASK: + return filter_pred_8_cpumask(pred, event); case FILTER_PRED_FN_S8: return filter_pred_s8(pred, event); case FILTER_PRED_FN_U8: @@ -1606,6 +1659,7 @@ static int parse_pred(const char *str, void *data, switch (field->filter_type) { case FILTER_CPUMASK: + case FILTER_OTHER: break; default: parse_error(pe, FILT_ERR_ILLEGAL_FIELD_OP, pos + i); @@ -1658,8 +1712,24 @@ static int parse_pred(const char *str, void *data, /* Move along */ i++; - if (field->filter_type == FILTER_CPUMASK) + if (field->filter_type == FILTER_CPUMASK) { pred->fn_num = FILTER_PRED_FN_CPUMASK; + } else { + switch (field->size) { + case 8: + pred->fn_num = FILTER_PRED_FN_64_CPUMASK; + break; + case 4: + pred->fn_num = FILTER_PRED_FN_32_CPUMASK; + break; + case 2: + pred->fn_num = FILTER_PRED_FN_16_CPUMASK; + break; + case 1: + pred->fn_num = FILTER_PRED_FN_8_CPUMASK; + break; + } + } /* This is either a string, or an integer */ } else if (str[i] == '\'' || str[i] == '"') { -- cgit v1.2.3-59-g8ed1b From 347d24fc8286e766696cfed112d6a353cb62cb68 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:50 +0100 Subject: tracing/filters: Enable filtering the CPU common field by a cpumask The tracing_cpumask lets us specify which CPUs are traced in a buffer instance, but doesn't let us do this on a per-event basis (unless one creates an instance per event). A previous commit added filtering scalar fields by a user-given cpumask, make this work with the CPU common field as well. This enables doing things like $ trace-cmd record -e 'sched_switch' -f 'CPU & CPUS{12-52}' \ -e 'sched_wakeup' -f 'target_cpu & CPUS{12-52}' Link: https://lkml.kernel.org/r/20230707172155.70873-5-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 1e14f801685a..3009d0c61b53 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -68,6 +68,7 @@ enum filter_pred_fn { FILTER_PRED_FN_PCHAR_USER, FILTER_PRED_FN_PCHAR, FILTER_PRED_FN_CPU, + FILTER_PRED_FN_CPU_CPUMASK, FILTER_PRED_FN_CPUMASK, FILTER_PRED_FN_FUNCTION, FILTER_PRED_FN_, @@ -937,6 +938,14 @@ static int filter_pred_cpu(struct filter_pred *pred, void *event) } } +/* Filter predicate for current CPU vs user-provided cpumask */ +static int filter_pred_cpu_cpumask(struct filter_pred *pred, void *event) +{ + int cpu = raw_smp_processor_id(); + + return do_filter_scalar_cpumask(pred->op, cpu, pred->mask); +} + /* Filter predicate for cpumask field vs user-provided cpumask */ static int filter_pred_cpumask(struct filter_pred *pred, void *event) { @@ -1440,6 +1449,8 @@ static int filter_pred_fn_call(struct filter_pred *pred, void *event) return filter_pred_pchar(pred, event); case FILTER_PRED_FN_CPU: return filter_pred_cpu(pred, event); + case FILTER_PRED_FN_CPU_CPUMASK: + return filter_pred_cpu_cpumask(pred, event); case FILTER_PRED_FN_CPUMASK: return filter_pred_cpumask(pred, event); case FILTER_PRED_FN_FUNCTION: @@ -1659,6 +1670,7 @@ static int parse_pred(const char *str, void *data, switch (field->filter_type) { case FILTER_CPUMASK: + case FILTER_CPU: case FILTER_OTHER: break; default: @@ -1714,6 +1726,8 @@ static int parse_pred(const char *str, void *data, i++; if (field->filter_type == FILTER_CPUMASK) { pred->fn_num = FILTER_PRED_FN_CPUMASK; + } else if (field->filter_type == FILTER_CPU) { + pred->fn_num = FILTER_PRED_FN_CPU_CPUMASK; } else { switch (field->size) { case 8: -- cgit v1.2.3-59-g8ed1b From fe4fa4ec9b464f5a93b2010247822fdd3a2a2115 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:51 +0100 Subject: tracing/filters: Optimise cpumask vs cpumask filtering when user mask is a single CPU Steven noted that when the user-provided cpumask contains a single CPU, then the filtering function can use a scalar as input instead of a full-fledged cpumask. Reuse do_filter_scalar_cpumask() when the input mask has a weight of one. Link: https://lkml.kernel.org/r/20230707172155.70873-6-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Suggested-by: Steven Rostedt Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 3009d0c61b53..2fe65ddeb34e 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -70,6 +70,7 @@ enum filter_pred_fn { FILTER_PRED_FN_CPU, FILTER_PRED_FN_CPU_CPUMASK, FILTER_PRED_FN_CPUMASK, + FILTER_PRED_FN_CPUMASK_CPU, FILTER_PRED_FN_FUNCTION, FILTER_PRED_FN_, FILTER_PRED_TEST_VISITED, @@ -957,6 +958,22 @@ static int filter_pred_cpumask(struct filter_pred *pred, void *event) return do_filter_cpumask(pred->op, mask, cmp); } +/* Filter predicate for cpumask field vs user-provided scalar */ +static int filter_pred_cpumask_cpu(struct filter_pred *pred, void *event) +{ + u32 item = *(u32 *)(event + pred->offset); + int loc = item & 0xffff; + const struct cpumask *mask = (event + loc); + unsigned int cpu = pred->val; + + /* + * This inverts the usual usage of the function (field is first element, + * user parameter is second), but that's fine because the (scalar, mask) + * operations used are symmetric. + */ + return do_filter_scalar_cpumask(pred->op, cpu, mask); +} + /* Filter predicate for COMM. */ static int filter_pred_comm(struct filter_pred *pred, void *event) { @@ -1453,6 +1470,8 @@ static int filter_pred_fn_call(struct filter_pred *pred, void *event) return filter_pred_cpu_cpumask(pred, event); case FILTER_PRED_FN_CPUMASK: return filter_pred_cpumask(pred, event); + case FILTER_PRED_FN_CPUMASK_CPU: + return filter_pred_cpumask_cpu(pred, event); case FILTER_PRED_FN_FUNCTION: return filter_pred_function(pred, event); case FILTER_PRED_TEST_VISITED: @@ -1666,6 +1685,7 @@ static int parse_pred(const char *str, void *data, } else if (!strncmp(str + i, "CPUS", 4)) { unsigned int maskstart; + bool single; char *tmp; switch (field->filter_type) { @@ -1724,8 +1744,21 @@ static int parse_pred(const char *str, void *data, /* Move along */ i++; + + /* + * Optimisation: if the user-provided mask has a weight of one + * then we can treat it as a scalar input. + */ + single = cpumask_weight(pred->mask) == 1; + if (single && field->filter_type == FILTER_CPUMASK) { + pred->val = cpumask_first(pred->mask); + kfree(pred->mask); + } + if (field->filter_type == FILTER_CPUMASK) { - pred->fn_num = FILTER_PRED_FN_CPUMASK; + pred->fn_num = single ? + FILTER_PRED_FN_CPUMASK_CPU : + FILTER_PRED_FN_CPUMASK; } else if (field->filter_type == FILTER_CPU) { pred->fn_num = FILTER_PRED_FN_CPU_CPUMASK; } else { -- cgit v1.2.3-59-g8ed1b From ca77dd8ce4658d1006d747dd1995ebb9375199c8 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:52 +0100 Subject: tracing/filters: Optimise scalar vs cpumask filtering when the user mask is a single CPU Steven noted that when the user-provided cpumask contains a single CPU, then the filtering function can use a scalar as input instead of a full-fledged cpumask. When the mask contains a single CPU, directly re-use the unsigned field predicate functions. Transform '&' into '==' beforehand. Link: https://lkml.kernel.org/r/20230707172155.70873-7-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Suggested-by: Steven Rostedt Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 2fe65ddeb34e..54d642fabb7f 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1750,7 +1750,7 @@ static int parse_pred(const char *str, void *data, * then we can treat it as a scalar input. */ single = cpumask_weight(pred->mask) == 1; - if (single && field->filter_type == FILTER_CPUMASK) { + if (single && field->filter_type != FILTER_CPU) { pred->val = cpumask_first(pred->mask); kfree(pred->mask); } @@ -1761,6 +1761,11 @@ static int parse_pred(const char *str, void *data, FILTER_PRED_FN_CPUMASK; } else if (field->filter_type == FILTER_CPU) { pred->fn_num = FILTER_PRED_FN_CPU_CPUMASK; + } else if (single) { + pred->op = pred->op == OP_BAND ? OP_EQ : pred->op; + pred->fn_num = select_comparison_fn(pred->op, field->size, false); + if (pred->op == OP_NE) + pred->not = 1; } else { switch (field->size) { case 8: -- cgit v1.2.3-59-g8ed1b From 1cffbe6c62f10e200813ef19caea49bbde71d474 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:53 +0100 Subject: tracing/filters: Optimise CPU vs cpumask filtering when the user mask is a single CPU Steven noted that when the user-provided cpumask contains a single CPU, then the filtering function can use a scalar as input instead of a full-fledged cpumask. In this case we can directly re-use filter_pred_cpu(), we just need to transform '&' into '==' before executing it. Link: https://lkml.kernel.org/r/20230707172155.70873-8-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Suggested-by: Steven Rostedt Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 54d642fabb7f..fd72dacc5d1b 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1750,7 +1750,7 @@ static int parse_pred(const char *str, void *data, * then we can treat it as a scalar input. */ single = cpumask_weight(pred->mask) == 1; - if (single && field->filter_type != FILTER_CPU) { + if (single) { pred->val = cpumask_first(pred->mask); kfree(pred->mask); } @@ -1760,7 +1760,12 @@ static int parse_pred(const char *str, void *data, FILTER_PRED_FN_CPUMASK_CPU : FILTER_PRED_FN_CPUMASK; } else if (field->filter_type == FILTER_CPU) { - pred->fn_num = FILTER_PRED_FN_CPU_CPUMASK; + if (single) { + pred->op = pred->op == OP_BAND ? OP_EQ : pred->op; + pred->fn_num = FILTER_PRED_FN_CPU; + } else { + pred->fn_num = FILTER_PRED_FN_CPU_CPUMASK; + } } else if (single) { pred->op = pred->op == OP_BAND ? OP_EQ : pred->op; pred->fn_num = select_comparison_fn(pred->op, field->size, false); -- cgit v1.2.3-59-g8ed1b From 38c6f68083895e5e732239dbb277e87fa76f5cd3 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:54 +0100 Subject: tracing/filters: Further optimise scalar vs cpumask comparison Per the previous commits, we now only enter do_filter_scalar_cpumask() with a mask of weight greater than one. Optimise the equality checks. Link: https://lkml.kernel.org/r/20230707172155.70873-9-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Suggested-by: Steven Rostedt Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index fd72dacc5d1b..3a529214a21b 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -667,6 +667,25 @@ do_filter_cpumask(int op, const struct cpumask *mask, const struct cpumask *cmp) /* Optimisation of do_filter_cpumask() for scalar fields */ static inline int do_filter_scalar_cpumask(int op, unsigned int cpu, const struct cpumask *mask) +{ + /* + * Per the weight-of-one cpumask optimisations, the mask passed in this + * function has a weight >= 2, so it is never equal to a single scalar. + */ + switch (op) { + case OP_EQ: + return false; + case OP_NE: + return true; + case OP_BAND: + return cpumask_test_cpu(cpu, mask); + default: + return 0; + } +} + +static inline int +do_filter_cpumask_scalar(int op, const struct cpumask *mask, unsigned int cpu) { switch (op) { case OP_EQ: @@ -966,12 +985,7 @@ static int filter_pred_cpumask_cpu(struct filter_pred *pred, void *event) const struct cpumask *mask = (event + loc); unsigned int cpu = pred->val; - /* - * This inverts the usual usage of the function (field is first element, - * user parameter is second), but that's fine because the (scalar, mask) - * operations used are symmetric. - */ - return do_filter_scalar_cpumask(pred->op, cpu, mask); + return do_filter_cpumask_scalar(pred->op, mask, cpu); } /* Filter predicate for COMM. */ -- cgit v1.2.3-59-g8ed1b From fa828efb9cb3afae46163b02988fbb4fff1122bc Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 7 Jul 2023 18:21:55 +0100 Subject: tracing/filters: Document cpumask filtering Cpumask, scalar and CPU fields can now be filtered by a user-provided cpumask, document the syntax. Link: https://lkml.kernel.org/r/20230707172155.70873-10-vschneid@redhat.com Cc: Masami Hiramatsu Cc: Jonathan Corbet Cc: Juri Lelli Cc: Daniel Bristot de Oliveira Cc: Marcelo Tosatti Cc: Leonardo Bras Cc: Frederic Weisbecker Signed-off-by: Valentin Schneider Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/events.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Documentation/trace/events.rst b/Documentation/trace/events.rst index f5fcb8e1218f..34108d5a55b4 100644 --- a/Documentation/trace/events.rst +++ b/Documentation/trace/events.rst @@ -219,6 +219,20 @@ the function "security_prepare_creds" and less than the end of that function. The ".function" postfix can only be attached to values of size long, and can only be compared with "==" or "!=". +Cpumask fields or scalar fields that encode a CPU number can be filtered using +a user-provided cpumask in cpulist format. The format is as follows:: + + CPUS{$cpulist} + +Operators available to cpumask filtering are: + +& (intersection), ==, != + +For example, this will filter events that have their .target_cpu field present +in the given cpumask:: + + target_cpu & CPUS{17-42} + 5.2 Setting filters ------------------- -- cgit v1.2.3-59-g8ed1b From efde97a175e89fceb36cd2c352700661183992bc Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Thu, 3 Aug 2023 22:40:28 +0800 Subject: tracing: Remove unused function declarations Commit 9457158bbc0e ("tracing: Fix reset of time stamps during trace_clock changes") left behind tracing_reset_current() declaration. Also commit 6954e415264e ("tracing: Place trace_pid_list logic into abstract functions") removed trace_free_pid_list() implementation but leave declaration. Link: https://lore.kernel.org/linux-trace-kernel/20230803144028.25492-1-yuehaibing@huawei.com Cc: Signed-off-by: Yue Haibing Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 5b1f9e24764a..b6e44a39b4ce 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -604,7 +604,6 @@ trace_buffer_iter(struct trace_iterator *iter, int cpu) int tracer_init(struct tracer *t, struct trace_array *tr); int tracing_is_enabled(void); void tracing_reset_online_cpus(struct array_buffer *buf); -void tracing_reset_current(int cpu); void tracing_reset_all_online_cpus(void); void tracing_reset_all_online_cpus_unlocked(void); int tracing_open_generic(struct inode *inode, struct file *filp); @@ -705,7 +704,6 @@ void trace_filter_add_remove_task(struct trace_pid_list *pid_list, void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos); void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos); int trace_pid_show(struct seq_file *m, void *v); -void trace_free_pid_list(struct trace_pid_list *pid_list); int trace_pid_write(struct trace_pid_list *filtered_pids, struct trace_pid_list **new_pid_list, const char __user *ubuf, size_t cnt); -- cgit v1.2.3-59-g8ed1b From c8f05f2f41d7e4950d6767e6ebe9fba3dcaf45ff Mon Sep 17 00:00:00 2001 From: Zhang Zekun Date: Fri, 4 Aug 2023 09:36:36 +0800 Subject: ftrace: Remove empty declaration ftrace_enable_daemon() and ftrace_disable_daemon() The definition of ftrace_enable_daemon() and ftrace_disable_daemon() has been removed since commit cb7be3b2fc2c ("ftrace: remove daemon"), remain the declarations in the header files, so remove it. Link: https://lore.kernel.org/linux-trace-kernel/20230804013636.115940-1-zhangzekun11@huawei.com Cc: Cc: Signed-off-by: Zhang Zekun Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index aad9cf8876b5..e8921871ef9a 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -862,13 +862,8 @@ extern int skip_trace(unsigned long ip); extern void ftrace_module_init(struct module *mod); extern void ftrace_module_enable(struct module *mod); extern void ftrace_release_mod(struct module *mod); - -extern void ftrace_disable_daemon(void); -extern void ftrace_enable_daemon(void); #else /* CONFIG_DYNAMIC_FTRACE */ static inline int skip_trace(unsigned long ip) { return 0; } -static inline void ftrace_disable_daemon(void) { } -static inline void ftrace_enable_daemon(void) { } static inline void ftrace_module_init(struct module *mod) { } static inline void ftrace_module_enable(struct module *mod) { } static inline void ftrace_release_mod(struct module *mod) { } -- cgit v1.2.3-59-g8ed1b From a943188dabf78585fc5ddae0803b8b2c94f1c824 Mon Sep 17 00:00:00 2001 From: Eric Vaughn Date: Thu, 10 Aug 2023 12:43:37 -0700 Subject: tracing/user_events: Optimize safe list traversals Several of the list traversals in the user_events facility use safe list traversals where they could be using the unsafe versions instead. Replace these safe traversals with their unsafe counterparts in the interest of optimization. Link: https://lore.kernel.org/linux-trace-kernel/20230810194337.695983-1-ervaughn@linux.microsoft.com Suggested-by: Beau Belgrave Signed-off-by: Eric Vaughn Acked-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_user.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 33cb6af31f39..6f046650e527 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -1328,14 +1328,14 @@ static int user_field_set_string(struct ftrace_event_field *field, static int user_event_set_print_fmt(struct user_event *user, char *buf, int len) { - struct ftrace_event_field *field, *next; + struct ftrace_event_field *field; struct list_head *head = &user->fields; int pos = 0, depth = 0; const char *str_func; pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); - list_for_each_entry_safe_reverse(field, next, head, link) { + list_for_each_entry_reverse(field, head, link) { if (depth != 0) pos += snprintf(buf + pos, LEN_OR_ZERO, " "); @@ -1347,7 +1347,7 @@ static int user_event_set_print_fmt(struct user_event *user, char *buf, int len) pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); - list_for_each_entry_safe_reverse(field, next, head, link) { + list_for_each_entry_reverse(field, head, link) { if (user_field_is_dyn_string(field->type, &str_func)) pos += snprintf(buf + pos, LEN_OR_ZERO, ", %s(%s)", str_func, field->name); @@ -1732,7 +1732,7 @@ static int user_event_create(const char *raw_command) static int user_event_show(struct seq_file *m, struct dyn_event *ev) { struct user_event *user = container_of(ev, struct user_event, devent); - struct ftrace_event_field *field, *next; + struct ftrace_event_field *field; struct list_head *head; int depth = 0; @@ -1740,7 +1740,7 @@ static int user_event_show(struct seq_file *m, struct dyn_event *ev) head = trace_get_fields(&user->call); - list_for_each_entry_safe_reverse(field, next, head, link) { + list_for_each_entry_reverse(field, head, link) { if (depth == 0) seq_puts(m, " "); else @@ -1816,13 +1816,14 @@ out: static bool user_fields_match(struct user_event *user, int argc, const char **argv) { - struct ftrace_event_field *field, *next; + struct ftrace_event_field *field; struct list_head *head = &user->fields; int i = 0; - list_for_each_entry_safe_reverse(field, next, head, link) + list_for_each_entry_reverse(field, head, link) { if (!user_field_match(field, argc, argv, &i)) return false; + } if (i != argc) return false; -- cgit v1.2.3-59-g8ed1b From 086629773ec96216d06c72c801602cc56ebece27 Mon Sep 17 00:00:00 2001 From: Sishuai Gong Date: Thu, 17 Aug 2023 20:00:31 -0400 Subject: tracefs: Avoid changing i_mode to a temp value Right now inode->i_mode is updated twice to reach the desired value in tracefs_apply_options(). Because there is no lock protecting the two writes, other threads might read the intermediate value of inode->i_mode. Thread-1 Thread-2 // tracefs_apply_options() //e.g., acl_permission_check inode->i_mode &= ~S_IALLUGO; unsigned int mode = inode->i_mode; inode->i_mode |= opts->mode; I think there is no need to introduce a lock but it is better to only update inode->i_mode ONCE, so the readers will either see the old or latest value, rather than an intermediate/temporary value. Note, the race is not a security concern as the intermediate value is more locked down than either the start or end version. This is more just to do the conversion cleanly. Link: https://lore.kernel.org/linux-trace-kernel/AB5B0A1C-75D9-4E82-A7F0-CF7D0715587B@gmail.com Signed-off-by: Sishuai Gong Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index bb6de89eb446..c7a10f965602 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -310,6 +310,7 @@ static int tracefs_apply_options(struct super_block *sb, bool remount) struct tracefs_fs_info *fsi = sb->s_fs_info; struct inode *inode = d_inode(sb->s_root); struct tracefs_mount_opts *opts = &fsi->mount_opts; + umode_t tmp_mode; /* * On remount, only reset mode/uid/gid if they were provided as mount @@ -317,8 +318,9 @@ static int tracefs_apply_options(struct super_block *sb, bool remount) */ if (!remount || opts->opts & BIT(Opt_mode)) { - inode->i_mode &= ~S_IALLUGO; - inode->i_mode |= opts->mode; + tmp_mode = READ_ONCE(inode->i_mode) & ~S_IALLUGO; + tmp_mode |= opts->mode; + WRITE_ONCE(inode->i_mode, tmp_mode); } if (!remount || opts->opts & BIT(Opt_uid)) -- cgit v1.2.3-59-g8ed1b From 8c96b70171584f38940eb2ba65b84eee38b549ba Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 22 Aug 2023 05:33:13 -0400 Subject: tracefs: Remove kerneldoc from struct eventfs_file The struct eventfs_file is a local structure and should not be parsed by kernel doc. It also does not fully follow the kerneldoc format and is causing kerneldoc to spit out errors. Replace the /** to /* so that kerneldoc no longer processes this structure. Also format the comments of the delete union of the structure to be a bit better. Link: https://lore.kernel.org/linux-trace-kernel/20230818201414.2729745-1-willy@infradead.org/ Link: https://lore.kernel.org/linux-trace-kernel/20230822053313.77aa3397@rorschach.local.home Cc: Mark Rutland Reported-by: Matthew Wilcox (Oracle) Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index da8d2e73cc47..237c6f370ad9 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -27,7 +27,7 @@ struct eventfs_inode { struct list_head e_top_files; }; -/** +/* * struct eventfs_file - hold the properties of the eventfs files and * directories. * @name: the name of the file or directory to create @@ -48,10 +48,16 @@ struct eventfs_file { struct eventfs_inode *ei; const struct file_operations *fop; const struct inode_operations *iop; + /* + * Union - used for deletion + * @del_list: list of eventfs_file to delete + * @rcu: eventfs_file to delete in RCU + * @is_freed: node is freed if one of the above is set + */ union { - struct list_head del_list; /* list of eventfs_file to delete */ - struct rcu_head rcu; /* eventfs_file to delete */ - unsigned long is_freed; /* Freed if one of the above is set */ + struct list_head del_list; + struct rcu_head rcu; + unsigned long is_freed; }; void *data; umode_t mode; -- cgit v1.2.3-59-g8ed1b