diff options
Diffstat (limited to 'tools/perf/util/machine.c')
| -rw-r--r-- | tools/perf/util/machine.c | 941 |
1 files changed, 775 insertions, 166 deletions
diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index fb5c2cd44d30..76316e459c3d 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -16,6 +16,7 @@ #include "map_symbol.h" #include "branch.h" #include "mem-events.h" +#include "path.h" #include "srcline.h" #include "symbol.h" #include "sort.h" @@ -33,6 +34,8 @@ #include "asm/bug.h" #include "bpf-event.h" #include <internal/lib.h> // page_size +#include "cgroup.h" +#include "arm64-frame-pointer-unwind-support.h" #include <linux/ctype.h> #include <symbol/kallsyms.h> @@ -81,12 +84,23 @@ static int machine__set_mmap_name(struct machine *machine) return machine->mmap_name ? 0 : -ENOMEM; } +static void thread__set_guest_comm(struct thread *thread, pid_t pid) +{ + char comm[64]; + + snprintf(comm, sizeof(comm), "[guest/%d]", pid); + thread__set_comm(thread, comm, 0); +} + int machine__init(struct machine *machine, const char *root_dir, pid_t pid) { int err = -ENOMEM; memset(machine, 0, sizeof(*machine)); - maps__init(&machine->kmaps, machine); + machine->kmaps = maps__new(machine); + if (machine->kmaps == NULL) + return -ENOMEM; + RB_CLEAR_NODE(&machine->rb_node); dsos__init(&machine->dsos); @@ -105,7 +119,7 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) machine->root_dir = strdup(root_dir); if (machine->root_dir == NULL) - return -ENOMEM; + goto out; if (machine__set_mmap_name(machine)) goto out; @@ -113,13 +127,11 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) if (pid != HOST_KERNEL_ID) { struct thread *thread = machine__findnew_thread(machine, -1, pid); - char comm[64]; if (thread == NULL) goto out; - snprintf(comm, sizeof(comm), "[guest/%d]", pid); - thread__set_comm(thread, comm, 0); + thread__set_guest_comm(thread, pid); thread__put(thread); } @@ -128,6 +140,7 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) out: if (err) { + zfree(&machine->kmaps); zfree(&machine->root_dir); zfree(&machine->mmap_name); } @@ -217,12 +230,13 @@ void machine__exit(struct machine *machine) return; machine__destroy_kernel_maps(machine); - maps__exit(&machine->kmaps); + maps__delete(machine->kmaps); dsos__exit(&machine->dsos); machine__exit_vdso(machine); zfree(&machine->root_dir); zfree(&machine->mmap_name); zfree(&machine->current_tid); + zfree(&machine->kallsyms_filename); for (i = 0; i < THREADS__TABLE_SIZE; i++) { struct threads *threads = &machine->threads[i]; @@ -292,6 +306,8 @@ struct machine *machines__add(struct machines *machines, pid_t pid, rb_link_node(&machine->rb_node, parent, p); rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost); + machine->machines = machines; + return machine; } @@ -368,6 +384,102 @@ out: return machine; } +struct machine *machines__find_guest(struct machines *machines, pid_t pid) +{ + struct machine *machine = machines__find(machines, pid); + + if (!machine) + machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID); + return machine; +} + +/* + * A common case for KVM test programs is that the test program acts as the + * hypervisor, creating, running and destroying the virtual machine, and + * providing the guest object code from its own object code. In this case, + * the VM is not running an OS, but only the functions loaded into it by the + * hypervisor test program, and conveniently, loaded at the same virtual + * addresses. + * + * Normally to resolve addresses, MMAP events are needed to map addresses + * back to the object code and debug symbols for that object code. + * + * Currently, there is no way to get such mapping information from guests + * but, in the scenario described above, the guest has the same mappings + * as the hypervisor, so support for that scenario can be achieved. + * + * To support that, copy the host thread's maps to the guest thread's maps. + * Note, we do not discover the guest until we encounter a guest event, + * which works well because it is not until then that we know that the host + * thread's maps have been set up. + * + * This function returns the guest thread. Apart from keeping the data + * structures sane, using a thread belonging to the guest machine, instead + * of the host thread, allows it to have its own comm (refer + * thread__set_guest_comm()). + */ +static struct thread *findnew_guest_code(struct machine *machine, + struct machine *host_machine, + pid_t pid) +{ + struct thread *host_thread; + struct thread *thread; + int err; + + if (!machine) + return NULL; + + thread = machine__findnew_thread(machine, -1, pid); + if (!thread) + return NULL; + + /* Assume maps are set up if there are any */ + if (thread->maps->nr_maps) + return thread; + + host_thread = machine__find_thread(host_machine, -1, pid); + if (!host_thread) + goto out_err; + + thread__set_guest_comm(thread, pid); + + /* + * Guest code can be found in hypervisor process at the same address + * so copy host maps. + */ + err = maps__clone(thread, host_thread->maps); + thread__put(host_thread); + if (err) + goto out_err; + + return thread; + +out_err: + thread__zput(thread); + return NULL; +} + +struct thread *machines__findnew_guest_code(struct machines *machines, pid_t pid) +{ + struct machine *host_machine = machines__find(machines, HOST_KERNEL_ID); + struct machine *machine = machines__findnew(machines, pid); + + return findnew_guest_code(machine, host_machine, pid); +} + +struct thread *machine__findnew_guest_code(struct machine *machine, pid_t pid) +{ + struct machines *machines = machine->machines; + struct machine *host_machine; + + if (!machines) + return NULL; + + host_machine = machines__find(machines, HOST_KERNEL_ID); + + return findnew_guest_code(machine, host_machine, pid); +} + void machines__process_guests(struct machines *machines, machine__process_t process, void *data) { @@ -588,6 +700,24 @@ struct thread *machine__find_thread(struct machine *machine, pid_t pid, return th; } +/* + * Threads are identified by pid and tid, and the idle task has pid == tid == 0. + * So here a single thread is created for that, but actually there is a separate + * idle task per cpu, so there should be one 'struct thread' per cpu, but there + * is only 1. That causes problems for some tools, requiring workarounds. For + * example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu(). + */ +struct thread *machine__idle_thread(struct machine *machine) +{ + struct thread *thread = machine__findnew_thread(machine, 0, 0); + + if (!thread || thread__set_comm(thread, "swapper", 0) || + thread__set_namespaces(thread, 0, NULL)) + pr_err("problem inserting idle task for machine pid %d\n", machine->pid); + + return thread; +} + struct comm *machine__thread_exec_comm(struct machine *machine, struct thread *thread) { @@ -654,6 +784,22 @@ int machine__process_namespaces_event(struct machine *machine __maybe_unused, return err; } +int machine__process_cgroup_event(struct machine *machine, + union perf_event *event, + struct perf_sample *sample __maybe_unused) +{ + struct cgroup *cgrp; + + if (dump_trace) + perf_event__fprintf_cgroup(event, stdout); + + cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path); + if (cgrp == NULL) + return -ENOMEM; + + return 0; +} + int machine__process_lost_event(struct machine *machine __maybe_unused, union perf_event *event, struct perf_sample *sample __maybe_unused) { @@ -686,7 +832,7 @@ static struct dso *machine__findnew_module_dso(struct machine *machine, dso__set_module_info(dso, m, machine); dso__set_long_name(dso, strdup(filename), true); - dso->kernel = DSO_TYPE_KERNEL; + dso->kernel = DSO_SPACE__KERNEL; } dso__get(dso); @@ -711,6 +857,14 @@ int machine__process_itrace_start_event(struct machine *machine __maybe_unused, return 0; } +int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused, + union perf_event *event) +{ + if (dump_trace) + perf_event__fprintf_aux_output_hw_id(event, stdout); + return 0; +} + int machine__process_switch_event(struct machine *machine __maybe_unused, union perf_event *event) { @@ -724,24 +878,37 @@ static int machine__process_ksymbol_register(struct machine *machine, struct perf_sample *sample __maybe_unused) { struct symbol *sym; - struct map *map = maps__find(&machine->kmaps, event->ksymbol.addr); + struct map *map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); if (!map) { struct dso *dso = dso__new(event->ksymbol.name); if (dso) { - dso->kernel = DSO_TYPE_KERNEL; + dso->kernel = DSO_SPACE__KERNEL; map = map__new2(0, dso); + dso__put(dso); } if (!dso || !map) { - dso__put(dso); return -ENOMEM; } + if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) { + map->dso->binary_type = DSO_BINARY_TYPE__OOL; + map->dso->data.file_size = event->ksymbol.len; + dso__set_loaded(map->dso); + } + map->start = event->ksymbol.addr; map->end = map->start + event->ksymbol.len; - maps__insert(&machine->kmaps, map); + maps__insert(machine__kernel_maps(machine), map); + map__put(map); + dso__set_loaded(dso); + + if (is_bpf_image(event->ksymbol.name)) { + dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE; + dso__set_long_name(dso, "", false); + } } sym = symbol__new(map->map_ip(map, map->start), @@ -757,11 +924,20 @@ static int machine__process_ksymbol_unregister(struct machine *machine, union perf_event *event, struct perf_sample *sample __maybe_unused) { + struct symbol *sym; struct map *map; - map = maps__find(&machine->kmaps, event->ksymbol.addr); - if (map) - maps__remove(&machine->kmaps, map); + map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); + if (!map) + return 0; + + if (map != machine->vmlinux_map) + maps__remove(machine__kernel_maps(machine), map); + else { + sym = dso__find_symbol(map->dso, map->map_ip(map, map->start)); + if (sym) + dso__delete_symbol(map->dso, sym); + } return 0; } @@ -779,6 +955,47 @@ int machine__process_ksymbol(struct machine *machine __maybe_unused, return machine__process_ksymbol_register(machine, event, sample); } +int machine__process_text_poke(struct machine *machine, union perf_event *event, + struct perf_sample *sample __maybe_unused) +{ + struct map *map = maps__find(machine__kernel_maps(machine), event->text_poke.addr); + u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + if (dump_trace) + perf_event__fprintf_text_poke(event, machine, stdout); + + if (!event->text_poke.new_len) + return 0; + + if (cpumode != PERF_RECORD_MISC_KERNEL) { + pr_debug("%s: unsupported cpumode - ignoring\n", __func__); + return 0; + } + + if (map && map->dso) { + u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len; + int ret; + + /* + * Kernel maps might be changed when loading symbols so loading + * must be done prior to using kernel maps. + */ + map__load(map); + ret = dso__data_write_cache_addr(map->dso, map, machine, + event->text_poke.addr, + new_bytes, + event->text_poke.new_len); + if (ret != event->text_poke.new_len) + pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n", + event->text_poke.addr); + } else { + pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n", + event->text_poke.addr); + } + + return 0; +} + static struct map *machine__addnew_module_map(struct machine *machine, u64 start, const char *filename) { @@ -797,9 +1014,9 @@ static struct map *machine__addnew_module_map(struct machine *machine, u64 start if (map == NULL) goto out; - maps__insert(&machine->kmaps, map); + maps__insert(machine__kernel_maps(machine), map); - /* Put the map here because maps__insert alread got it */ + /* Put the map here because maps__insert already got it */ map__put(map); out: /* put the dso here, corresponding to machine__findnew_module_dso */ @@ -895,14 +1112,14 @@ static struct dso *machine__get_kernel(struct machine *machine) vmlinux_name = symbol_conf.vmlinux_name; kernel = machine__findnew_kernel(machine, vmlinux_name, - "[kernel]", DSO_TYPE_KERNEL); + "[kernel]", DSO_SPACE__KERNEL); } else { if (symbol_conf.default_guest_vmlinux_name) vmlinux_name = symbol_conf.default_guest_vmlinux_name; kernel = machine__findnew_kernel(machine, vmlinux_name, "[guest.kernel]", - DSO_TYPE_GUEST_KERNEL); + DSO_SPACE__KERNEL_GUEST); } if (kernel != NULL && (!kernel->has_build_id)) @@ -911,10 +1128,6 @@ static struct dso *machine__get_kernel(struct machine *machine) return kernel; } -struct process_args { - u64 start; -}; - void machine__get_kallsyms_filename(struct machine *machine, char *buf, size_t bufsz) { @@ -983,7 +1196,7 @@ int machine__create_extra_kernel_map(struct machine *machine, strlcpy(kmap->name, xm->name, KMAP_NAME_LEN); - maps__insert(&machine->kmaps, map); + maps__insert(machine__kernel_maps(machine), map); pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n", kmap->name, map->start, map->end); @@ -1028,7 +1241,7 @@ static u64 find_entry_trampoline(struct dso *dso) int machine__map_x86_64_entry_trampolines(struct machine *machine, struct dso *kernel) { - struct maps *kmaps = &machine->kmaps; + struct maps *kmaps = machine__kernel_maps(machine); int nr_cpus_avail, cpu; bool found = false; struct map *map; @@ -1098,7 +1311,7 @@ __machine__create_kernel_maps(struct machine *machine, struct dso *kernel) return -1; machine->vmlinux_map->map_ip = machine->vmlinux_map->unmap_ip = identity__map_ip; - maps__insert(&machine->kmaps, machine->vmlinux_map); + maps__insert(machine__kernel_maps(machine), machine->vmlinux_map); return 0; } @@ -1111,7 +1324,7 @@ void machine__destroy_kernel_maps(struct machine *machine) return; kmap = map__kmap(map); - maps__remove(&machine->kmaps, map); + maps__remove(machine__kernel_maps(machine), map); if (kmap && kmap->ref_reloc_sym) { zfree((char **)&kmap->ref_reloc_sym->name); zfree(&kmap->ref_reloc_sym); @@ -1206,7 +1419,7 @@ int machine__load_kallsyms(struct machine *machine, const char *filename) * kernel, with modules between them, fixup the end of all * sections. */ - maps__fixup_end(&machine->kmaps); + maps__fixup_end(machine__kernel_maps(machine)); } return ret; @@ -1300,7 +1513,7 @@ static int maps__set_modules_path_dir(struct maps *maps, const char *dir_name, i struct stat st; /*sshfs might return bad dent->d_type, so we have to stat*/ - snprintf(path, sizeof(path), "%s/%s", dir_name, dent->d_name); + path__join(path, sizeof(path), dir_name, dent->d_name); if (stat(path, &st)) continue; @@ -1354,7 +1567,7 @@ static int machine__set_modules_path(struct machine *machine) machine->root_dir, version); free(version); - return maps__set_modules_path_dir(&machine->kmaps, modules_path, 0); + return maps__set_modules_path_dir(machine__kernel_maps(machine), modules_path, 0); } int __weak arch__fix_module_text_start(u64 *start __maybe_unused, u64 *size __maybe_unused, @@ -1427,11 +1640,11 @@ static void machine__update_kernel_mmap(struct machine *machine, struct map *map = machine__kernel_map(machine); map__get(map); - maps__remove(&machine->kmaps, map); + maps__remove(machine__kernel_maps(machine), map); machine__set_kernel_mmap(machine, start, end); - maps__insert(&machine->kmaps, map); + maps__insert(machine__kernel_maps(machine), map); map__put(map); } @@ -1502,60 +1715,65 @@ static bool machine__uses_kcore(struct machine *machine) } static bool perf_event__is_extra_kernel_mmap(struct machine *machine, - union perf_event *event) + struct extra_kernel_map *xm) { return machine__is(machine, "x86_64") && - is_entry_trampoline(event->mmap.filename); + is_entry_trampoline(xm->name); } static int machine__process_extra_kernel_map(struct machine *machine, - union perf_event *event) + struct extra_kernel_map *xm) { struct dso *kernel = machine__kernel_dso(machine); - struct extra_kernel_map xm = { - .start = event->mmap.start, - .end = event->mmap.start + event->mmap.len, - .pgoff = event->mmap.pgoff, - }; if (kernel == NULL) return -1; - strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN); - - return machine__create_extra_kernel_map(machine, kernel, &xm); + return machine__create_extra_kernel_map(machine, kernel, xm); } static int machine__process_kernel_mmap_event(struct machine *machine, - union perf_event *event) + struct extra_kernel_map *xm, + struct build_id *bid) { struct map *map; - enum dso_kernel_type kernel_type; + enum dso_space_type dso_space; bool is_kernel_mmap; + const char *mmap_name = machine->mmap_name; /* If we have maps from kcore then we do not need or want any others */ if (machine__uses_kcore(machine)) return 0; if (machine__is_host(machine)) - kernel_type = DSO_TYPE_KERNEL; + dso_space = DSO_SPACE__KERNEL; else - kernel_type = DSO_TYPE_GUEST_KERNEL; - - is_kernel_mmap = memcmp(event->mmap.filename, - machine->mmap_name, - strlen(machine->mmap_name) - 1) == 0; - if (event->mmap.filename[0] == '/' || - (!is_kernel_mmap && event->mmap.filename[0] == '[')) { - map = machine__addnew_module_map(machine, event->mmap.start, - event->mmap.filename); + dso_space = DSO_SPACE__KERNEL_GUEST; + + is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0; + if (!is_kernel_mmap && !machine__is_host(machine)) { + /* + * If the event was recorded inside the guest and injected into + * the host perf.data file, then it will match a host mmap_name, + * so try that - see machine__set_mmap_name(). + */ + mmap_name = "[kernel.kallsyms]"; + is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0; + } + if (xm->name[0] == '/' || + (!is_kernel_mmap && xm->name[0] == '[')) { + map = machine__addnew_module_map(machine, xm->start, + xm->name); if (map == NULL) goto out_problem; - map->end = map->start + event->mmap.len; + map->end = map->start + xm->end - xm->start; + + if (build_id__is_defined(bid)) + dso__set_build_id(map->dso, bid); + } else if (is_kernel_mmap) { - const char *symbol_name = (event->mmap.filename + - strlen(machine->mmap_name)); + const char *symbol_name = xm->name + strlen(mmap_name); /* * Should be there already, from the build-id table in * the header. @@ -1600,7 +1818,7 @@ static int machine__process_kernel_mmap_event(struct machine *machine, if (kernel == NULL) goto out_problem; - kernel->kernel = kernel_type; + kernel->kernel = dso_space; if (__machine__create_kernel_maps(machine, kernel) < 0) { dso__put(kernel); goto out_problem; @@ -1609,18 +1827,20 @@ static int machine__process_kernel_mmap_event(struct machine *machine, if (strstr(kernel->long_name, "vmlinux")) dso__set_short_name(kernel, "[kernel.vmlinux]", false); - machine__update_kernel_mmap(machine, event->mmap.start, - event->mmap.start + event->mmap.len); + machine__update_kernel_mmap(machine, xm->start, xm->end); + + if (build_id__is_defined(bid)) + dso__set_build_id(kernel, bid); /* * Avoid using a zero address (kptr_restrict) for the ref reloc * symbol. Effectively having zero here means that at record * time /proc/sys/kernel/kptr_restrict was non zero. */ - if (event->mmap.pgoff != 0) { + if (xm->pgoff != 0) { map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, symbol_name, - event->mmap.pgoff); + xm->pgoff); } if (machine__is_default_guest(machine)) { @@ -1629,8 +1849,8 @@ static int machine__process_kernel_mmap_event(struct machine *machine, */ dso__load(kernel, machine__kernel_map(machine)); } - } else if (perf_event__is_extra_kernel_mmap(machine, event)) { - return machine__process_extra_kernel_map(machine, event); + } else if (perf_event__is_extra_kernel_mmap(machine, xm)) { + return machine__process_extra_kernel_map(machine, xm); } return 0; out_problem: @@ -1649,14 +1869,27 @@ int machine__process_mmap2_event(struct machine *machine, .ino = event->mmap2.ino, .ino_generation = event->mmap2.ino_generation, }; + struct build_id __bid, *bid = NULL; int ret = 0; if (dump_trace) perf_event__fprintf_mmap2(event, stdout); + if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { + bid = &__bid; + build_id__init(bid, event->mmap2.build_id, event->mmap2.build_id_size); + } + if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL || sample->cpumode == PERF_RECORD_MISC_KERNEL) { - ret = machine__process_kernel_mmap_event(machine, event); + struct extra_kernel_map xm = { + .start = event->mmap2.start, + .end = event->mmap2.start + event->mmap2.len, + .pgoff = event->mmap2.pgoff, + }; + + strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN); + ret = machine__process_kernel_mmap_event(machine, &xm, bid); if (ret < 0) goto out_problem; return 0; @@ -1670,7 +1903,7 @@ int machine__process_mmap2_event(struct machine *machine, map = map__new(machine, event->mmap2.start, event->mmap2.len, event->mmap2.pgoff, &dso_id, event->mmap2.prot, - event->mmap2.flags, + event->mmap2.flags, bid, event->mmap2.filename, thread); if (map == NULL) @@ -1706,7 +1939,14 @@ int machine__process_mmap_event(struct machine *machine, union perf_event *event if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL || sample->cpumode == PERF_RECORD_MISC_KERNEL) { - ret = machine__process_kernel_mmap_event(machine, event); + struct extra_kernel_map xm = { + .start = event->mmap.start, + .end = event->mmap.start + event->mmap.len, + .pgoff = event->mmap.pgoff, + }; + + strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN); + ret = machine__process_kernel_mmap_event(machine, &xm, NULL); if (ret < 0) goto out_problem; return 0; @@ -1722,7 +1962,7 @@ int machine__process_mmap_event(struct machine *machine, union perf_event *event map = map__new(machine, event->mmap.start, event->mmap.len, event->mmap.pgoff, - NULL, prot, 0, event->mmap.filename, thread); + NULL, prot, 0, NULL, event->mmap.filename, thread); if (map == NULL) goto out_problem_map; @@ -1828,7 +2068,7 @@ int machine__process_fork_event(struct machine *machine, union perf_event *event * maps because that is what the kernel just did. * * But when synthesizing, this should not be done. If we do, we end up - * with overlapping maps as we process the sythesized MMAP2 events that + * with overlapping maps as we process the synthesized MMAP2 events that * get delivered shortly thereafter. * * Use the FORK event misc flags in an internal way to signal this @@ -1878,6 +2118,8 @@ int machine__process_event(struct machine *machine, union perf_event *event, ret = machine__process_mmap_event(machine, event, sample); break; case PERF_RECORD_NAMESPACES: ret = machine__process_namespaces_event(machine, event, sample); break; + case PERF_RECORD_CGROUP: + ret = machine__process_cgroup_event(machine, event, sample); break; case PERF_RECORD_MMAP2: ret = machine__process_mmap2_event(machine, event, sample); break; case PERF_RECORD_FORK: @@ -1899,6 +2141,10 @@ int machine__process_event(struct machine *machine, union perf_event *event, ret = machine__process_ksymbol(machine, event, sample); break; case PERF_RECORD_BPF_EVENT: ret = machine__process_bpf(machine, event, sample); break; + case PERF_RECORD_TEXT_POKE: + ret = machine__process_text_poke(machine, event, sample); break; + case PERF_RECORD_AUX_OUTPUT_HW_ID: + ret = machine__process_aux_output_hw_id_event(machine, event); break; default: ret = -1; break; @@ -1910,8 +2156,8 @@ int machine__process_event(struct machine *machine, union perf_event *event, static bool symbol__match_regex(struct symbol *sym, regex_t *regex) { if (!regexec(regex, sym->name, 0, NULL, 0)) - return 1; - return 0; + return true; + return false; } static void ip__resolve_ams(struct thread *thread, @@ -1932,15 +2178,17 @@ static void ip__resolve_ams(struct thread *thread, ams->addr = ip; ams->al_addr = al.addr; + ams->al_level = al.level; ams->ms.maps = al.maps; ams->ms.sym = al.sym; ams->ms.map = al.map; ams->phys_addr = 0; + ams->data_page_size = 0; } static void ip__resolve_data(struct thread *thread, u8 m, struct addr_map_symbol *ams, - u64 addr, u64 phys_addr) + u64 addr, u64 phys_addr, u64 daddr_page_size) { struct addr_location al; @@ -1950,10 +2198,12 @@ static void ip__resolve_data(struct thread *thread, ams->addr = addr; ams->al_addr = al.addr; + ams->al_level = al.level; ams->ms.maps = al.maps; ams->ms.sym = al.sym; ams->ms.map = al.map; ams->phys_addr = phys_addr; + ams->data_page_size = daddr_page_size; } struct mem_info *sample__resolve_mem(struct perf_sample *sample, @@ -1966,7 +2216,8 @@ struct mem_info *sample__resolve_mem(struct perf_sample *sample, ip__resolve_ams(al->thread, &mi->iaddr, sample->ip); ip__resolve_data(al->thread, al->cpumode, &mi->daddr, - sample->addr, sample->phys_addr); + sample->addr, sample->phys_addr, + sample->data_page_size); mi->data_src.val = sample->data_src; return mi; @@ -2017,6 +2268,7 @@ static int add_callchain_ip(struct thread *thread, al.filtered = 0; al.sym = NULL; + al.srcline = NULL; if (!cpumode) { thread__find_cpumode_addr_location(thread, ip, &al); } else { @@ -2081,15 +2333,16 @@ struct branch_info *sample__resolve_bstack(struct perf_sample *sample, { unsigned int i; const struct branch_stack *bs = sample->branch_stack; + struct branch_entry *entries = perf_sample__branch_entries(sample); struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info)); if (!bi) return NULL; for (i = 0; i < bs->nr; i++) { - ip__resolve_ams(al->thread, &bi[i].to, bs->entries[i].to); - ip__resolve_ams(al->thread, &bi[i].from, bs->entries[i].from); - bi[i].flags = bs->entries[i].flags; + ip__resolve_ams(al->thread, &bi[i].to, entries[i].to); + ip__resolve_ams(al->thread, &bi[i].from, entries[i].from); + bi[i].flags = entries[i].flags; } return bi; } @@ -2158,8 +2411,305 @@ static int remove_loops(struct branch_entry *l, int nr, return nr; } +static int lbr_callchain_add_kernel_ip(struct thread *thread, + struct callchain_cursor *cursor, + struct perf_sample *sample, + struct symbol **parent, + struct addr_location *root_al, + u64 branch_from, + bool callee, int end) +{ + struct ip_callchain *chain = sample->callchain; + u8 cpumode = PERF_RECORD_MISC_USER; + int err, i; + + if (callee) { + for (i = 0; i < end + 1; i++) { + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, chain->ips[i], + false, NULL, NULL, branch_from); + if (err) + return err; + } + return 0; + } + + for (i = end; i >= 0; i--) { + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, chain->ips[i], + false, NULL, NULL, branch_from); + if (err) + return err; + } + + return 0; +} + +static void save_lbr_cursor_node(struct thread *thread, + struct callchain_cursor *cursor, + int idx) +{ + struct lbr_stitch *lbr_stitch = thread->lbr_stitch; + + if (!lbr_stitch) + return; + + if (cursor->pos == cursor->nr) { + lbr_stitch->prev_lbr_cursor[idx].valid = false; + return; + } + + if (!cursor->curr) + cursor->curr = cursor->first; + else + cursor->curr = cursor->curr->next; + memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr, + sizeof(struct callchain_cursor_node)); + + lbr_stitch->prev_lbr_cursor[idx].valid = true; + cursor->pos++; +} + +static int lbr_callchain_add_lbr_ip(struct thread *thread, + struct callchain_cursor *cursor, + struct perf_sample *sample, + struct symbol **parent, + struct addr_location *root_al, + u64 *branch_from, + bool callee) +{ + struct branch_stack *lbr_stack = sample->branch_stack; + struct branch_entry *entries = perf_sample__branch_entries(sample); + u8 cpumode = PERF_RECORD_MISC_USER; + int lbr_nr = lbr_stack->nr; + struct branch_flags *flags; + int err, i; + u64 ip; + + /* + * The curr and pos are not used in writing session. They are cleared + * in callchain_cursor_commit() when the writing session is closed. + * Using curr and pos to track the current cursor node. + */ + if (thread->lbr_stitch) { + cursor->curr = NULL; + cursor->pos = cursor->nr; + if (cursor->nr) { + cursor->curr = cursor->first; + for (i = 0; i < (int)(cursor->nr - 1); i++) + cursor->curr = cursor->curr->next; + } + } + + if (callee) { + /* Add LBR ip from first entries.to */ + ip = entries[0].to; + flags = &entries[0].flags; + *branch_from = entries[0].from; + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, ip, + true, flags, NULL, + *branch_from); + if (err) + return err; + + /* + * The number of cursor node increases. + * Move the current cursor node. + * But does not need to save current cursor node for entry 0. + * It's impossible to stitch the whole LBRs of previous sample. + */ + if (thread->lbr_stitch && (cursor->pos != cursor->nr)) { + if (!cursor->curr) + cursor->curr = cursor->first; + else + cursor->curr = cursor->curr->next; + cursor->pos++; + } + + /* Add LBR ip from entries.from one by one. */ + for (i = 0; i < lbr_nr; i++) { + ip = entries[i].from; + flags = &entries[i].flags; + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, ip, + true, flags, NULL, + *branch_from); + if (err) + return err; + save_lbr_cursor_node(thread, cursor, i); + } + return 0; + } + + /* Add LBR ip from entries.from one by one. */ + for (i = lbr_nr - 1; i >= 0; i--) { + ip = entries[i].from; + flags = &entries[i].flags; + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, ip, + true, flags, NULL, + *branch_from); + if (err) + return err; + save_lbr_cursor_node(thread, cursor, i); + } + + /* Add LBR ip from first entries.to */ + ip = entries[0].to; + flags = &entries[0].flags; + *branch_from = entries[0].from; + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, ip, + true, flags, NULL, + *branch_from); + if (err) + return err; + + return 0; +} + +static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread, + struct callchain_cursor *cursor) +{ + struct lbr_stitch *lbr_stitch = thread->lbr_stitch; + struct callchain_cursor_node *cnode; + struct stitch_list *stitch_node; + int err; + + list_for_each_entry(stitch_node, &lbr_stitch->lists, node) { + cnode = &stitch_node->cursor; + + err = callchain_cursor_append(cursor, cnode->ip, + &cnode->ms, + cnode->branch, + &cnode->branch_flags, + cnode->nr_loop_iter, + cnode->iter_cycles, + cnode->branch_from, + cnode->srcline); + if (err) + return err; + } + return 0; +} + +static struct stitch_list *get_stitch_node(struct thread *thread) +{ + struct lbr_stitch *lbr_stitch = thread->lbr_stitch; + struct stitch_list *stitch_node; + + if (!list_empty(&lbr_stitch->free_lists)) { + stitch_node = list_first_entry(&lbr_stitch->free_lists, + struct stitch_list, node); + list_del(&stitch_node->node); + + return stitch_node; + } + + return malloc(sizeof(struct stitch_list)); +} + +static bool has_stitched_lbr(struct thread *thread, + struct perf_sample *cur, + struct perf_sample *prev, + unsigned int max_lbr, + bool callee) +{ + struct branch_stack *cur_stack = cur->branch_stack; + struct branch_entry *cur_entries = perf_sample__branch_entries(cur); + struct branch_stack *prev_stack = prev->branch_stack; + struct branch_entry *prev_entries = perf_sample__branch_entries(prev); + struct lbr_stitch *lbr_stitch = thread->lbr_stitch; + int i, j, nr_identical_branches = 0; + struct stitch_list *stitch_node; + u64 cur_base, distance; + + if (!cur_stack || !prev_stack) + return false; + + /* Find the physical index of the base-of-stack for current sample. */ + cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1; + + distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) : + (max_lbr + prev_stack->hw_idx - cur_base); + /* Previous sample has shorter stack. Nothing can be stitched. */ + if (distance + 1 > prev_stack->nr) + return false; + + /* + * Check if there are identical LBRs between two samples. + * Identical LBRs must have same from, to and flags values. Also, + * they have to be saved in the same LBR registers (same physical + * index). + * + * Starts from the base-of-stack of current sample. + */ + for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) { + if ((prev_entries[i].from != cur_entries[j].from) || + (prev_entries[i].to != cur_entries[j].to) || + (prev_entries[i].flags.value != cur_entries[j].flags.value)) + break; + nr_identical_branches++; + } + + if (!nr_identical_branches) + return false; + + /* + * Save the LBRs between the base-of-stack of previous sample + * and the base-of-stack of current sample into lbr_stitch->lists. + * These LBRs will be stitched later. + */ + for (i = prev_stack->nr - 1; i > (int)distance; i--) { + + if (!lbr_stitch->prev_lbr_cursor[i].valid) + continue; + + stitch_node = get_stitch_node(thread); + if (!stitch_node) + return false; + + memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i], + sizeof(struct callchain_cursor_node)); + + if (callee) + list_add(&stitch_node->node, &lbr_stitch->lists); + else + list_add_tail(&stitch_node->node, &lbr_stitch->lists); + } + + return true; +} + +static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr) +{ + if (thread->lbr_stitch) + return true; + + thread->lbr_stitch = zalloc(sizeof(*thread->lbr_stitch)); + if (!thread->lbr_stitch) + goto err; + + thread->lbr_stitch->prev_lbr_cursor = calloc(max_lbr + 1, sizeof(struct callchain_cursor_node)); + if (!thread->lbr_stitch->prev_lbr_cursor) + goto free_lbr_stitch; + + INIT_LIST_HEAD(&thread->lbr_stitch->lists); + INIT_LIST_HEAD(&thread->lbr_stitch->free_lists); + + return true; + +free_lbr_stitch: + zfree(&thread->lbr_stitch); +err: + pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n"); + thread->lbr_stitch_enable = false; + return false; +} + /* - * Recolve LBR callstack chain sample + * Resolve LBR callstack chain sample * Return: * 1 on success get LBR callchain information * 0 no available LBR callchain information, should try fp @@ -2170,12 +2720,16 @@ static int resolve_lbr_callchain_sample(struct thread *thread, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, - int max_stack) + int max_stack, + unsigned int max_lbr) { + bool callee = (callchain_param.order == ORDER_CALLEE); struct ip_callchain *chain = sample->callchain; int chain_nr = min(max_stack, (int)chain->nr), i; - u8 cpumode = PERF_RECORD_MISC_USER; - u64 ip, branch_from = 0; + struct lbr_stitch *lbr_stitch; + bool stitched_lbr = false; + u64 branch_from = 0; + int err; for (i = 0; i < chain_nr; i++) { if (chain->ips[i] == PERF_CONTEXT_USER) @@ -2183,72 +2737,65 @@ static int resolve_lbr_callchain_sample(struct thread *thread, } /* LBR only affects the user callchain */ - if (i != chain_nr) { - struct branch_stack *lbr_stack = sample->branch_stack; - int lbr_nr = lbr_stack->nr, j, k; - bool branch; - struct branch_flags *flags; - /* - * LBR callstack can only get user call chain. - * The mix_chain_nr is kernel call chain - * number plus LBR user call chain number. - * i is kernel call chain number, - * 1 is PERF_CONTEXT_USER, - * lbr_nr + 1 is the user call chain number. - * For details, please refer to the comments - * in callchain__printf - */ - int mix_chain_nr = i + 1 + lbr_nr + 1; + if (i == chain_nr) + return 0; - for (j = 0; j < mix_chain_nr; j++) { - int err; - branch = false; - flags = NULL; + if (thread->lbr_stitch_enable && !sample->no_hw_idx && + (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) { + lbr_stitch = thread->lbr_stitch; - if (callchain_param.order == ORDER_CALLEE) { - if (j < i + 1) - ip = chain->ips[j]; - else if (j > i + 1) { - k = j - i - 2; - ip = lbr_stack->entries[k].from; - branch = true; - flags = &lbr_stack->entries[k].flags; - } else { - ip = lbr_stack->entries[0].to; - branch = true; - flags = &lbr_stack->entries[0].flags; - branch_from = - lbr_stack->entries[0].from; - } - } else { - if (j < lbr_nr) { - k = lbr_nr - j - 1; - ip = lbr_stack->entries[k].from; - branch = true; - flags = &lbr_stack->entries[k].flags; - } - else if (j > lbr_nr) - ip = chain->ips[i + 1 - (j - lbr_nr)]; - else { - ip = lbr_stack->entries[0].to; - branch = true; - flags = &lbr_stack->entries[0].flags; - branch_from = - lbr_stack->entries[0].from; - } - } + stitched_lbr = has_stitched_lbr(thread, sample, + &lbr_stitch->prev_sample, + max_lbr, callee); - err = add_callchain_ip(thread, cursor, parent, - root_al, &cpumode, ip, - branch, flags, NULL, - branch_from); + if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) { + list_replace_init(&lbr_stitch->lists, + &lbr_stitch->free_lists); + } + memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample)); + } + + if (callee) { + /* Add kernel ip */ + err = lbr_callchain_add_kernel_ip(thread, cursor, sample, + parent, root_al, branch_from, + true, i); + if (err) + goto error; + + err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent, + root_al, &branch_from, true); + if (err) + goto error; + + if (stitched_lbr) { + err = lbr_callchain_add_stitched_lbr_ip(thread, cursor); if (err) - return (err < 0) ? err : 0; + goto error; + } + + } else { + if (stitched_lbr) { + err = lbr_callchain_add_stitched_lbr_ip(thread, cursor); + if (err) + goto error; } - return 1; + err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent, + root_al, &branch_from, false); + if (err) + goto error; + + /* Add kernel ip */ + err = lbr_callchain_add_kernel_ip(thread, cursor, sample, + parent, root_al, branch_from, + false, i); + if (err) + goto error; } + return 1; - return 0; +error: + return (err < 0) ? err : 0; } static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread, @@ -2272,6 +2819,15 @@ static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread, return err; } +static u64 get_leaf_frame_caller(struct perf_sample *sample, + struct thread *thread, int usr_idx) +{ + if (machine__normalized_is(thread->maps->machine, "arm64")) + return get_leaf_frame_caller_aarch64(sample, thread, usr_idx); + else + return 0; +} + static int thread__resolve_callchain_sample(struct thread *thread, struct callchain_cursor *cursor, struct evsel *evsel, @@ -2281,19 +2837,24 @@ static int thread__resolve_callchain_sample(struct thread *thread, int max_stack) { struct branch_stack *branch = sample->branch_stack; + struct branch_entry *entries = perf_sample__branch_entries(sample); struct ip_callchain *chain = sample->callchain; int chain_nr = 0; u8 cpumode = PERF_RECORD_MISC_USER; - int i, j, err, nr_entries; + int i, j, err, nr_entries, usr_idx; int skip_idx = -1; int first_call = 0; + u64 leaf_frame_caller; if (chain) chain_nr = chain->nr; - if (perf_evsel__has_branch_callstack(evsel)) { + if (evsel__has_branch_callstack(evsel)) { + struct perf_env *env = evsel__env(evsel); + err = resolve_lbr_callchain_sample(thread, cursor, sample, parent, - root_al, max_stack); + root_al, max_stack, + !env ? 0 : env->max_branches); if (err) return (err < 0) ? err : 0; } @@ -2328,7 +2889,7 @@ static int thread__resolve_callchain_sample(struct thread *thread, for (i = 0; i < nr; i++) { if (callchain_param.order == ORDER_CALLEE) { - be[i] = branch->entries[i]; + be[i] = entries[i]; if (chain == NULL) continue; @@ -2347,7 +2908,7 @@ static int thread__resolve_callchain_sample(struct thread *thread, be[i].from >= chain->ips[first_call] - 8) first_call++; } else - be[i] = branch->entries[branch->nr - i - 1]; + be[i] = entries[branch->nr - i - 1]; } memset(iter, 0, sizeof(struct iterations) * nr); @@ -2408,6 +2969,34 @@ check_calls: continue; } + /* + * PERF_CONTEXT_USER allows us to locate where the user stack ends. + * Depending on callchain_param.order and the position of PERF_CONTEXT_USER, + * the index will be different in order to add the missing frame + * at the right place. + */ + + usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1; + + if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) { + + leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx); + + /* + * check if leaf_frame_Caller != ip to not add the same + * value twice. + */ + + if (leaf_frame_caller && leaf_frame_caller != ip) { + + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, leaf_frame_caller, + false, NULL, NULL, 0); + if (err) + return (err < 0) ? err : 0; + } + } + err = add_callchain_ip(thread, cursor, parent, root_al, &cpumode, ip, false, NULL, NULL, 0); @@ -2499,7 +3088,7 @@ static int thread__resolve_callchain_unwind(struct thread *thread, return 0; return unwind__get_entries(unwind_entry, cursor, - thread, sample, max_stack); + thread, sample, max_stack, false); } int thread__resolve_callchain(struct thread *thread, @@ -2591,9 +3180,7 @@ int machines__for_each_thread(struct machines *machines, pid_t machine__get_current_tid(struct machine *machine, int cpu) { - int nr_cpus = min(machine->env->nr_cpus_online, MAX_NR_CPUS); - - if (cpu < 0 || cpu >= nr_cpus || !machine->current_tid) + if (cpu < 0 || (size_t)cpu >= machine->current_tid_sz) return -1; return machine->current_tid[cpu]; @@ -2603,26 +3190,16 @@ int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid, pid_t tid) { struct thread *thread; - int nr_cpus = min(machine->env->nr_cpus_online, MAX_NR_CPUS); + const pid_t init_val = -1; if (cpu < 0) return -EINVAL; - if (!machine->current_tid) { - int i; - - machine->current_tid = calloc(nr_cpus, sizeof(pid_t)); - if (!machine->current_tid) - return -ENOMEM; - for (i = 0; i < nr_cpus; i++) - machine->current_tid[i] = -1; - } - - if (cpu >= nr_cpus) { - pr_err("Requested CPU %d too large. ", cpu); - pr_err("Consider raising MAX_NR_CPUS\n"); - return -EINVAL; - } + if (realloc_array_as_needed(machine->current_tid, + machine->current_tid_sz, + (unsigned int)cpu, + &init_val)) + return -ENOMEM; machine->current_tid[cpu] = tid; @@ -2637,14 +3214,19 @@ int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid, } /* - * Compares the raw arch string. N.B. see instead perf_env__arch() if a - * normalized arch is needed. + * Compares the raw arch string. N.B. see instead perf_env__arch() or + * machine__normalized_is() if a normalized arch is needed. */ bool machine__is(struct machine *machine, const char *arch) { return machine && !strcmp(perf_env__raw_arch(machine->env), arch); } +bool machine__normalized_is(struct machine *machine, const char *arch) +{ + return machine && !strcmp(perf_env__arch(machine->env), arch); +} + int machine__nr_cpus_avail(struct machine *machine) { return machine ? perf_env__nr_cpus_avail(machine->env) : 0; @@ -2727,3 +3309,30 @@ char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, ch *addrp = map->unmap_ip(map, sym->start); return sym->name; } + +int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv) +{ + struct dso *pos; + int err = 0; + + list_for_each_entry(pos, &machine->dsos.head, node) { + if (fn(pos, machine, priv)) + err = -1; + } + return err; +} + +int machine__for_each_kernel_map(struct machine *machine, machine__map_t fn, void *priv) +{ + struct maps *maps = machine__kernel_maps(machine); + struct map *map; + int err = 0; + + for (map = maps__first(maps); map != NULL; map = map__next(map)) { + err = fn(map, priv); + if (err != 0) { + break; + } + } + return err; +} |