diff options
Diffstat (limited to 'tools/perf/Documentation')
27 files changed, 1382 insertions, 105 deletions
diff --git a/tools/perf/Documentation/arm-coresight.txt b/tools/perf/Documentation/arm-coresight.txt new file mode 100644 index 000000000000..c117fc50a2a9 --- /dev/null +++ b/tools/perf/Documentation/arm-coresight.txt @@ -0,0 +1,5 @@ +Arm CoreSight Support +===================== + +For full documentation, see Documentation/trace/coresight/coresight-perf.rst +in the kernel tree. diff --git a/tools/perf/Documentation/guest-files.txt b/tools/perf/Documentation/guest-files.txt new file mode 100644 index 000000000000..8cc0b092f996 --- /dev/null +++ b/tools/perf/Documentation/guest-files.txt @@ -0,0 +1,16 @@ +include::guestmount.txt[] + +--guestkallsyms=<path>:: + Guest OS /proc/kallsyms file copy. perf reads it to get guest + kernel symbols. Users copy it out from guest OS. + +--guestmodules=<path>:: + Guest OS /proc/modules file copy. perf reads it to get guest + kernel module information. Users copy it out from guest OS. + +--guestvmlinux=<path>:: + Guest OS kernel vmlinux. + +--guest-code:: + Indicate that guest code can be found in the hypervisor process, + which is a common case for KVM test programs. diff --git a/tools/perf/Documentation/guestmount.txt b/tools/perf/Documentation/guestmount.txt new file mode 100644 index 000000000000..6edf12363add --- /dev/null +++ b/tools/perf/Documentation/guestmount.txt @@ -0,0 +1,11 @@ +--guestmount=<path>:: + Guest OS root file system mount directory. Users mount guest OS + root directories under <path> by a specific filesystem access method, + typically, sshfs. + For example, start 2 guest OS, one's pid is 8888 and the other's is 9999: +[verse] + $ mkdir \~/guestmount + $ cd \~/guestmount + $ sshfs -o allow_other,direct_io -p 5551 localhost:/ 8888/ + $ sshfs -o allow_other,direct_io -p 5552 localhost:/ 9999/ + $ perf {GMEXAMPLECMD} --guestmount=~/guestmount {GMEXAMPLESUBCMD} diff --git a/tools/perf/Documentation/intel-hybrid.txt b/tools/perf/Documentation/intel-hybrid.txt index c9302096dc46..e7a776ad25d7 100644 --- a/tools/perf/Documentation/intel-hybrid.txt +++ b/tools/perf/Documentation/intel-hybrid.txt @@ -21,11 +21,6 @@ cat /sys/devices/cpu_atom/cpus It indicates cpu0-cpu15 are core cpus and cpu16-cpu23 are atom cpus. -Quickstart - -List hybrid event ------------------ - As before, use perf-list to list the symbolic event. perf list @@ -40,7 +35,6 @@ the event is belong to. Same event name but with different pmu can be supported. Enable hybrid event with a specific pmu ---------------------------------------- To enable a core only event or atom only event, following syntax is supported: @@ -53,7 +47,6 @@ For example, count the 'cycles' event on core cpus. perf stat -e cpu_core/cycles/ Create two events for one hardware event automatically ------------------------------------------------------- When creating one event and the event is available on both atom and core, two events are created automatically. One is for atom, the other is for @@ -132,7 +125,6 @@ For perf-stat result, it displays two events: The first 'cycles' is core event, the second 'cycles' is atom event. Thread mode example: --------------------- perf-stat reports the scaled counts for hybrid event and with a percentage displayed. The percentage is the event's running time/enabling time. @@ -176,14 +168,12 @@ perf_event_attr: 604,097,080 cpu_atom/cycles/ (99.57%) perf-record: ------------- If there is no '-e' specified in perf record, on hybrid platform, it creates two default 'cycles' and adds them to event list. One is for core, the other is for atom. perf-stat: ----------- If there is no '-e' specified in perf stat, on hybrid platform, besides of software events, following events are created and diff --git a/tools/perf/Documentation/itrace.txt b/tools/perf/Documentation/itrace.txt index c52755481e2f..0916bbfe64cb 100644 --- a/tools/perf/Documentation/itrace.txt +++ b/tools/perf/Documentation/itrace.txt @@ -7,6 +7,8 @@ p synthesize power events (incl. PSB events for Intel PT) o synthesize other events recorded due to the use of aux-output (refer to perf record) + I synthesize interrupt or similar (asynchronous) events + (e.g. Intel PT Event Trace) e synthesize error events d create a debug log f synthesize first level cache events @@ -62,6 +64,7 @@ debug messages will or will not be logged. Each flag must be preceded by either '+' or '-'. The flags are: a all perf events + e output only on errors (size configurable - see linkperf:perf-config[1]) o output to stdout If supported, the 'q' option may be repeated to increase the effect. diff --git a/tools/perf/Documentation/perf-annotate.txt b/tools/perf/Documentation/perf-annotate.txt index 33c2521cba4a..18fcc52809fb 100644 --- a/tools/perf/Documentation/perf-annotate.txt +++ b/tools/perf/Documentation/perf-annotate.txt @@ -147,6 +147,11 @@ include::itrace.txt[] The period/hits keywords set the base the percentage is computed on - the samples period or the number of samples (hits). +--percent-limit:: + Do not show functions which have an overhead under that percent on + stdio or stdio2 (Default: 0). Note that this is about selection of + functions to display, not about lines within the function. + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-report[1] diff --git a/tools/perf/Documentation/perf-arm-spe.txt b/tools/perf/Documentation/perf-arm-spe.txt new file mode 100644 index 000000000000..bf03222e9a68 --- /dev/null +++ b/tools/perf/Documentation/perf-arm-spe.txt @@ -0,0 +1,218 @@ +perf-arm-spe(1) +================ + +NAME +---- +perf-arm-spe - Support for Arm Statistical Profiling Extension within Perf tools + +SYNOPSIS +-------- +[verse] +'perf record' -e arm_spe// + +DESCRIPTION +----------- + +The SPE (Statistical Profiling Extension) feature provides accurate attribution of latencies and + events down to individual instructions. Rather than being interrupt-driven, it picks an +instruction to sample and then captures data for it during execution. Data includes execution time +in cycles. For loads and stores it also includes data address, cache miss events, and data origin. + +The sampling has 5 stages: + + 1. Choose an operation + 2. Collect data about the operation + 3. Optionally discard the record based on a filter + 4. Write the record to memory + 5. Interrupt when the buffer is full + +Choose an operation +~~~~~~~~~~~~~~~~~~~ + +This is chosen from a sample population, for SPE this is an IMPLEMENTATION DEFINED choice of all +architectural instructions or all micro-ops. Sampling happens at a programmable interval. The +architecture provides a mechanism for the SPE driver to infer the minimum interval at which it should +sample. This minimum interval is used by the driver if no interval is specified. A pseudo-random +perturbation is also added to the sampling interval by default. + +Collect data about the operation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Program counter, PMU events, timings and data addresses related to the operation are recorded. +Sampling ensures there is only one sampled operation is in flight. + +Optionally discard the record based on a filter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Based on programmable criteria, choose whether to keep the record or discard it. If the record is +discarded then the flow stops here for this sample. + +Write the record to memory +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The record is appended to a memory buffer + +Interrupt when the buffer is full +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the buffer fills, an interrupt is sent and the driver signals Perf to collect the records. +Perf saves the raw data in the perf.data file. + +Opening the file +---------------- + +Up until this point no decoding of the SPE data was done by either the kernel or Perf. Only when the +recorded file is opened with 'perf report' or 'perf script' does the decoding happen. When decoding +the data, Perf generates "synthetic samples" as if these were generated at the time of the +recording. These samples are the same as if normal sampling was done by Perf without using SPE, +although they may have more attributes associated with them. For example a normal sample may have +just the instruction pointer, but an SPE sample can have data addresses and latency attributes. + +Why Sampling? +------------- + + - Sampling, rather than tracing, cuts down the profiling problem to something more manageable for + hardware. Only one sampled operation is in flight at a time. + + - Allows precise attribution data, including: Full PC of instruction, data virtual and physical + addresses. + + - Allows correlation between an instruction and events, such as TLB and cache miss. (Data source + indicates which particular cache was hit, but the meaning is implementation defined because + different implementations can have different cache configurations.) + +However, SPE does not provide any call-graph information, and relies on statistical methods. + +Collisions +---------- + +When an operation is sampled while a previous sampled operation has not finished, a collision +occurs. The new sample is dropped. Collisions affect the integrity of the data, so the sample rate +should be set to avoid collisions. + +The 'sample_collision' PMU event can be used to determine the number of lost samples. Although this +count is based on collisions _before_ filtering occurs. Therefore this can not be used as an exact +number for samples dropped that would have made it through the filter, but can be a rough +guide. + +The effect of microarchitectural sampling +----------------------------------------- + +If an implementation samples micro-operations instead of instructions, the results of sampling must +be weighted accordingly. + +For example, if a given instruction A is always converted into two micro-operations, A0 and A1, it +becomes twice as likely to appear in the sample population. + +The coarse effect of conversions, and, if applicable, sampling of speculative operations, can be +estimated from the 'sample_pop' and 'inst_retired' PMU events. + +Kernel Requirements +------------------- + +The ARM_SPE_PMU config must be set to build as either a module or statically. + +Depending on CPU model, the kernel may need to be booted with page table isolation disabled +(kpti=off). If KPTI needs to be disabled, this will fail with a console message "profiling buffer +inaccessible. Try passing 'kpti=off' on the kernel command line". + +Capturing SPE with perf command-line tools +------------------------------------------ + +You can record a session with SPE samples: + + perf record -e arm_spe// -- ./mybench + +The sample period is set from the -c option, and because the minimum interval is used by default +it's recommended to set this to a higher value. The value is written to PMSIRR.INTERVAL. + +Config parameters +~~~~~~~~~~~~~~~~~ + +These are placed between the // in the event and comma separated. For example '-e +arm_spe/load_filter=1,min_latency=10/' + + branch_filter=1 - collect branches only (PMSFCR.B) + event_filter=<mask> - filter on specific events (PMSEVFR) - see bitfield description below + jitter=1 - use jitter to avoid resonance when sampling (PMSIRR.RND) + load_filter=1 - collect loads only (PMSFCR.LD) + min_latency=<n> - collect only samples with this latency or higher* (PMSLATFR) + pa_enable=1 - collect physical address (as well as VA) of loads/stores (PMSCR.PA) - requires privilege + pct_enable=1 - collect physical timestamp instead of virtual timestamp (PMSCR.PCT) - requires privilege + store_filter=1 - collect stores only (PMSFCR.ST) + ts_enable=1 - enable timestamping with value of generic timer (PMSCR.TS) + ++++*+++ Latency is the total latency from the point at which sampling started on that instruction, rather +than only the execution latency. + +Only some events can be filtered on; these include: + + bit 1 - instruction retired (i.e. omit speculative instructions) + bit 3 - L1D refill + bit 5 - TLB refill + bit 7 - mispredict + bit 11 - misaligned access + +So to sample just retired instructions: + + perf record -e arm_spe/event_filter=2/ -- ./mybench + +or just mispredicted branches: + + perf record -e arm_spe/event_filter=0x80/ -- ./mybench + +Viewing the data +~~~~~~~~~~~~~~~~~ + +By default perf report and perf script will assign samples to separate groups depending on the +attributes/events of the SPE record. Because instructions can have multiple events associated with +them, the samples in these groups are not necessarily unique. For example perf report shows these +groups: + + Available samples + 0 arm_spe// + 0 dummy:u + 21 l1d-miss + 897 l1d-access + 5 llc-miss + 7 llc-access + 2 tlb-miss + 1K tlb-access + 36 branch-miss + 0 remote-access + 900 memory + +The arm_spe// and dummy:u events are implementation details and are expected to be empty. + +To get a full list of unique samples that are not sorted into groups, set the itrace option to +generate 'instruction' samples. The period option is also taken into account, so set it to 1 +instruction unless you want to further downsample the already sampled SPE data: + + perf report --itrace=i1i + +Memory access details are also stored on the samples and this can be viewed with: + + perf report --mem-mode + +Common errors +~~~~~~~~~~~~~ + + - "Cannot find PMU `arm_spe'. Missing kernel support?" + + Module not built or loaded, KPTI not disabled (see above), or running on a VM + + - "Arm SPE CONTEXT packets not found in the traces." + + Root privilege is required to collect context packets. But these only increase the accuracy of + assigning PIDs to kernel samples. For userspace sampling this can be ignored. + + - Excessively large perf.data file size + + Increase sampling interval (see above) + + +SEE ALSO +-------- + +linkperf:perf-record[1], linkperf:perf-script[1], linkperf:perf-report[1], +linkperf:perf-inject[1] diff --git a/tools/perf/Documentation/perf-buildid-cache.txt b/tools/perf/Documentation/perf-buildid-cache.txt index cd8ce6e8ec12..7e44b419d301 100644 --- a/tools/perf/Documentation/perf-buildid-cache.txt +++ b/tools/perf/Documentation/perf-buildid-cache.txt @@ -74,12 +74,15 @@ OPTIONS used when creating a uprobe for a process that resides in a different mount namespace from the perf(1) utility. ---debuginfod=URLs:: +--debuginfod[=URLs]:: Specify debuginfod URL to be used when retrieving perf.data binaries, it follows the same syntax as the DEBUGINFOD_URLS variable, like: buildid-cache.debuginfod=http://192.168.122.174:8002 + If the URLs is not specified, the value of DEBUGINFOD_URLS + system environment variable is used. + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-report[1], linkperf:perf-buildid-list[1] diff --git a/tools/perf/Documentation/perf-buildid-list.txt b/tools/perf/Documentation/perf-buildid-list.txt index 25c52efcc7f0..e1e8fdbe06b9 100644 --- a/tools/perf/Documentation/perf-buildid-list.txt +++ b/tools/perf/Documentation/perf-buildid-list.txt @@ -33,6 +33,10 @@ OPTIONS -k:: --kernel:: Show running kernel build id. +-m:: +--kernel-maps:: + Show buildid, start/end text address, and path of running kernel and + its modules. -v:: --verbose:: Be more verbose. diff --git a/tools/perf/Documentation/perf-c2c.txt b/tools/perf/Documentation/perf-c2c.txt index 3b6a2c84ea02..5c5eb2def83e 100644 --- a/tools/perf/Documentation/perf-c2c.txt +++ b/tools/perf/Documentation/perf-c2c.txt @@ -19,9 +19,10 @@ C2C stands for Cache To Cache. The perf c2c tool provides means for Shared Data C2C/HITM analysis. It allows you to track down the cacheline contentions. -On x86, the tool is based on load latency and precise store facility events +On Intel, the tool is based on load latency and precise store facility events provided by Intel CPUs. On PowerPC, the tool uses random instruction sampling -with thresholding feature. +with thresholding feature. On AMD, the tool uses IBS op pmu (due to hardware +limitations, perf c2c is not supported on Zen3 cpus). These events provide: - memory address of the access @@ -49,7 +50,8 @@ RECORD OPTIONS -l:: --ldlat:: - Configure mem-loads latency. (x86 only) + Configure mem-loads latency. Supported on Intel and Arm64 processors + only. Ignored on other archs. -k:: --all-kernel:: @@ -109,7 +111,9 @@ REPORT OPTIONS -d:: --display:: - Switch to HITM type (rmt, lcl) to display and sort on. Total HITMs as default. + Switch to HITM type (rmt, lcl) or peer snooping type (peer) to display + and sort on. Total HITMs (tot) as default, except Arm64 uses peer mode + as default. --stitch-lbr:: Show callgraph with stitched LBRs, which may have more complete @@ -133,11 +137,15 @@ Following perf record options are configured by default: -W,-d,--phys-data,--sample-cpu Unless specified otherwise with '-e' option, following events are monitored by -default on x86: +default on Intel: cpu/mem-loads,ldlat=30/P cpu/mem-stores/P +following on AMD: + + ibs_op// + and following on PowerPC: cpu/mem-loads/ @@ -174,12 +182,18 @@ For each cacheline in the 1) list we display following data: Cacheline - cacheline address (hex number) - Rmt/Lcl Hitm + Rmt/Lcl Hitm (Display with HITM types) - cacheline percentage of all Remote/Local HITM accesses - LLC Load Hitm - Total, LclHitm, RmtHitm + Peer Snoop (Display with peer type) + - cacheline percentage of all peer accesses + + LLC Load Hitm - Total, LclHitm, RmtHitm (For display with HITM types) - count of Total/Local/Remote load HITMs + Load Peer - Total, Local, Remote (For display with peer type) + - count of Total/Local/Remote load from peer cache or DRAM + Total records - sum of all cachelines accesses @@ -189,9 +203,10 @@ For each cacheline in the 1) list we display following data: Total stores - sum of all store accesses - Store Reference - L1Hit, L1Miss + Store Reference - L1Hit, L1Miss, N/A L1Hit - store accesses that hit L1 L1Miss - store accesses that missed L1 + N/A - store accesses with memory level is not available Core Load Hit - FB, L1, L2 - count of load hits in FB (Fill Buffer), L1 and L2 cache @@ -200,18 +215,24 @@ For each cacheline in the 1) list we display following data: - count of LLC load accesses, includes LLC hits and LLC HITMs RMT Load Hit - RmtHit, RmtHitm - - count of remote load accesses, includes remote hits and remote HITMs + - count of remote load accesses, includes remote hits and remote HITMs; + on Arm neoverse cores, RmtHit is used to account remote accesses, + includes remote DRAM or any upward cache level in remote node Load Dram - Lcl, Rmt - count of local and remote DRAM accesses For each offset in the 2) list we display following data: - HITM - Rmt, Lcl + HITM - Rmt, Lcl (Display with HITM types) - % of Remote/Local HITM accesses for given offset within cacheline - Store Refs - L1 Hit, L1 Miss - - % of store accesses that hit/missed L1 for given offset within cacheline + Peer Snoop - Rmt, Lcl (Display with peer type) + - % of Remote/Local peer accesses for given offset within cacheline + + Store Refs - L1 Hit, L1 Miss, N/A + - % of store accesses that hit L1, missed L1 and N/A (no available) memory + level for given offset within cacheline Data address - Offset - offset address @@ -225,9 +246,12 @@ For each offset in the 2) list we display following data: Code address - code address responsible for the accesses - cycles - rmt hitm, lcl hitm, load + cycles - rmt hitm, lcl hitm, load (Display with HITM types) - sum of cycles for given accesses - Remote/Local HITM and generic load + cycles - rmt peer, lcl peer, load (Display with peer type) + - sum of cycles for given accesses - Remote/Local peer load and generic load + cpu cnt - number of cpus that participated on the access @@ -249,7 +273,8 @@ The 'Node' field displays nodes that accesses given cacheline offset. Its output comes in 3 flavors: - node IDs separated by ',' - node IDs with stats for each ID, in following format: - Node{cpus %hitms %stores} + Node{cpus %hitms %stores} (Display with HITM types) + Node{cpus %peers %stores} (Display with peer type) - node IDs with list of affected CPUs in following format: Node{cpu list} diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt index 3bb75c1f25e8..39c890ead2dc 100644 --- a/tools/perf/Documentation/perf-config.txt +++ b/tools/perf/Documentation/perf-config.txt @@ -587,6 +587,15 @@ record.*:: Use 'n' control blocks in asynchronous (Posix AIO) trace writing mode ('n' default: 1, max: 4). + record.debuginfod:: + Specify debuginfod URL to be used when cacheing perf.data binaries, + it follows the same syntax as the DEBUGINFOD_URLS variable, like: + + http://192.168.122.174:8002 + + If the URLs is 'system', the value of DEBUGINFOD_URLS system environment + variable is used. + diff.*:: diff.order:: This option sets the number of columns to sort the result. @@ -720,6 +729,13 @@ auxtrace.*:: If the directory does not exist or has the wrong file type, the current directory is used. +itrace.*:: + + debug-log-buffer-size:: + Log size in bytes to output when using the option --itrace=d+e + Refer 'itrace' option of linkperf:perf-script[1] or + linkperf:perf-report[1]. The default is 16384. + daemon.*:: daemon.base:: diff --git a/tools/perf/Documentation/perf-dlfilter.txt b/tools/perf/Documentation/perf-dlfilter.txt index 594f5a5a0c9e..fb22e3b31dc5 100644 --- a/tools/perf/Documentation/perf-dlfilter.txt +++ b/tools/perf/Documentation/perf-dlfilter.txt @@ -107,9 +107,31 @@ struct perf_dlfilter_sample { __u64 raw_callchain_nr; /* Number of raw_callchain entries */ const __u64 *raw_callchain; /* Refer <linux/perf_event.h> */ const char *event; + __s32 machine_pid; + __s32 vcpu; }; ---- +Note: 'machine_pid' and 'vcpu' are not original members, but were added together later. +'size' can be used to determine their presence at run time. +PERF_DLFILTER_HAS_MACHINE_PID will be defined if they are present at compile time. +For example: +[source,c] +---- +#include <perf/perf_dlfilter.h> +#include <stddef.h> +#include <stdbool.h> + +static inline bool have_machine_pid(const struct perf_dlfilter_sample *sample) +{ +#ifdef PERF_DLFILTER_HAS_MACHINE_PID + return sample->size >= offsetof(struct perf_dlfilter_sample, vcpu) + sizeof(sample->vcpu); +#else + return false; +#endif +} +---- + The perf_dlfilter_fns structure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tools/perf/Documentation/perf-ftrace.txt b/tools/perf/Documentation/perf-ftrace.txt index 6e82b7cc0bf0..df4595563801 100644 --- a/tools/perf/Documentation/perf-ftrace.txt +++ b/tools/perf/Documentation/perf-ftrace.txt @@ -9,32 +9,24 @@ perf-ftrace - simple wrapper for kernel's ftrace functionality SYNOPSIS -------- [verse] -'perf ftrace' <command> +'perf ftrace' {trace|latency} <command> DESCRIPTION ----------- -The 'perf ftrace' command is a simple wrapper of kernel's ftrace -functionality. It only supports single thread tracing currently and -just reads trace_pipe in text and then write it to stdout. +The 'perf ftrace' command provides a collection of subcommands which use +kernel's ftrace infrastructure. -The following options apply to perf ftrace. + 'perf ftrace trace' is a simple wrapper of the ftrace. It only supports + single thread tracing currently and just reads trace_pipe in text and then + write it to stdout. -OPTIONS -------- + 'perf ftrace latency' calculates execution latency of a given function + (optionally with BPF) and display it as a histogram. --t:: ---tracer=:: - Tracer to use when neither -G nor -F option is not - specified: function_graph or function. +The following options apply to perf ftrace. --v:: ---verbose:: - Increase the verbosity level. - --F:: ---funcs:: - List available functions to trace. It accepts a pattern to - only list interested functions. +COMMON OPTIONS +-------------- -p:: --pid=:: @@ -43,10 +35,6 @@ OPTIONS --tid=:: Trace on existing thread id (comma separated list). --D:: ---delay:: - Time (ms) to wait before starting tracing after program start. - -a:: --all-cpus:: Force system-wide collection. Scripts run without a <command> @@ -61,6 +49,28 @@ OPTIONS Ranges of CPUs are specified with -: 0-2. Default is to trace on all online CPUs. +-v:: +--verbose:: + Increase the verbosity level. + + +OPTIONS for 'perf ftrace trace' +------------------------------- + +-t:: +--tracer=:: + Tracer to use when neither -G nor -F option is not + specified: function_graph or function. + +-F:: +--funcs:: + List available functions to trace. It accepts a pattern to + only list interested functions. + +-D:: +--delay:: + Time (ms) to wait before starting tracing after program start. + -m:: --buffer-size:: Set the size of per-cpu tracing buffer, <size> is expected to @@ -114,6 +124,25 @@ OPTIONS thresh=<n> - Setup trace duration threshold in microseconds. depth=<n> - Set max depth for function graph tracer to follow. + +OPTIONS for 'perf ftrace latency' +--------------------------------- + +-T:: +--trace-funcs=:: + Set the function name to get the histogram. Unlike perf ftrace trace, + it only allows single function to calculate the histogram. + +-b:: +--use-bpf:: + Use BPF to measure function latency instead of using the ftrace (it + uses function_graph tracer internally). + +-n:: +--use-nsec:: + Use nano-second instead of micro-second as a base unit of the histogram. + + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-trace[1] diff --git a/tools/perf/Documentation/perf-inject.txt b/tools/perf/Documentation/perf-inject.txt index 0570a1ccd344..c972032f4ca0 100644 --- a/tools/perf/Documentation/perf-inject.txt +++ b/tools/perf/Documentation/perf-inject.txt @@ -25,10 +25,17 @@ OPTIONS ------- -b:: --build-ids:: - Inject build-ids into the output stream + Inject build-ids of DSOs hit by samples into the output stream. + This means it needs to process all SAMPLE records to find the DSOs. ---buildid-all: - Inject build-ids of all DSOs into the output stream +--buildid-all:: + Inject build-ids of all DSOs into the output stream regardless of hits + and skip SAMPLE processing. + +--known-build-ids=:: + Override build-ids to inject using these comma-separated pairs of + build-id and path. Understands file://filename to read these pairs + from a file, which can be generated with perf buildid-list. -v:: --verbose:: @@ -85,6 +92,27 @@ include::itrace.txt[] without updating it. Currently this option is supported only by Intel PT, refer linkperf:perf-intel-pt[1] +--guest-data=<path>,<pid>[,<time offset>[,<time scale>]]:: + Insert events from a perf.data file recorded in a virtual machine at + the same time as the input perf.data file was recorded on the host. + The Process ID (PID) of the QEMU hypervisor process must be provided, + and the time offset and time scale (multiplier) will likely be needed + to convert guest time stamps into host time stamps. For example, for + x86 the TSC Offset and Multiplier could be provided for a virtual machine + using Linux command line option no-kvmclock. + Currently only mmap, mmap2, comm, task, context_switch, ksymbol, + and text_poke events are inserted, as well as build ID information. + The QEMU option -name debug-threads=on is needed so that thread names + can be used to determine which thread is running which VCPU. Note + libvirt seems to use this by default. + When using perf record in the guest, option --sample-identifier + should be used, and also --buildid-all and --switch-events may be + useful. + +:GMEXAMPLECMD: inject +:GMEXAMPLESUBCMD: +include::guestmount.txt[] + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-report[1], linkperf:perf-archive[1], diff --git a/tools/perf/Documentation/perf-intel-pt.txt b/tools/perf/Documentation/perf-intel-pt.txt index cbb920f5d056..92464a5d7eaf 100644 --- a/tools/perf/Documentation/perf-intel-pt.txt +++ b/tools/perf/Documentation/perf-intel-pt.txt @@ -108,9 +108,10 @@ displayed as follows: perf script --itrace=ibxwpe -F+flags -The flags are "bcrosyiABExgh" which stand for branch, call, return, conditional, +The flags are "bcrosyiABExghDt" which stand for branch, call, return, conditional, system, asynchronous, interrupt, transaction abort, trace begin, trace end, -in transaction, VM-entry, and VM-exit respectively. +in transaction, VM-entry, VM-exit, interrupt disabled, and interrupt disable +toggle respectively. perf script also supports higher level ways to dump instruction traces: @@ -266,7 +267,7 @@ Note that, as with all events, the event is suffixed with event modifiers: H host p precise ip -'h', 'G' and 'H' are for virtualization which is not supported by Intel PT. +'h', 'G' and 'H' are for virtualization which are not used by Intel PT. 'p' is also not relevant to Intel PT. So only options 'u' and 'k' are meaningful for Intel PT. @@ -467,6 +468,8 @@ ptw Enable PTWRITE packets which are produced when a ptwrite instruction which contains "1" if the feature is supported and "0" otherwise. + As an alternative, refer to "Emulated PTWRITE" further below. + fup_on_ptw Enable a FUP packet to follow the PTWRITE packet. The FUP packet provides the address of the ptwrite instruction. In the absence of fup_on_ptw, the decoder will use the address of the previous branch @@ -483,6 +486,30 @@ pwr_evt Enable power events. The power events provide information about which contains "1" if the feature is supported and "0" otherwise. +event Enable Event Trace. The events provide information about asynchronous + events. + + Support for this feature is indicated by: + + /sys/bus/event_source/devices/intel_pt/caps/event_trace + + which contains "1" if the feature is supported and + "0" otherwise. + +notnt Disable TNT packets. Without TNT packets, it is not possible to walk + executable code to reconstruct control flow, however FUP, TIP, TIP.PGE + and TIP.PGD packets still indicate asynchronous control flow, and (if + return compression is disabled - see noretcomp) return statements. + The advantage of eliminating TNT packets is reducing the size of the + trace and corresponding tracing overhead. + + Support for this feature is indicated by: + + /sys/bus/event_source/devices/intel_pt/caps/tnt_disable + + which contains "1" if the feature is supported and + "0" otherwise. + AUX area sampling option ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -876,6 +903,8 @@ The letters are: p synthesize "power" events (incl. PSB events) c synthesize branches events (calls only) r synthesize branches events (returns only) + o synthesize PEBS-via-PT events + I synthesize Event Trace events e synthesize tracing error events d create a debug log g synthesize a call chain (use with i or x) @@ -914,12 +943,15 @@ event packets are recorded only if the "pwr_evt" config term was used. Refer to the config terms section above. The power events record information about C-state changes, whereas CBR is indicative of CPU frequency. perf script "event,synth" fields display information like this: + cbr: cbr: 22 freq: 2189 MHz (200%) mwait: hints: 0x60 extensions: 0x1 pwre: hw: 0 cstate: 2 sub-cstate: 0 exstop: ip: 1 pwrx: deepest cstate: 2 last cstate: 2 wake reason: 0x4 + Where: + "cbr" includes the frequency and the percentage of maximum non-turbo "mwait" shows mwait hints and extensions "pwre" shows C-state transitions (to a C-state deeper than C0) and @@ -927,6 +959,7 @@ Where: "exstop" indicates execution stopped and whether the IP was recorded exactly, "pwrx" indicates return to C0 + For more details refer to the Intel 64 and IA-32 Architectures Software Developer Manuals. @@ -940,8 +973,10 @@ are quite important. Users must know if what they are seeing is a complete picture or not. The "e" option may be followed by flags which affect what errors will or will not be reported. Each flag must be preceded by either '+' or '-'. The flags supported by Intel PT are: + -o Suppress overflow errors -l Suppress trace data lost errors + For example, for errors but not overflow or data lost errors: --itrace=e-o-l @@ -951,11 +986,16 @@ decoded packets and instructions. Note that this option slows down the decoder and that the resulting file may be very large. The "d" option may be followed by flags which affect what debug messages will or will not be logged. Each flag must be preceded by either '+' or '-'. The flags support by Intel PT are: + -a Suppress logging of perf events +a Log all perf events + +e Output only on decoding errors (size configurable) +o Output to stdout instead of "intel_pt.log" + By default, logged perf events are filtered by any specified time ranges, but -flag +a overrides that. +flag +a overrides that. The +e flag can be useful for analyzing errors. By +default, the log size in that case is 16384 bytes, but can be altered by +linkperf:perf-config[1] e.g. perf config itrace.debug-log-buffer-size=30000 In addition, the period of the "instructions" event can be specified. e.g. @@ -1189,10 +1229,10 @@ XED include::build-xed.txt[] -Tracing Virtual Machines ------------------------- +Tracing Virtual Machines (kernel only) +-------------------------------------- -Currently, only kernel tracing is supported and only with either "timeless" decoding +Currently, kernel tracing is supported with either "timeless" decoding (i.e. no TSC timestamps) or VM Time Correlation. VM Time Correlation is an extra step using 'perf inject' and requires unchanging VMX TSC Offset and no VMX TSC Scaling. @@ -1371,6 +1411,415 @@ There were none. :17006 17006 [001] 11500.262869216: ffffffff8220116e error_entry+0xe ([guest.kernel.kallsyms]) pushq %rax +Tracing Virtual Machines (including user space) +----------------------------------------------- + +It is possible to use perf record to record sideband events within a virtual machine, so that an Intel PT trace on the host can be decoded. +Sideband events from the guest perf.data file can be injected into the host perf.data file using perf inject. + +Here is an example of the steps needed: + +On the guest machine: + +Check that no-kvmclock kernel command line option was used to boot: + +Note, this is essential to enable time correlation between host and guest machines. + + $ cat /proc/cmdline + BOOT_IMAGE=/boot/vmlinuz-5.10.0-16-amd64 root=UUID=cb49c910-e573-47e0-bce7-79e293df8e1d ro no-kvmclock + +There is no BPF support at present so, if possible, disable JIT compiling: + + $ echo 0 | sudo tee /proc/sys/net/core/bpf_jit_enable + 0 + +Start perf record to collect sideband events: + + $ sudo perf record -o guest-sideband-testing-guest-perf.data --sample-identifier --buildid-all --switch-events --kcore -a -e dummy + +On the host machine: + +Start perf record to collect Intel PT trace: + +Note, the host trace will get very big, very fast, so the steps from starting to stopping the host trace really need to be done so that they happen in the shortest time possible. + + $ sudo perf record -o guest-sideband-testing-host-perf.data -m,64M --kcore -a -e intel_pt/cyc/ + +On the guest machine: + +Run a small test case, just 'uname' in this example: + + $ uname + Linux + +On the host machine: + +Stop the Intel PT trace: + + ^C + [ perf record: Woken up 1 times to write data ] + [ perf record: Captured and wrote 76.122 MB guest-sideband-testing-host-perf.data ] + +On the guest machine: + +Stop the Intel PT trace: + + ^C + [ perf record: Woken up 1 times to write data ] + [ perf record: Captured and wrote 1.247 MB guest-sideband-testing-guest-perf.data ] + +And then copy guest-sideband-testing-guest-perf.data to the host (not shown here). + +On the host machine: + +With the 2 perf.data recordings, and with their ownership changed to the user. + +Identify the TSC Offset: + + $ perf inject -i guest-sideband-testing-host-perf.data --vm-time-correlation=dry-run + VMCS: 0x103fc6 TSC Offset 0xfffffa6ae070cb20 + VMCS: 0x103ff2 TSC Offset 0xfffffa6ae070cb20 + VMCS: 0x10fdaa TSC Offset 0xfffffa6ae070cb20 + VMCS: 0x24d57c TSC Offset 0xfffffa6ae070cb20 + +Correct Intel PT TSC timestamps for the guest machine: + + $ perf inject -i guest-sideband-testing-host-perf.data --vm-time-correlation=0xfffffa6ae070cb20 --force + +Identify the guest machine PID: + + $ perf script -i guest-sideband-testing-host-perf.data --no-itrace --show-task-events | grep KVM + CPU 0/KVM 0 [000] 0.000000: PERF_RECORD_COMM: CPU 0/KVM:13376/13381 + CPU 1/KVM 0 [000] 0.000000: PERF_RECORD_COMM: CPU 1/KVM:13376/13382 + CPU 2/KVM 0 [000] 0.000000: PERF_RECORD_COMM: CPU 2/KVM:13376/13383 + CPU 3/KVM 0 [000] 0.000000: PERF_RECORD_COMM: CPU 3/KVM:13376/13384 + +Note, the QEMU option -name debug-threads=on is needed so that thread names +can be used to determine which thread is running which VCPU as above. libvirt seems to use this by default. + +Create a guestmount, assuming the guest machine is 'vm_to_test': + + $ mkdir -p ~/guestmount/13376 + $ sshfs -o direct_io vm_to_test:/ ~/guestmount/13376 + +Inject the guest perf.data file into the host perf.data file: + +Note, due to the guestmount option, guest object files and debug files will be copied into the build ID cache from the guest machine, with the notable exception of VDSO. +If needed, VDSO can be copied manually in a fashion similar to that used by the perf-archive script. + + $ perf inject -i guest-sideband-testing-host-perf.data -o inj --guestmount ~/guestmount --guest-data=guest-sideband-testing-guest-perf.data,13376,0xfffffa6ae070cb20 + +Show an excerpt from the result. In this case the CPU and time range have been to chosen to show interaction between guest and host when 'uname' is starting to run on the guest machine: + +Notes: + + - the CPU displayed, [002] in this case, is always the host CPU + - events happening in the virtual machine start with VM:13376 VCPU:003, which shows the hypervisor PID 13376 and the VCPU number + - only calls and errors are displayed i.e. --itrace=ce + - branches entering and exiting the virtual machine are split, and show as 2 branches to/from "0 [unknown] ([unknown])" + + $ perf script -i inj --itrace=ce -F+machine_pid,+vcpu,+addr,+pid,+tid,-period --ns --time 7919.408803365,7919.408804631 -C 2 + CPU 3/KVM 13376/13384 [002] 7919.408803365: branches: ffffffffc0f8ebe0 vmx_vcpu_enter_exit+0xc0 ([kernel.kallsyms]) => ffffffffc0f8edc0 __vmx_vcpu_run+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803365: branches: ffffffffc0f8edd5 __vmx_vcpu_run+0x15 ([kernel.kallsyms]) => ffffffffc0f8eca0 vmx_update_host_rsp+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803365: branches: ffffffffc0f8ee1b __vmx_vcpu_run+0x5b ([kernel.kallsyms]) => ffffffffc0f8ed60 vmx_vmenter+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803461: branches: ffffffffc0f8ed62 vmx_vmenter+0x2 ([kernel.kallsyms]) => 0 [unknown] ([unknown]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408803461: branches: 0 [unknown] ([unknown]) => 7f851c9b5a5c init_cacheinfo+0x3ac (/usr/lib/x86_64-linux-gnu/libc-2.31.so) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408803567: branches: 7f851c9b5a5a init_cacheinfo+0x3aa (/usr/lib/x86_64-linux-gnu/libc-2.31.so) => 0 [unknown] ([unknown]) + CPU 3/KVM 13376/13384 [002] 7919.408803567: branches: 0 [unknown] ([unknown]) => ffffffffc0f8ed80 vmx_vmexit+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803596: branches: ffffffffc0f6619a vmx_vcpu_run+0x26a ([kernel.kallsyms]) => ffffffffb2255c60 x86_virt_spec_ctrl+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803801: branches: ffffffffc0f66445 vmx_vcpu_run+0x515 ([kernel.kallsyms]) => ffffffffb2290b30 native_write_msr+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803850: branches: ffffffffc0f661f8 vmx_vcpu_run+0x2c8 ([kernel.kallsyms]) => ffffffffc1092300 kvm_load_host_xsave_state+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803850: branches: ffffffffc1092327 kvm_load_host_xsave_state+0x27 ([kernel.kallsyms]) => ffffffffc1092220 kvm_load_host_xsave_state.part.0+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803862: branches: ffffffffc0f662cf vmx_vcpu_run+0x39f ([kernel.kallsyms]) => ffffffffc0f63f90 vmx_recover_nmi_blocking+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803862: branches: ffffffffc0f662e9 vmx_vcpu_run+0x3b9 ([kernel.kallsyms]) => ffffffffc0f619a0 __vmx_complete_interrupts+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803872: branches: ffffffffc109cfb2 vcpu_enter_guest+0x752 ([kernel.kallsyms]) => ffffffffc0f5f570 vmx_handle_exit_irqoff+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803881: branches: ffffffffc109d028 vcpu_enter_guest+0x7c8 ([kernel.kallsyms]) => ffffffffb234f900 __srcu_read_lock+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803897: branches: ffffffffc109d06f vcpu_enter_guest+0x80f ([kernel.kallsyms]) => ffffffffc0f72e30 vmx_handle_exit+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803897: branches: ffffffffc0f72e3d vmx_handle_exit+0xd ([kernel.kallsyms]) => ffffffffc0f727c0 __vmx_handle_exit+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803897: branches: ffffffffc0f72b15 __vmx_handle_exit+0x355 ([kernel.kallsyms]) => ffffffffc0f60ae0 vmx_flush_pml_buffer+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803903: branches: ffffffffc0f72994 __vmx_handle_exit+0x1d4 ([kernel.kallsyms]) => ffffffffc10b7090 kvm_emulate_cpuid+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803903: branches: ffffffffc10b70f1 kvm_emulate_cpuid+0x61 ([kernel.kallsyms]) => ffffffffc10b6e10 kvm_cpuid+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803941: branches: ffffffffc10b7125 kvm_emulate_cpuid+0x95 ([kernel.kallsyms]) => ffffffffc1093110 kvm_skip_emulated_instruction+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803941: branches: ffffffffc109311f kvm_skip_emulated_instruction+0xf ([kernel.kallsyms]) => ffffffffc0f5e180 vmx_get_rflags+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803951: branches: ffffffffc109312a kvm_skip_emulated_instruction+0x1a ([kernel.kallsyms]) => ffffffffc0f5fd30 vmx_skip_emulated_instruction+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803951: branches: ffffffffc0f5fd79 vmx_skip_emulated_instruction+0x49 ([kernel.kallsyms]) => ffffffffc0f5fb50 skip_emulated_instruction+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803956: branches: ffffffffc0f5fc68 skip_emulated_instruction+0x118 ([kernel.kallsyms]) => ffffffffc0f6a940 vmx_cache_reg+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803964: branches: ffffffffc0f5fc11 skip_emulated_instruction+0xc1 ([kernel.kallsyms]) => ffffffffc0f5f9e0 vmx_set_interrupt_shadow+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803980: branches: ffffffffc109f8b1 vcpu_run+0x71 ([kernel.kallsyms]) => ffffffffc10ad2f0 kvm_cpu_has_pending_timer+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803980: branches: ffffffffc10ad2fb kvm_cpu_has_pending_timer+0xb ([kernel.kallsyms]) => ffffffffc10b0490 apic_has_pending_timer+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803991: branches: ffffffffc109f899 vcpu_run+0x59 ([kernel.kallsyms]) => ffffffffc109c860 vcpu_enter_guest+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803993: branches: ffffffffc109cd4c vcpu_enter_guest+0x4ec ([kernel.kallsyms]) => ffffffffc0f69140 vmx_prepare_switch_to_guest+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803996: branches: ffffffffc109cd7d vcpu_enter_guest+0x51d ([kernel.kallsyms]) => ffffffffb234f930 __srcu_read_unlock+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803996: branches: ffffffffc109cd9c vcpu_enter_guest+0x53c ([kernel.kallsyms]) => ffffffffc0f609b0 vmx_sync_pir_to_irr+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408803996: branches: ffffffffc0f60a6d vmx_sync_pir_to_irr+0xbd ([kernel.kallsyms]) => ffffffffc10adc20 kvm_lapic_find_highest_irr+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804010: branches: ffffffffc0f60abd vmx_sync_pir_to_irr+0x10d ([kernel.kallsyms]) => ffffffffc0f60820 vmx_set_rvi+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804019: branches: ffffffffc109ceca vcpu_enter_guest+0x66a ([kernel.kallsyms]) => ffffffffb2249840 fpregs_assert_state_consistent+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804021: branches: ffffffffc109cf10 vcpu_enter_guest+0x6b0 ([kernel.kallsyms]) => ffffffffc0f65f30 vmx_vcpu_run+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804024: branches: ffffffffc0f6603b vmx_vcpu_run+0x10b ([kernel.kallsyms]) => ffffffffb229bed0 __get_current_cr3_fast+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804024: branches: ffffffffc0f66055 vmx_vcpu_run+0x125 ([kernel.kallsyms]) => ffffffffb2253050 cr4_read_shadow+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804030: branches: ffffffffc0f6608d vmx_vcpu_run+0x15d ([kernel.kallsyms]) => ffffffffc10921e0 kvm_load_guest_xsave_state+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804030: branches: ffffffffc1092207 kvm_load_guest_xsave_state+0x27 ([kernel.kallsyms]) => ffffffffc1092110 kvm_load_guest_xsave_state.part.0+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804032: branches: ffffffffc0f660c6 vmx_vcpu_run+0x196 ([kernel.kallsyms]) => ffffffffb22061a0 perf_guest_get_msrs+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804032: branches: ffffffffb22061a9 perf_guest_get_msrs+0x9 ([kernel.kallsyms]) => ffffffffb220cda0 intel_guest_get_msrs+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804039: branches: ffffffffc0f66109 vmx_vcpu_run+0x1d9 ([kernel.kallsyms]) => ffffffffc0f652c0 clear_atomic_switch_msr+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804040: branches: ffffffffc0f66119 vmx_vcpu_run+0x1e9 ([kernel.kallsyms]) => ffffffffc0f73f60 intel_pmu_lbr_is_enabled+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804042: branches: ffffffffc0f73f81 intel_pmu_lbr_is_enabled+0x21 ([kernel.kallsyms]) => ffffffffc10b68e0 kvm_find_cpuid_entry+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804045: branches: ffffffffc0f66454 vmx_vcpu_run+0x524 ([kernel.kallsyms]) => ffffffffc0f61ff0 vmx_update_hv_timer+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804057: branches: ffffffffc0f66142 vmx_vcpu_run+0x212 ([kernel.kallsyms]) => ffffffffc10af100 kvm_wait_lapic_expire+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804057: branches: ffffffffc0f66156 vmx_vcpu_run+0x226 ([kernel.kallsyms]) => ffffffffb2255c60 x86_virt_spec_ctrl+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804057: branches: ffffffffc0f66161 vmx_vcpu_run+0x231 ([kernel.kallsyms]) => ffffffffc0f8eb20 vmx_vcpu_enter_exit+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804057: branches: ffffffffc0f8eb44 vmx_vcpu_enter_exit+0x24 ([kernel.kallsyms]) => ffffffffb2353e10 rcu_note_context_switch+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804057: branches: ffffffffb2353e1c rcu_note_context_switch+0xc ([kernel.kallsyms]) => ffffffffb2353db0 rcu_qs+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804066: branches: ffffffffc0f8ebe0 vmx_vcpu_enter_exit+0xc0 ([kernel.kallsyms]) => ffffffffc0f8edc0 __vmx_vcpu_run+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804066: branches: ffffffffc0f8edd5 __vmx_vcpu_run+0x15 ([kernel.kallsyms]) => ffffffffc0f8eca0 vmx_update_host_rsp+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804066: branches: ffffffffc0f8ee1b __vmx_vcpu_run+0x5b ([kernel.kallsyms]) => ffffffffc0f8ed60 vmx_vmenter+0x0 ([kernel.kallsyms]) + CPU 3/KVM 13376/13384 [002] 7919.408804162: branches: ffffffffc0f8ed62 vmx_vmenter+0x2 ([kernel.kallsyms]) => 0 [unknown] ([unknown]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804162: branches: 0 [unknown] ([unknown]) => 7f851c9b5a5c init_cacheinfo+0x3ac (/usr/lib/x86_64-linux-gnu/libc-2.31.so) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804273: branches: 7f851cb7c0e4 _dl_init+0x74 (/usr/lib/x86_64-linux-gnu/ld-2.31.so) => 7f851cb7bf50 call_init.part.0+0x0 (/usr/lib/x86_64-linux-gnu/ld-2.31.so) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804526: branches: 55e0c00136f0 _start+0x0 (/usr/bin/uname) => ffffffff83200ac0 asm_exc_page_fault+0x0 ([kernel.kallsyms]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804526: branches: ffffffff83200ac3 asm_exc_page_fault+0x3 ([kernel.kallsyms]) => ffffffff83201290 error_entry+0x0 ([kernel.kallsyms]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804534: branches: ffffffff832012fa error_entry+0x6a ([kernel.kallsyms]) => ffffffff830b59a0 sync_regs+0x0 ([kernel.kallsyms]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804631: branches: ffffffff83200ad9 asm_exc_page_fault+0x19 ([kernel.kallsyms]) => ffffffff830b8210 exc_page_fault+0x0 ([kernel.kallsyms]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804631: branches: ffffffff830b82a4 exc_page_fault+0x94 ([kernel.kallsyms]) => ffffffff830b80e0 __kvm_handle_async_pf+0x0 ([kernel.kallsyms]) + VM:13376 VCPU:003 uname 3404/3404 [002] 7919.408804631: branches: ffffffff830b80ed __kvm_handle_async_pf+0xd ([kernel.kallsyms]) => ffffffff830b80c0 kvm_read_and_reset_apf_flags+0x0 ([kernel.kallsyms]) + + +Tracing Virtual Machines - Guest Code +------------------------------------- + +A common case for KVM test programs is that the test program acts as the +hypervisor, creating, running and destroying the virtual machine, and +providing the guest object code from its own object code. In this case, +the VM is not running an OS, but only the functions loaded into it by the +hypervisor test program, and conveniently, loaded at the same virtual +addresses. To support that, option "--guest-code" has been added to perf script +and perf kvm report. + +Here is an example tracing a test program from the kernel's KVM selftests: + + # perf record --kcore -e intel_pt/cyc/ -- tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test + [ perf record: Woken up 1 times to write data ] + [ perf record: Captured and wrote 0.280 MB perf.data ] + # perf script --guest-code --itrace=bep --ns -F-period,+addr,+flags + [SNIP] + tsc_msrs_test 18436 [007] 10897.962087733: branches: call ffffffffc13b2ff5 __vmx_vcpu_run+0x15 (vmlinux) => ffffffffc13b2f50 vmx_update_host_rsp+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962087733: branches: return ffffffffc13b2f5d vmx_update_host_rsp+0xd (vmlinux) => ffffffffc13b2ffa __vmx_vcpu_run+0x1a (vmlinux) + tsc_msrs_test 18436 [007] 10897.962087733: branches: call ffffffffc13b303b __vmx_vcpu_run+0x5b (vmlinux) => ffffffffc13b2f80 vmx_vmenter+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962087836: branches: vmentry ffffffffc13b2f82 vmx_vmenter+0x2 (vmlinux) => 0 [unknown] ([unknown]) + [guest/18436] 18436 [007] 10897.962087836: branches: vmentry 0 [unknown] ([unknown]) => 402c81 guest_code+0x131 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962087836: branches: call 402c81 guest_code+0x131 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 40dba0 ucall+0x0 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962088248: branches: vmexit 40dba0 ucall+0x0 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 0 [unknown] ([unknown]) + tsc_msrs_test 18436 [007] 10897.962088248: branches: vmexit 0 [unknown] ([unknown]) => ffffffffc13b2fa0 vmx_vmexit+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962088248: branches: jmp ffffffffc13b2fa0 vmx_vmexit+0x0 (vmlinux) => ffffffffc13b2fd2 vmx_vmexit+0x32 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962088256: branches: return ffffffffc13b2fd2 vmx_vmexit+0x32 (vmlinux) => ffffffffc13b3040 __vmx_vcpu_run+0x60 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962088270: branches: return ffffffffc13b30b6 __vmx_vcpu_run+0xd6 (vmlinux) => ffffffffc13b2f2e vmx_vcpu_enter_exit+0x4e (vmlinux) + [SNIP] + tsc_msrs_test 18436 [007] 10897.962089321: branches: call ffffffffc13b2ff5 __vmx_vcpu_run+0x15 (vmlinux) => ffffffffc13b2f50 vmx_update_host_rsp+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089321: branches: return ffffffffc13b2f5d vmx_update_host_rsp+0xd (vmlinux) => ffffffffc13b2ffa __vmx_vcpu_run+0x1a (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089321: branches: call ffffffffc13b303b __vmx_vcpu_run+0x5b (vmlinux) => ffffffffc13b2f80 vmx_vmenter+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089424: branches: vmentry ffffffffc13b2f82 vmx_vmenter+0x2 (vmlinux) => 0 [unknown] ([unknown]) + [guest/18436] 18436 [007] 10897.962089424: branches: vmentry 0 [unknown] ([unknown]) => 40dba0 ucall+0x0 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962089701: branches: jmp 40dc1b ucall+0x7b (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 40dc39 ucall+0x99 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962089701: branches: jcc 40dc3c ucall+0x9c (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 40dc20 ucall+0x80 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962089701: branches: jcc 40dc3c ucall+0x9c (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 40dc20 ucall+0x80 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962089701: branches: jcc 40dc37 ucall+0x97 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 40dc50 ucall+0xb0 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) + [guest/18436] 18436 [007] 10897.962089878: branches: vmexit 40dc55 ucall+0xb5 (/home/user/git/work/tools/testing/selftests/kselftest_install/kvm/tsc_msrs_test) => 0 [unknown] ([unknown]) + tsc_msrs_test 18436 [007] 10897.962089878: branches: vmexit 0 [unknown] ([unknown]) => ffffffffc13b2fa0 vmx_vmexit+0x0 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089878: branches: jmp ffffffffc13b2fa0 vmx_vmexit+0x0 (vmlinux) => ffffffffc13b2fd2 vmx_vmexit+0x32 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089887: branches: return ffffffffc13b2fd2 vmx_vmexit+0x32 (vmlinux) => ffffffffc13b3040 __vmx_vcpu_run+0x60 (vmlinux) + tsc_msrs_test 18436 [007] 10897.962089901: branches: return ffffffffc13b30b6 __vmx_vcpu_run+0xd6 (vmlinux) => ffffffffc13b2f2e vmx_vcpu_enter_exit+0x4e (vmlinux) + [SNIP] + + # perf kvm --guest-code --guest --host report -i perf.data --stdio | head -20 + + # To display the perf.data header info, please use --header/--header-only options. + # + # + # Total Lost Samples: 0 + # + # Samples: 12 of event 'instructions' + # Event count (approx.): 2274583 + # + # Children Self Command Shared Object Symbol + # ........ ........ ............. .................... ........................................... + # + 54.70% 0.00% tsc_msrs_test [kernel.vmlinux] [k] entry_SYSCALL_64_after_hwframe + | + ---entry_SYSCALL_64_after_hwframe + do_syscall_64 + | + |--29.44%--syscall_exit_to_user_mode + | exit_to_user_mode_prepare + | task_work_run + | __fput + + +Event Trace +----------- + +Event Trace records information about asynchronous events, for example interrupts, +faults, VM exits and entries. The information is recorded in CFE and EVD packets, +and also the Interrupt Flag is recorded on the MODE.Exec packet. The CFE packet +contains a type field to identify one of the following: + + 1 INTR interrupt, fault, exception, NMI + 2 IRET interrupt return + 3 SMI system management interrupt + 4 RSM resume from system management mode + 5 SIPI startup interprocessor interrupt + 6 INIT INIT signal + 7 VMENTRY VM-Entry + 8 VMEXIT VM-Entry + 9 VMEXIT_INTR VM-Exit due to interrupt + 10 SHUTDOWN Shutdown + +For more details, refer to the Intel 64 and IA-32 Architectures Software +Developer Manuals (version 076 or later). + +The capability to do Event Trace is indicated by the +/sys/bus/event_source/devices/intel_pt/caps/event_trace file. + +Event trace is selected for recording using the "event" config term. e.g. + + perf record -e intel_pt/event/u uname + +Event trace events are output using the --itrace I option. e.g. + + perf script --itrace=Ie + +perf script displays events containing CFE type, vector and event data, +in the form: + + evt: hw int (t) cfe: INTR IP: 1 vector: 3 PFA: 0x8877665544332211 + +The IP flag indicates if the event binds to an IP, which includes any case where +flow control packet generation is enabled, as well as when CFE packet IP bit is +set. + +perf script displays events containing changes to the Interrupt Flag in the form: + + iflag: t IFLAG: 1->0 via branch + +where "via branch" indicates a branch (interrupt or return from interrupt) and +"non branch" indicates an instruction such as CFI, STI or POPF). + +In addition, the current state of the interrupt flag is indicated by the presence +or absence of the "D" (interrupt disabled) perf script flag. If the interrupt +flag is changed, then the "t" flag is also included i.e. + + no flag, interrupts enabled IF=1 + t interrupts become disabled IF=1 -> IF=0 + D interrupts are disabled IF=0 + Dt interrupts become enabled IF=0 -> IF=1 + +The intel-pt-events.py script illustrates how to access Event Trace information +using a Python script. + + +TNT Disable +----------- + +TNT packets are disabled using the "notnt" config term. e.g. + + perf record -e intel_pt/notnt/u uname + +In that case the --itrace q option is forced because walking executable code +to reconstruct the control flow is not possible. + + +Emulated PTWRITE +---------------- + +Later perf tools support a method to emulate the ptwrite instruction, which +can be useful if hardware does not support the ptwrite instruction. + +Instead of using the ptwrite instruction, a function is used which produces +a trace that encodes the payload data into TNT packets. Here is an example +of the function: + + #include <stdint.h> + + void perf_emulate_ptwrite(uint64_t x) + __attribute__((externally_visible, noipa, no_instrument_function, naked)); + + #define PERF_EMULATE_PTWRITE_8_BITS \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" \ + "1: shl %rax\n" \ + " jc 1f\n" + + /* Undefined instruction */ + #define PERF_EMULATE_PTWRITE_UD2 ".byte 0x0f, 0x0b\n" + + #define PERF_EMULATE_PTWRITE_MAGIC PERF_EMULATE_PTWRITE_UD2 ".ascii \"perf,ptwrite \"\n" + + void perf_emulate_ptwrite(uint64_t x __attribute__ ((__unused__))) + { + /* Assumes SysV ABI : x passed in rdi */ + __asm__ volatile ( + "jmp 1f\n" + PERF_EMULATE_PTWRITE_MAGIC + "1: mov %rdi, %rax\n" + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + PERF_EMULATE_PTWRITE_8_BITS + "1: ret\n" + ); + } + +For example, a test program with the function above: + + #include <stdio.h> + #include <stdint.h> + #include <stdlib.h> + + #include "perf_emulate_ptwrite.h" + + int main(int argc, char *argv[]) + { + uint64_t x = 0; + + if (argc > 1) + x = strtoull(argv[1], NULL, 0); + perf_emulate_ptwrite(x); + return 0; + } + +Can be compiled and traced: + + $ gcc -Wall -Wextra -O3 -g -o eg_ptw eg_ptw.c + $ perf record -e intel_pt//u ./eg_ptw 0x1234567890abcdef + [ perf record: Woken up 1 times to write data ] + [ perf record: Captured and wrote 0.017 MB perf.data ] + $ perf script --itrace=ew + eg_ptw 19875 [007] 8061.235912: ptwrite: IP: 0 payload: 0x1234567890abcdef 55701249a196 perf_emulate_ptwrite+0x16 (/home/user/eg_ptw) + $ + + +EXAMPLE +------- + +Examples can be found on perf wiki page "Perf tools support for Intel® Processor Trace": + +https://perf.wiki.kernel.org/index.php/Perf_tools_support_for_Intel%C2%AE_Processor_Trace + SEE ALSO -------- diff --git a/tools/perf/Documentation/perf-kvm.txt b/tools/perf/Documentation/perf-kvm.txt index cf95baef7b61..2ad3f5d9f72b 100644 --- a/tools/perf/Documentation/perf-kvm.txt +++ b/tools/perf/Documentation/perf-kvm.txt @@ -77,23 +77,11 @@ OPTIONS Collect host side performance profile. --guest:: Collect guest side performance profile. ---guestmount=<path>:: - Guest os root file system mount directory. Users mounts guest os - root directories under <path> by a specific filesystem access method, - typically, sshfs. For example, start 2 guest os. The one's pid is 8888 - and the other's is 9999. - #mkdir ~/guestmount; cd ~/guestmount - #sshfs -o allow_other,direct_io -p 5551 localhost:/ 8888/ - #sshfs -o allow_other,direct_io -p 5552 localhost:/ 9999/ - #perf kvm --host --guest --guestmount=~/guestmount top ---guestkallsyms=<path>:: - Guest os /proc/kallsyms file copy. 'perf' kvm' reads it to get guest - kernel symbols. Users copy it out from guest os. ---guestmodules=<path>:: - Guest os /proc/modules file copy. 'perf' kvm' reads it to get guest - kernel module information. Users copy it out from guest os. ---guestvmlinux=<path>:: - Guest os kernel vmlinux. + +:GMEXAMPLECMD: kvm --host --guest +:GMEXAMPLESUBCMD: top +include::guest-files.txt[] + -v:: --verbose:: Be more verbose (show counter open errors, etc). diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt new file mode 100644 index 000000000000..3c36324712b6 --- /dev/null +++ b/tools/perf/Documentation/perf-kwork.txt @@ -0,0 +1,180 @@ +perf-kowrk(1) +============= + +NAME +---- +perf-kwork - Tool to trace/measure kernel work properties (latencies) + +SYNOPSIS +-------- +[verse] +'perf kwork' {record} + +DESCRIPTION +----------- +There are several variants of 'perf kwork': + + 'perf kwork record <command>' to record the kernel work + of an arbitrary workload. + + 'perf kwork report' to report the per kwork runtime. + + 'perf kwork latency' to report the per kwork latencies. + + 'perf kwork timehist' provides an analysis of kernel work events. + + Example usage: + perf kwork record -- sleep 1 + perf kwork report + perf kwork report -b + perf kwork latency + perf kwork latency -b + perf kwork timehist + + By default it shows the individual work events such as irq, workqeueu, + including the run time and delay (time between raise and actually entry): + + Runtime start Runtime end Cpu Kwork name Runtime Delaytime + (TYPE)NAME:NUM (msec) (msec) + ----------------- ----------------- ------ ------------------------- ---------- ---------- + 1811186.976062 1811186.976327 [0000] (s)RCU:9 0.266 0.114 + 1811186.978452 1811186.978547 [0000] (s)SCHED:7 0.095 0.171 + 1811186.980327 1811186.980490 [0000] (s)SCHED:7 0.162 0.083 + 1811186.981221 1811186.981271 [0000] (s)SCHED:7 0.050 0.077 + 1811186.984267 1811186.984318 [0000] (s)SCHED:7 0.051 0.075 + 1811186.987252 1811186.987315 [0000] (s)SCHED:7 0.063 0.081 + 1811186.987785 1811186.987843 [0006] (s)RCU:9 0.058 0.645 + 1811186.988319 1811186.988383 [0000] (s)SCHED:7 0.064 0.143 + 1811186.989404 1811186.989607 [0002] (s)TIMER:1 0.203 0.111 + 1811186.989660 1811186.989732 [0002] (s)SCHED:7 0.072 0.310 + 1811186.991295 1811186.991407 [0002] eth0:10 0.112 + 1811186.991639 1811186.991734 [0002] (s)NET_RX:3 0.095 0.277 + 1811186.989860 1811186.991826 [0002] (w)vmstat_shepherd 1.966 0.345 + ... + + Times are in msec.usec. + +OPTIONS +------- +-D:: +--dump-raw-trace=:: + Display verbose dump of the sched data. + +-f:: +--force:: + Don't complain, do it. + +-k:: +--kwork:: + List of kwork to profile (irq, softirq, workqueue, etc) + +-v:: +--verbose:: + Be more verbose. (show symbol address, etc) + +OPTIONS for 'perf kwork report' +---------------------------- + +-b:: +--use-bpf:: + Use BPF to measure kwork runtime + +-C:: +--cpu:: + Only show events for the given CPU(s) (comma separated list). + +-i:: +--input:: + Input file name. (default: perf.data unless stdin is a fifo) + +-n:: +--name:: + Only show events for the given name. + +-s:: +--sort:: + Sort by key(s): runtime, max, count + +-S:: +--with-summary:: + Show summary with statistics + +--time:: + Only analyze samples within given time window: <start>,<stop>. Times + have the format seconds.microseconds. If start is not given (i.e., time + string is ',x.y') then analysis starts at the beginning of the file. If + stop time is not given (i.e, time string is 'x.y,') then analysis goes + to end of file. + +OPTIONS for 'perf kwork latency' +---------------------------- + +-b:: +--use-bpf:: + Use BPF to measure kwork latency + +-C:: +--cpu:: + Only show events for the given CPU(s) (comma separated list). + +-i:: +--input:: + Input file name. (default: perf.data unless stdin is a fifo) + +-n:: +--name:: + Only show events for the given name. + +-s:: +--sort:: + Sort by key(s): avg, max, count + +--time:: + Only analyze samples within given time window: <start>,<stop>. Times + have the format seconds.microseconds. If start is not given (i.e., time + string is ',x.y') then analysis starts at the beginning of the file. If + stop time is not given (i.e, time string is 'x.y,') then analysis goes + to end of file. + +OPTIONS for 'perf kwork timehist' +--------------------------------- + +-C:: +--cpu:: + Only show events for the given CPU(s) (comma separated list). + +-g:: +--call-graph:: + Display call chains if present (default off). + +-i:: +--input:: + Input file name. (default: perf.data unless stdin is a fifo) + +-k:: +--vmlinux=<file>:: + Vmlinux pathname + +-n:: +--name:: + Only show events for the given name. + +--kallsyms=<file>:: + Kallsyms pathname + +--max-stack:: + Maximum number of functions to display in backtrace, default 5. + +--symfs=<directory>:: + Look for files with symbols relative to this directory. + +--time:: + Only analyze samples within given time window: <start>,<stop>. Times + have the format seconds.microseconds. If start is not given (i.e., time + string is ',x.y') then analysis starts at the beginning of the file. If + stop time is not given (i.e, time string is 'x.y,') then analysis goes + to end of file. + +SEE ALSO +-------- +linkperf:perf-record[1] diff --git a/tools/perf/Documentation/perf-list.txt b/tools/perf/Documentation/perf-list.txt index 4dc8d0af19df..57384a97c04f 100644 --- a/tools/perf/Documentation/perf-list.txt +++ b/tools/perf/Documentation/perf-list.txt @@ -81,7 +81,11 @@ On AMD systems it is implemented using IBS (up to precise-level 2). The precise modifier works with event types 0x76 (cpu-cycles, CPU clocks not halted) and 0xC1 (micro-ops retired). Both events map to IBS execution sampling (IBS op) with the IBS Op Counter Control bit -(IbsOpCntCtl) set respectively (see AMD64 Architecture Programmer’s +(IbsOpCntCtl) set respectively (see the +Core Complex (CCX) -> Processor x86 Core -> Instruction Based Sampling (IBS) +section of the [AMD Processor Programming Reference (PPR)] relevant to the +family, model and stepping of the processor being used). + Manual Volume 2: System Programming, 13.3 Instruction-Based Sampling). Examples to use IBS: @@ -94,10 +98,12 @@ RAW HARDWARE EVENT DESCRIPTOR Even when an event is not available in a symbolic form within perf right now, it can be encoded in a per processor specific way. -For instance For x86 CPUs NNN represents the raw register encoding with the +For instance on x86 CPUs, N is a hexadecimal value that represents the raw register encoding with the layout of IA32_PERFEVTSELx MSRs (see [Intel® 64 and IA-32 Architectures Software Developer's Manual Volume 3B: System Programming Guide] Figure 30-1 Layout -of IA32_PERFEVTSELx MSRs) or AMD's PerfEvtSeln (see [AMD64 Architecture Programmer’s Manual Volume 2: System Programming], Page 344, -Figure 13-7 Performance Event-Select Register (PerfEvtSeln)). +of IA32_PERFEVTSELx MSRs) or AMD's PERF_CTL MSRs (see the +Core Complex (CCX) -> Processor x86 Core -> MSR Registers section of the +[AMD Processor Programming Reference (PPR)] relevant to the family, model +and stepping of the processor being used). Note: Only the following bit fields can be set in x86 counter registers: event, umask, edge, inv, cmask. Esp. guest/host only and @@ -126,6 +132,38 @@ It's also possible to use pmu syntax: perf record -e cpu/r1a8/ ... perf record -e cpu/r0x1a8/ ... +Some processors, like those from AMD, support event codes and unit masks +larger than a byte. In such cases, the bits corresponding to the event +configuration parameters can be seen with: + + cat /sys/bus/event_source/devices/<pmu>/format/<config> + +Example: + +If the AMD docs for an EPYC 7713 processor describe an event as: + + Event Umask Event Mask + Num. Value Mnemonic Description + + 28FH 03H op_cache_hit_miss.op_cache_hit Counts Op Cache micro-tag + hit events. + +raw encoding of 0x0328F cannot be used since the upper nibble of the +EventSelect bits have to be specified via bits 32-35 as can be seen with: + + cat /sys/bus/event_source/devices/cpu/format/event + +raw encoding of 0x20000038F should be used instead: + + perf stat -e r20000038f -a sleep 1 + perf record -e r20000038f ... + +It's also possible to use pmu syntax: + + perf record -e r20000038f -a sleep 1 + perf record -e cpu/r20000038f/ ... + perf record -e cpu/r0x20000038f/ ... + You should refer to the processor specific documentation for getting these details. Some of them are referenced in the SEE ALSO section below. @@ -316,4 +354,4 @@ SEE ALSO linkperf:perf-stat[1], linkperf:perf-top[1], linkperf:perf-record[1], http://www.intel.com/sdm/[Intel® 64 and IA-32 Architectures Software Developer's Manual Volume 3B: System Programming Guide], -http://support.amd.com/us/Processor_TechDocs/24593_APM_v2.pdf[AMD64 Architecture Programmer’s Manual Volume 2: System Programming] +https://bugzilla.kernel.org/show_bug.cgi?id=206537[AMD Processor Programming Reference (PPR)] diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index 1b4d452923d7..3b1e16563b79 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -8,7 +8,7 @@ perf-lock - Analyze lock events SYNOPSIS -------- [verse] -'perf lock' {record|report|script|info} +'perf lock' {record|report|script|info|contention} DESCRIPTION ----------- @@ -27,6 +27,8 @@ and statistics with this 'perf lock' command. 'perf lock info' shows metadata like threads or addresses of lock instances. + 'perf lock contention' shows contention statistics. + COMMON OPTIONS -------------- @@ -38,6 +40,10 @@ COMMON OPTIONS --verbose:: Be more verbose (show symbol address, etc). +-q:: +--quiet:: + Do not show any message. (Suppress -v) + -D:: --dump-raw-trace:: Dump raw trace in ASCII. @@ -46,6 +52,13 @@ COMMON OPTIONS --force:: Don't complain, do it. +--vmlinux=<file>:: + vmlinux pathname + +--kallsyms=<file>:: + kallsyms pathname + + REPORT OPTIONS -------------- @@ -54,6 +67,42 @@ REPORT OPTIONS Sorting key. Possible values: acquired (default), contended, avg_wait, wait_total, wait_max, wait_min. +-F:: +--field=<value>:: + Output fields. By default it shows all the fields but users can + customize that using this. Possible values: acquired, contended, + avg_wait, wait_total, wait_max, wait_min. + +-c:: +--combine-locks:: + Merge lock instances in the same class (based on name). + +-t:: +--threads:: + The -t option is to show per-thread lock stat like below: + + $ perf lock report -t -F acquired,contended,avg_wait + + Name acquired contended avg wait (ns) + + perf 240569 9 5784 + swapper 106610 19 543 + :15789 17370 2 14538 + ContainerMgr 8981 6 874 + sleep 5275 1 11281 + ContainerThread 4416 4 944 + RootPressureThr 3215 5 1215 + rcu_preempt 2954 0 0 + ContainerMgr 2560 0 0 + unnamed 1873 0 0 + EventManager_De 1845 1 636 + futex-default-S 1609 0 0 + +-E:: +--entries=<value>:: + Display this many entries. + + INFO OPTIONS ------------ @@ -65,6 +114,61 @@ INFO OPTIONS --map:: dump map of lock instances (address:name table) + +CONTENTION OPTIONS +-------------- + +-k:: +--key=<value>:: + Sorting key. Possible values: contended, wait_total (default), + wait_max, wait_min, avg_wait. + +-F:: +--field=<value>:: + Output fields. By default it shows all but the wait_min fields + and users can customize that using this. Possible values: + contended, wait_total, wait_max, wait_min, avg_wait. + +-t:: +--threads:: + Show per-thread lock contention stat + +-b:: +--use-bpf:: + Use BPF program to collect lock contention stats instead of + using the input data. + +-a:: +--all-cpus:: + System-wide collection from all CPUs. + +-C:: +--cpu:: + Collect samples only on the list of CPUs provided. Multiple CPUs can be + provided as a comma-separated list with no space: 0,1. Ranges of CPUs + are specified with -: 0-2. Default is to monitor all CPUs. + +-p:: +--pid=:: + Record events on existing process ID (comma separated list). + +--tid=:: + Record events on existing thread ID (comma separated list). + +--map-nr-entries:: + Maximum number of BPF map entries (default: 10240). + +--max-stack:: + Maximum stack depth when collecting lock contention (default: 8). + +--stack-skip + Number of stack depth to skip when finding a lock caller (default: 3). + +-E:: +--entries=<value>:: + Display this many entries. + + SEE ALSO -------- linkperf:perf[1] diff --git a/tools/perf/Documentation/perf-mem.txt b/tools/perf/Documentation/perf-mem.txt index 66177511c5c4..005c95580b1e 100644 --- a/tools/perf/Documentation/perf-mem.txt +++ b/tools/perf/Documentation/perf-mem.txt @@ -85,7 +85,8 @@ RECORD OPTIONS Be more verbose (show counter open errors, etc) --ldlat <n>:: - Specify desired latency for loads event. (x86 only) + Specify desired latency for loads event. Supported on Intel and Arm64 + processors only. Ignored on other archs. In addition, for report all perf report options are valid, and for record all perf record options. diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index 3cf7bac67239..e41ae950fdc3 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -30,8 +30,10 @@ OPTIONS - a symbolic event name (use 'perf list' to list all events) - - a raw PMU event (eventsel+umask) in the form of rNNN where NNN is a - hexadecimal event descriptor. + - a raw PMU event in the form of rN where N is a hexadecimal value + that represents the raw register encoding with the layout of the + event control registers as described by entries in + /sys/bus/event_source/devices/cpu/format/*. - a symbolic or raw PMU event followed by an optional colon and a list of event modifiers, e.g., cpu-cycles:p. See the @@ -273,6 +275,11 @@ OPTIONS User can change the size by passing the size after comma like "--call-graph dwarf,4096". + When "fp" recording is used, perf tries to save stack enties + up to the number specified in sysctl.kernel.perf_event_max_stack + by default. User can change the number by passing it after comma + like "--call-graph fp,32". + -q:: --quiet:: Don't print any message, useful for scripting. @@ -311,6 +318,11 @@ OPTIONS --sample-cpu:: Record the sample cpu. +--sample-identifier:: + Record the sample identifier i.e. PERF_SAMPLE_IDENTIFIER bit set in + the sample_type member of the struct perf_event_attr argument to the + perf_event_open system call. + -n:: --no-samples:: Don't sample. @@ -385,6 +397,10 @@ following filters are defined: - abort_tx: only when the target is a hardware transaction abort - cond: conditional branches - save_type: save branch type during sampling in case binary is not available later + For the platforms with Intel Arch LBR support (12th-Gen+ client or + 4th-Gen Xeon+ server), the save branch type is unconditionally enabled + when the taken branch stack sampling is enabled. + - priv: save privilege state during sampling in case binary is not available later + The option requires at least one branch type among any, any_call, any_ret, ind_call, cond. @@ -395,6 +411,7 @@ is enabled for all the sampling events. The sampled branch type is the same for The various filters must be specified as a comma separated list: --branch-filter any_ret,u,k Note that this feature may not be available on all processors. +-W:: --weight:: Enable weightened sampling. An additional weight is recorded per sample and can be displayed with the weight and local_weight sort keys. This currently works for TSX @@ -418,8 +435,10 @@ if combined with -a or -C options. -D:: --delay=:: After starting the program, wait msecs before measuring (-1: start with events -disabled). This is useful to filter out the startup phase of the program, which -is often very different. +disabled), or enable events only for specified ranges of msecs (e.g. +-D 10-20,30-40 means wait 10 msecs, enable for 10 msecs, wait 10 msecs, enable +for 10 msecs, then stop). Note, delaying enabling of events is useful to filter +out the startup phase of the program, which is often very different. -I:: --intr-regs:: @@ -711,6 +730,59 @@ measurements: wait -n ${perf_pid} exit $? +--threads=<spec>:: +Write collected trace data into several data files using parallel threads. +<spec> value can be user defined list of masks. Masks separated by colon +define CPUs to be monitored by a thread and affinity mask of that thread +is separated by slash: + + <cpus mask 1>/<affinity mask 1>:<cpus mask 2>/<affinity mask 2>:... + +CPUs or affinity masks must not overlap with other corresponding masks. +Invalid CPUs are ignored, but masks containing only invalid CPUs are not +allowed. + +For example user specification like the following: + + 0,2-4/2-4:1,5-7/5-7 + +specifies parallel threads layout that consists of two threads, +the first thread monitors CPUs 0 and 2-4 with the affinity mask 2-4, +the second monitors CPUs 1 and 5-7 with the affinity mask 5-7. + +<spec> value can also be a string meaning predefined parallel threads +layout: + + cpu - create new data streaming thread for every monitored cpu + core - create new thread to monitor CPUs grouped by a core + package - create new thread to monitor CPUs grouped by a package + numa - create new threed to monitor CPUs grouped by a NUMA domain + +Predefined layouts can be used on systems with large number of CPUs in +order not to spawn multiple per-cpu streaming threads but still avoid LOST +events in data directory files. Option specified with no or empty value +defaults to CPU layout. Masks defined or provided by the option value are +filtered through the mask provided by -C option. + +--debuginfod[=URLs]:: + Specify debuginfod URL to be used when cacheing perf.data binaries, + it follows the same syntax as the DEBUGINFOD_URLS variable, like: + + http://192.168.122.174:8002 + + If the URLs is not specified, the value of DEBUGINFOD_URLS + system environment variable is used. + +--off-cpu:: + Enable off-cpu profiling with BPF. The BPF program will collect + task scheduling information with (user) stacktrace and save them + as sample data of a software event named "offcpu-time". The + sample period will have the time the task slept in nanoseconds. + + Note that BPF can collect stack traces using frame pointer ("fp") + only, as of now. So the applications built without the frame + pointer might see bogus addresses. + include::intel-hybrid.txt[] SEE ALSO diff --git a/tools/perf/Documentation/perf-report.txt b/tools/perf/Documentation/perf-report.txt index 24efc0583c93..4533db2ee56b 100644 --- a/tools/perf/Documentation/perf-report.txt +++ b/tools/perf/Documentation/perf-report.txt @@ -73,7 +73,7 @@ OPTIONS Sort histogram entries by given key(s) - multiple keys can be specified in CSV format. Following sort keys are available: pid, comm, dso, symbol, parent, cpu, socket, srcline, weight, - local_weight, cgroup_id. + local_weight, cgroup_id, addr. Each key has following meaning: @@ -114,6 +114,7 @@ OPTIONS - local_ins_lat: Local instruction latency version - p_stage_cyc: On powerpc, this presents the number of cycles spent in a pipeline stage. And currently supported only on powerpc. + - addr: (Full) virtual address of the sampled instruction By default, comm, dso and symbol keys are used. (i.e. --sort comm,dso,symbol) diff --git a/tools/perf/Documentation/perf-script.txt b/tools/perf/Documentation/perf-script.txt index b0070718784d..68e37de5fae4 100644 --- a/tools/perf/Documentation/perf-script.txt +++ b/tools/perf/Documentation/perf-script.txt @@ -79,6 +79,9 @@ OPTIONS --dump-raw-trace=:: Display verbose dump of the trace data. +--dump-unsorted-raw-trace=:: + Same as --dump-raw-trace but not sorted in time order. + -L:: --Latency=:: Show latency attributes (irqs/preemption disabled, etc). @@ -129,8 +132,9 @@ OPTIONS Comma separated list of fields to print. Options are: comm, tid, pid, time, cpu, event, trace, ip, sym, dso, addr, symoff, srcline, period, iregs, uregs, brstack, brstacksym, flags, bpf-output, - brstackinsn, brstackoff, callindent, insn, insnlen, synth, phys_addr, - metric, misc, srccode, ipc, data_page_size, code_page_size, ins_lat. + brstackinsn, brstackinsnlen, brstackoff, callindent, insn, insnlen, synth, + phys_addr, metric, misc, srccode, ipc, data_page_size, code_page_size, ins_lat, + machine_pid, vcpu. Field list can be prepended with the type, trace, sw or hw, to indicate to which event type the field list applies. e.g., -F sw:comm,tid,time,ip,sym and -F trace:time,cpu,trace @@ -195,16 +199,19 @@ OPTIONS At this point usage is displayed, and perf-script exits. The flags field is synthesized and may have a value when Instruction - Trace decoding. The flags are "bcrosyiABExgh" which stand for branch, + Trace decoding. The flags are "bcrosyiABExghDt" which stand for branch, call, return, conditional, system, asynchronous, interrupt, - transaction abort, trace begin, trace end, in transaction, VM-Entry, and VM-Exit - respectively. Known combinations of flags are printed more nicely e.g. + transaction abort, trace begin, trace end, in transaction, VM-Entry, + VM-Exit, interrupt disabled and interrupt disable toggle respectively. + Known combinations of flags are printed more nicely e.g. "call" for "bc", "return" for "br", "jcc" for "bo", "jmp" for "b", "int" for "bci", "iret" for "bri", "syscall" for "bcs", "sysret" for "brs", "async" for "by", "hw int" for "bcyi", "tx abrt" for "bA", "tr strt" for "bB", "tr end" for "bE", "vmentry" for "bcg", "vmexit" for "bch". - However the "x" flag will be displayed separately in those - cases e.g. "jcc (x)" for a condition branch within a transaction. + However the "x", "D" and "t" flags will be displayed separately in those + cases e.g. "jcc (xD)" for a condition branch within a transaction + with interrupts disabled. Note, interrupts becoming disabled is "t", + whereas interrupts becoming enabled is "Dt". The callindent field is synthesized and may have a value when Instruction Trace decoding. For calls and returns, it will display the @@ -220,6 +227,10 @@ OPTIONS The ipc (instructions per cycle) field is synthesized and may have a value when Instruction Trace decoding. + The machine_pid and vcpu fields are derived from data resulting from using + perf inject to insert a perf.data file recorded inside a virtual machine into + a perf.data file recorded on the host at the same time. + Finally, a user may not set fields to none for all event types. i.e., -F "" is not allowed. @@ -238,6 +249,10 @@ OPTIONS is printed. This is the full execution path leading to the sample. This is only supported when the sample was recorded with perf record -b or -j any. + Use brstackinsnlen to print the brstackinsn lenght. For example, you + can’t know the next sequential instruction after an unconditional branch unless + you calculate that based on its length. + The brstackoff field will print an offset into a specific dso/binary. With the metric option perf script can compute metrics for @@ -492,6 +507,10 @@ include::itrace.txt[] The known limitations include exception handing such as setjmp/longjmp will have calls/returns not match. +:GMEXAMPLECMD: script +:GMEXAMPLESUBCMD: +include::guest-files.txt[] + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-script-perl[1], diff --git a/tools/perf/Documentation/perf-stat.txt b/tools/perf/Documentation/perf-stat.txt index 7e6fb7cbc0f4..d7ff1867feda 100644 --- a/tools/perf/Documentation/perf-stat.txt +++ b/tools/perf/Documentation/perf-stat.txt @@ -36,8 +36,10 @@ report:: - a symbolic event name (use 'perf list' to list all events) - - a raw PMU event (eventsel+umask) in the form of rNNN where NNN is a - hexadecimal event descriptor. + - a raw PMU event in the form of rN where N is a hexadecimal value + that represents the raw register encoding with the layout of the + event control registers as described by entries in + /sys/bus/event_source/devices/cpu/format/*. - a symbolic or raw PMU event followed by an optional colon and a list of event modifiers, e.g., cpu-cycles:p. See the @@ -452,6 +454,16 @@ Multiple events are created from a single event specification when: 2. Aliases, which are listed immediately after the Kernel PMU events by perf list, are used. +--hybrid-merge:: +Merge the hybrid event counts from all PMUs. + +For hybrid events, by default, the stat aggregates and reports the event +counts per PMU. But sometimes, it's also useful to aggregate event counts +from all PMUs. This option enables that behavior and reports the counts +without PMUs. + +For non-hybrid events, it should be no effect. + --smi-cost:: Measure SMI cost if msr/aperf/ and msr/smi/ events are supported. @@ -493,6 +505,10 @@ This option can be enabled in perf config by setting the variable $ perf config stat.no-csv-summary=true +--cputype:: +Only enable events on applying cpu with this type for hybrid platform +(e.g. core or atom)" + EXAMPLES -------- @@ -554,6 +570,27 @@ Additional metrics may be printed with all earlier fields being empty. include::intel-hybrid.txt[] +JSON FORMAT +----------- + +With -j, perf stat is able to print out a JSON format output +that can be used for parsing. + +- timestamp : optional usec time stamp in fractions of second (with -I) +- optional aggregate options: + - core : core identifier (with --per-core) + - die : die identifier (with --per-die) + - socket : socket identifier (with --per-socket) + - node : node identifier (with --per-node) + - thread : thread identifier (with --per-thread) +- counter-value : counter value +- unit : unit of the counter value or empty +- event : event name +- variance : optional variance if multiple values are collected (with -r) +- runtime : run time of counter +- metric-value : optional metric value +- metric-unit : optional unit of metric + SEE ALSO -------- linkperf:perf-top[1], linkperf:perf-list[1] diff --git a/tools/perf/Documentation/perf-top.txt b/tools/perf/Documentation/perf-top.txt index 9898a32b8d9c..c1fdba26bf53 100644 --- a/tools/perf/Documentation/perf-top.txt +++ b/tools/perf/Documentation/perf-top.txt @@ -38,9 +38,10 @@ Default is to monitor all CPUS. -e <event>:: --event=<event>:: Select the PMU event. Selection can be a symbolic event name - (use 'perf list' to list all events) or a raw PMU - event (eventsel+umask) in the form of rNNN where NNN is a - hexadecimal event descriptor. + (use 'perf list' to list all events) or a raw PMU event in the form + of rN where N is a hexadecimal value that represents the raw register + encoding with the layout of the event control registers as described + by entries in /sys/bus/event_source/devices/cpu/format/*. -E <entries>:: --entries=<entries>:: diff --git a/tools/perf/Documentation/perf.data-file-format.txt b/tools/perf/Documentation/perf.data-file-format.txt index f56d0e0fbff6..635ba043fd7d 100644 --- a/tools/perf/Documentation/perf.data-file-format.txt +++ b/tools/perf/Documentation/perf.data-file-format.txt @@ -419,18 +419,20 @@ Example: cpu_core cpu list : 0-15 cpu_atom cpu list : 16-23 - HEADER_HYBRID_CPU_PMU_CAPS = 31, + HEADER_PMU_CAPS = 31, - A list of hybrid CPU PMU capabilities. + List of pmu capabilities (except cpu pmu which is already + covered by HEADER_CPU_PMU_CAPS). Note that hybrid cpu pmu + capabilities are also stored here. struct { u32 nr_pmu; struct { - u32 nr_cpu_pmu_caps; + u32 nr_caps; { char name[]; char value[]; - } [nr_cpu_pmu_caps]; + } [nr_caps]; char pmu_name[]; } [nr_pmu]; }; @@ -607,6 +609,16 @@ struct compressed_event { char data[]; }; + PERF_RECORD_FINISHED_INIT = 82, + +Marks the end of records for the system, pre-existing threads in system wide +sessions, etc. Those are the ones prefixed PERF_RECORD_USER_*. + +This is used, for instance, to 'perf inject' events after init and before +regular events, those emitted by the kernel, to support combining guest and +host records. + + The header is followed by compressed data frame that can be decompressed into array of perf trace records. The size of the entire compressed event record including the header is limited by the max value of header.size. diff --git a/tools/perf/Documentation/perf.txt b/tools/perf/Documentation/perf.txt index 9c330cdfa973..ba3df49c169d 100644 --- a/tools/perf/Documentation/perf.txt +++ b/tools/perf/Documentation/perf.txt @@ -77,13 +77,13 @@ linkperf:perf-stat[1], linkperf:perf-top[1], linkperf:perf-record[1], linkperf:perf-report[1], linkperf:perf-list[1] -linkperf:perf-annotate[1],linkperf:perf-archive[1], +linkperf:perf-annotate[1],linkperf:perf-archive[1],linkperf:perf-arm-spe[1], linkperf:perf-bench[1], linkperf:perf-buildid-cache[1], linkperf:perf-buildid-list[1], linkperf:perf-c2c[1], linkperf:perf-config[1], linkperf:perf-data[1], linkperf:perf-diff[1], linkperf:perf-evlist[1], linkperf:perf-ftrace[1], linkperf:perf-help[1], linkperf:perf-inject[1], -linkperf:perf-intel-pt[1], linkperf:perf-kallsyms[1], +linkperf:perf-intel-pt[1], linkperf:perf-iostat[1], linkperf:perf-kallsyms[1], linkperf:perf-kmem[1], linkperf:perf-kvm[1], linkperf:perf-lock[1], linkperf:perf-mem[1], linkperf:perf-probe[1], linkperf:perf-sched[1], linkperf:perf-script[1], linkperf:perf-test[1], |