From 2987e32f75dcb40bce0f3ab1d5d48cc1c580fd8b Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 23 Oct 2014 13:45:15 +0300 Subject: perf script: Add Python script to export to postgresql Add a Python script to export to a postgresql database. The script requires the Python pyside module and the Qt PostgreSQL driver. The packages needed are probably named "python-pyside" and "libqt4-sql-psql" The caller of the script must be able to create postgresql databases. The script takes the database name as a parameter. The database and database tables are created. Data is written to flat files which are then imported using SQL COPY FROM. Example: $ perf record ls ... $ perf script report export-to-postgresql lsdb 2014-02-14 10:55:38.631431 Creating database... 2014-02-14 10:55:39.291958 Writing to intermediate files... 2014-02-14 10:55:39.350280 Copying to database... 2014-02-14 10:55:39.358536 Removing intermediate files... 2014-02-14 10:55:39.358665 Adding primary keys 2014-02-14 10:55:39.658697 Adding foreign keys 2014-02-14 10:55:39.667412 Done $ psql lsdb lsdb-# \d List of relations Schema | Name | Type | Owner --------+-----------------+-------+------- public | comm_threads | table | acme public | comms | table | acme public | dsos | table | acme public | machines | table | acme public | samples | table | acme public | samples_view | view | acme public | selected_events | table | acme public | symbols | table | acme public | threads | table | acme (9 rows) lsdb-# \d samples Table "public.samples" Column | Type | Modifiers ---------------+---------+----------- id | bigint | not null evsel_id | bigint | machine_id | bigint | thread_id | bigint | comm_id | bigint | dso_id | bigint | symbol_id | bigint | sym_offset | bigint | ip | bigint | time | bigint | cpu | integer | to_dso_id | bigint | to_symbol_id | bigint | to_sym_offset | bigint | to_ip | bigint | period | bigint | weight | bigint | transaction | bigint | data_src | bigint | Indexes: "samples_pkey" PRIMARY KEY, btree (id) Foreign-key constraints: "commfk" FOREIGN KEY (comm_id) REFERENCES comms(id) "dsofk" FOREIGN KEY (dso_id) REFERENCES dsos(id) "evselfk" FOREIGN KEY (evsel_id) REFERENCES selected_events(id) "machinefk" FOREIGN KEY (machine_id) REFERENCES machines(id) "symbolfk" FOREIGN KEY (symbol_id) REFERENCES symbols(id) "threadfk" FOREIGN KEY (thread_id) REFERENCES threads(id) "todsofk" FOREIGN KEY (to_dso_id) REFERENCES dsos(id) "tosymbolfk" FOREIGN KEY (to_symbol_id) REFERENCES symbols(id) lsdb-# \d samples_view View "public.samples_view" Column | Type | Modifiers -------------------+-------------------------+----------- id | bigint | time | bigint | cpu | integer | pid | integer | tid | integer | command | character varying(16) | event | character varying(80) | ip_hex | text | symbol | character varying(2048) | sym_offset | bigint | dso_short_name | character varying(256) | to_ip_hex | text | to_symbol | character varying(2048) | to_sym_offset | bigint | to_dso_short_name | character varying(256) | lsdb=# select * from samples_view; id| time |cpu | pid | tid |command| event | ip_hex | symbol |sym_off| dso_name|to_ip_hex|to_symbol|to_sym_off|to_dso_name --+------------+----+------+------+-------+--------+---------------+---------------------+-------+---------+---------+---------+----------+---------- 1 |12202825015 | -1 | 7339 | 7339 |:17339 | cycles | fffff8104d24a |native_write_msr_safe| 10 | [kernel]| 0 | unknown | 0| unknown 2 |12203258804 | -1 | 7339 | 7339 |:17339 | cycles | fffff8104d24a |native_write_msr_safe| 10 | [kernel]| 0 | unknown | 0| unknown 3 |12203988119 | -1 | 7339 | 7339 |:17339 | cycles | fffff8104d24a |native_write_msr_safe| 10 | [kernel]| 0 | unknown | 0| unknown My notes (which may be out-of-date) on setting up postgresql so you can create databases: fedora: $ sudo yum install postgresql postgresql-server python-pyside qt-postgresql $ sudo su - postgres -c initdb $ sudo service postgresql start $ sudo su - postgres $ createuser -s I used the the unix user name in createuser. If it fails, try createuser without -s and answer the following question to allow your user to create tables: Shall the new role be a superuser? (y/n) y ubuntu: $ sudo apt-get install postgresql $ sudo su - postgres $ createuser Shall the new role be a superuser? (y/n) y You may want to disable automatic startup. One way is to edit /etc/postgresql/9.3/main/start.conf. Another is to disable the init script e.g. sudo update-rc.d postgresql disable Signed-off-by: Adrian Hunter Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lkml.kernel.org/r/1414061124-26830-8-git-send-email-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-postgresql.py | 360 ++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 tools/perf/scripts/python/export-to-postgresql.py (limited to 'tools/perf/scripts/python/export-to-postgresql.py') diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py new file mode 100644 index 000000000000..d8f6df0093d6 --- /dev/null +++ b/tools/perf/scripts/python/export-to-postgresql.py @@ -0,0 +1,360 @@ +# export-to-postgresql.py: export perf data to a postgresql database +# Copyright (c) 2014, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +import os +import sys +import struct +import datetime + +from PySide.QtSql import * + +# Need to access PostgreSQL C library directly to use COPY FROM STDIN +from ctypes import * +libpq = CDLL("libpq.so.5") +PQconnectdb = libpq.PQconnectdb +PQconnectdb.restype = c_void_p +PQfinish = libpq.PQfinish +PQstatus = libpq.PQstatus +PQexec = libpq.PQexec +PQexec.restype = c_void_p +PQresultStatus = libpq.PQresultStatus +PQputCopyData = libpq.PQputCopyData +PQputCopyData.argtypes = [ c_void_p, c_void_p, c_int ] +PQputCopyEnd = libpq.PQputCopyEnd +PQputCopyEnd.argtypes = [ c_void_p, c_void_p ] + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +# These perf imports are not used at present +#from perf_trace_context import * +#from Core import * + +perf_db_export_mode = True + +def usage(): + print >> sys.stderr, "Usage is: export-to-postgresql.py []" + print >> sys.stderr, "where: columns 'all' or 'branches'" + raise Exception("Too few arguments") + +if (len(sys.argv) < 2): + usage() + +dbname = sys.argv[1] + +if (len(sys.argv) >= 3): + columns = sys.argv[2] +else: + columns = "all" + +if columns not in ("all", "branches"): + usage() + +branches = (columns == "branches") + +output_dir_name = os.getcwd() + "/" + dbname + "-perf-data" +os.mkdir(output_dir_name) + +def do_query(q, s): + if (q.exec_(s)): + return + raise Exception("Query failed: " + q.lastError().text()) + +print datetime.datetime.today(), "Creating database..." + +db = QSqlDatabase.addDatabase('QPSQL') +query = QSqlQuery(db) +db.setDatabaseName('postgres') +db.open() +try: + do_query(query, 'CREATE DATABASE ' + dbname) +except: + os.rmdir(output_dir_name) + raise +query.finish() +query.clear() +db.close() + +db.setDatabaseName(dbname) +db.open() + +query = QSqlQuery(db) +do_query(query, 'SET client_min_messages TO WARNING') + +do_query(query, 'CREATE TABLE selected_events (' + 'id bigint NOT NULL,' + 'name varchar(80))') +do_query(query, 'CREATE TABLE machines (' + 'id bigint NOT NULL,' + 'pid integer,' + 'root_dir varchar(4096))') +do_query(query, 'CREATE TABLE threads (' + 'id bigint NOT NULL,' + 'machine_id bigint,' + 'process_id bigint,' + 'pid integer,' + 'tid integer)') +do_query(query, 'CREATE TABLE comms (' + 'id bigint NOT NULL,' + 'comm varchar(16))') +do_query(query, 'CREATE TABLE comm_threads (' + 'id bigint NOT NULL,' + 'comm_id bigint,' + 'thread_id bigint)') +do_query(query, 'CREATE TABLE dsos (' + 'id bigint NOT NULL,' + 'machine_id bigint,' + 'short_name varchar(256),' + 'long_name varchar(4096),' + 'build_id varchar(64))') +do_query(query, 'CREATE TABLE symbols (' + 'id bigint NOT NULL,' + 'dso_id bigint,' + 'sym_start bigint,' + 'sym_end bigint,' + 'binding integer,' + 'name varchar(2048))') +if branches: + do_query(query, 'CREATE TABLE samples (' + 'id bigint NOT NULL,' + 'evsel_id bigint,' + 'machine_id bigint,' + 'thread_id bigint,' + 'comm_id bigint,' + 'dso_id bigint,' + 'symbol_id bigint,' + 'sym_offset bigint,' + 'ip bigint,' + 'time bigint,' + 'cpu integer,' + 'to_dso_id bigint,' + 'to_symbol_id bigint,' + 'to_sym_offset bigint,' + 'to_ip bigint)') +else: + do_query(query, 'CREATE TABLE samples (' + 'id bigint NOT NULL,' + 'evsel_id bigint,' + 'machine_id bigint,' + 'thread_id bigint,' + 'comm_id bigint,' + 'dso_id bigint,' + 'symbol_id bigint,' + 'sym_offset bigint,' + 'ip bigint,' + 'time bigint,' + 'cpu integer,' + 'to_dso_id bigint,' + 'to_symbol_id bigint,' + 'to_sym_offset bigint,' + 'to_ip bigint,' + 'period bigint,' + 'weight bigint,' + 'transaction bigint,' + 'data_src bigint)') + +do_query(query, 'CREATE VIEW samples_view AS ' + 'SELECT ' + 'id,' + 'time,' + 'cpu,' + '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' + '(SELECT tid FROM threads WHERE id = thread_id) AS tid,' + '(SELECT comm FROM comms WHERE id = comm_id) AS command,' + '(SELECT name FROM selected_events WHERE id = evsel_id) AS event,' + 'to_hex(ip) AS ip_hex,' + '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,' + 'sym_offset,' + '(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,' + 'to_hex(to_ip) AS to_ip_hex,' + '(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,' + 'to_sym_offset,' + '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name' + ' FROM samples') + + +file_header = struct.pack("!11sii", "PGCOPY\n\377\r\n\0", 0, 0) +file_trailer = "\377\377" + +def open_output_file(file_name): + path_name = output_dir_name + "/" + file_name + file = open(path_name, "w+") + file.write(file_header) + return file + +def close_output_file(file): + file.write(file_trailer) + file.close() + +def copy_output_file_direct(file, table_name): + close_output_file(file) + sql = "COPY " + table_name + " FROM '" + file.name + "' (FORMAT 'binary')" + do_query(query, sql) + +# Use COPY FROM STDIN because security may prevent postgres from accessing the files directly +def copy_output_file(file, table_name): + conn = PQconnectdb("dbname = " + dbname) + if (PQstatus(conn)): + raise Exception("COPY FROM STDIN PQconnectdb failed") + file.write(file_trailer) + file.seek(0) + sql = "COPY " + table_name + " FROM STDIN (FORMAT 'binary')" + res = PQexec(conn, sql) + if (PQresultStatus(res) != 4): + raise Exception("COPY FROM STDIN PQexec failed") + data = file.read(65536) + while (len(data)): + ret = PQputCopyData(conn, data, len(data)) + if (ret != 1): + raise Exception("COPY FROM STDIN PQputCopyData failed, error " + str(ret)) + data = file.read(65536) + ret = PQputCopyEnd(conn, None) + if (ret != 1): + raise Exception("COPY FROM STDIN PQputCopyEnd failed, error " + str(ret)) + PQfinish(conn) + +def remove_output_file(file): + name = file.name + file.close() + os.unlink(name) + +evsel_file = open_output_file("evsel_table.bin") +machine_file = open_output_file("machine_table.bin") +thread_file = open_output_file("thread_table.bin") +comm_file = open_output_file("comm_table.bin") +comm_thread_file = open_output_file("comm_thread_table.bin") +dso_file = open_output_file("dso_table.bin") +symbol_file = open_output_file("symbol_table.bin") +sample_file = open_output_file("sample_table.bin") + +def trace_begin(): + print datetime.datetime.today(), "Writing to intermediate files..." + # id == 0 means unknown. It is easier to create records for them than replace the zeroes with NULLs + evsel_table(0, "unknown") + machine_table(0, 0, "unknown") + thread_table(0, 0, 0, -1, -1) + comm_table(0, "unknown") + dso_table(0, 0, "unknown", "unknown", "") + symbol_table(0, 0, 0, 0, 0, "unknown") + +unhandled_count = 0 + +def trace_end(): + print datetime.datetime.today(), "Copying to database..." + copy_output_file(evsel_file, "selected_events") + copy_output_file(machine_file, "machines") + copy_output_file(thread_file, "threads") + copy_output_file(comm_file, "comms") + copy_output_file(comm_thread_file, "comm_threads") + copy_output_file(dso_file, "dsos") + copy_output_file(symbol_file, "symbols") + copy_output_file(sample_file, "samples") + + print datetime.datetime.today(), "Removing intermediate files..." + remove_output_file(evsel_file) + remove_output_file(machine_file) + remove_output_file(thread_file) + remove_output_file(comm_file) + remove_output_file(comm_thread_file) + remove_output_file(dso_file) + remove_output_file(symbol_file) + remove_output_file(sample_file) + os.rmdir(output_dir_name) + print datetime.datetime.today(), "Adding primary keys" + do_query(query, 'ALTER TABLE selected_events ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE machines ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE threads ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE comms ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE comm_threads ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE dsos ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE symbols ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE samples ADD PRIMARY KEY (id)') + + print datetime.datetime.today(), "Adding foreign keys" + do_query(query, 'ALTER TABLE threads ' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' + 'ADD CONSTRAINT processfk FOREIGN KEY (process_id) REFERENCES threads (id)') + do_query(query, 'ALTER TABLE comm_threads ' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id)') + do_query(query, 'ALTER TABLE dsos ' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id)') + do_query(query, 'ALTER TABLE symbols ' + 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id)') + do_query(query, 'ALTER TABLE samples ' + 'ADD CONSTRAINT evselfk FOREIGN KEY (evsel_id) REFERENCES selected_events (id),' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id),' + 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),' + 'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),' + 'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) REFERENCES symbols (id)') + + if (unhandled_count): + print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events" + print datetime.datetime.today(), "Done" + +def trace_unhandled(event_name, context, event_fields_dict): + global unhandled_count + unhandled_count += 1 + +def sched__sched_switch(*x): + pass + +def evsel_table(evsel_id, evsel_name, *x): + n = len(evsel_name) + fmt = "!hiqi" + str(n) + "s" + value = struct.pack(fmt, 2, 8, evsel_id, n, evsel_name) + evsel_file.write(value) + +def machine_table(machine_id, pid, root_dir, *x): + n = len(root_dir) + fmt = "!hiqiii" + str(n) + "s" + value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, root_dir) + machine_file.write(value) + +def thread_table(thread_id, machine_id, process_id, pid, tid, *x): + value = struct.pack("!hiqiqiqiiii", 5, 8, thread_id, 8, machine_id, 8, process_id, 4, pid, 4, tid) + thread_file.write(value) + +def comm_table(comm_id, comm_str, *x): + n = len(comm_str) + fmt = "!hiqi" + str(n) + "s" + value = struct.pack(fmt, 2, 8, comm_id, n, comm_str) + comm_file.write(value) + +def comm_thread_table(comm_thread_id, comm_id, thread_id, *x): + fmt = "!hiqiqiq" + value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id) + comm_thread_file.write(value) + +def dso_table(dso_id, machine_id, short_name, long_name, build_id, *x): + n1 = len(short_name) + n2 = len(long_name) + n3 = len(build_id) + fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s" + value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1, short_name, n2, long_name, n3, build_id) + dso_file.write(value) + +def symbol_table(symbol_id, dso_id, sym_start, sym_end, binding, symbol_name, *x): + n = len(symbol_name) + fmt = "!hiqiqiqiqiii" + str(n) + "s" + value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, sym_start, 8, sym_end, 4, binding, n, symbol_name) + symbol_file.write(value) + +def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, symbol_id, sym_offset, ip, time, cpu, to_dso_id, to_symbol_id, to_sym_offset, to_ip, period, weight, transaction, data_src, *x): + if branches: + value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiq", 15, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip) + else: + value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiq", 19, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src) + sample_file.write(value) -- cgit v1.2.3-59-g8ed1b From c29414f5cfd641d956c5287848fdd8f25bb2afa3 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 30 Oct 2014 16:09:44 +0200 Subject: perf tools: Add branch_type and in_tx to Python export Add branch_type and in_tx to Python db export and the export-to-postgresql.py script. Signed-off-by: Adrian Hunter Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lkml.kernel.org/r/1414678188-14946-4-git-send-email-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-postgresql.py | 32 ++++++++++++++++++---- .../util/scripting-engines/trace-event-python.c | 30 +++++++++++++++++++- 2 files changed, 55 insertions(+), 7 deletions(-) (limited to 'tools/perf/scripts/python/export-to-postgresql.py') diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py index d8f6df0093d6..bb79aecccf58 100644 --- a/tools/perf/scripts/python/export-to-postgresql.py +++ b/tools/perf/scripts/python/export-to-postgresql.py @@ -123,6 +123,10 @@ do_query(query, 'CREATE TABLE symbols (' 'sym_end bigint,' 'binding integer,' 'name varchar(2048))') +do_query(query, 'CREATE TABLE branch_types (' + 'id integer NOT NULL,' + 'name varchar(80))') + if branches: do_query(query, 'CREATE TABLE samples (' 'id bigint NOT NULL,' @@ -139,7 +143,9 @@ if branches: 'to_dso_id bigint,' 'to_symbol_id bigint,' 'to_sym_offset bigint,' - 'to_ip bigint)') + 'to_ip bigint,' + 'branch_type integer,' + 'in_tx boolean)') else: do_query(query, 'CREATE TABLE samples (' 'id bigint NOT NULL,' @@ -160,7 +166,9 @@ else: 'period bigint,' 'weight bigint,' 'transaction bigint,' - 'data_src bigint)') + 'data_src bigint,' + 'branch_type integer,' + 'in_tx boolean)') do_query(query, 'CREATE VIEW samples_view AS ' 'SELECT ' @@ -178,7 +186,9 @@ do_query(query, 'CREATE VIEW samples_view AS ' 'to_hex(to_ip) AS to_ip_hex,' '(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,' 'to_sym_offset,' - '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name' + '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,' + '(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,' + 'in_tx' ' FROM samples') @@ -234,6 +244,7 @@ comm_file = open_output_file("comm_table.bin") comm_thread_file = open_output_file("comm_thread_table.bin") dso_file = open_output_file("dso_table.bin") symbol_file = open_output_file("symbol_table.bin") +branch_type_file = open_output_file("branch_type_table.bin") sample_file = open_output_file("sample_table.bin") def trace_begin(): @@ -257,6 +268,7 @@ def trace_end(): copy_output_file(comm_thread_file, "comm_threads") copy_output_file(dso_file, "dsos") copy_output_file(symbol_file, "symbols") + copy_output_file(branch_type_file, "branch_types") copy_output_file(sample_file, "samples") print datetime.datetime.today(), "Removing intermediate files..." @@ -267,6 +279,7 @@ def trace_end(): remove_output_file(comm_thread_file) remove_output_file(dso_file) remove_output_file(symbol_file) + remove_output_file(branch_type_file) remove_output_file(sample_file) os.rmdir(output_dir_name) print datetime.datetime.today(), "Adding primary keys" @@ -277,6 +290,7 @@ def trace_end(): do_query(query, 'ALTER TABLE comm_threads ADD PRIMARY KEY (id)') do_query(query, 'ALTER TABLE dsos ADD PRIMARY KEY (id)') do_query(query, 'ALTER TABLE symbols ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE branch_types ADD PRIMARY KEY (id)') do_query(query, 'ALTER TABLE samples ADD PRIMARY KEY (id)') print datetime.datetime.today(), "Adding foreign keys" @@ -352,9 +366,15 @@ def symbol_table(symbol_id, dso_id, sym_start, sym_end, binding, symbol_name, *x value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, sym_start, 8, sym_end, 4, binding, n, symbol_name) symbol_file.write(value) -def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, symbol_id, sym_offset, ip, time, cpu, to_dso_id, to_symbol_id, to_sym_offset, to_ip, period, weight, transaction, data_src, *x): +def branch_type_table(branch_type, name, *x): + n = len(name) + fmt = "!hiii" + str(n) + "s" + value = struct.pack(fmt, 2, 4, branch_type, n, name) + branch_type_file.write(value) + +def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, symbol_id, sym_offset, ip, time, cpu, to_dso_id, to_symbol_id, to_sym_offset, to_ip, period, weight, transaction, data_src, branch_type, in_tx, *x): if branches: - value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiq", 15, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip) + value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiiiB", 17, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 4, branch_type, 1, in_tx) else: - value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiq", 19, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src) + value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiB", 21, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src, 4, branch_type, 1, in_tx) sample_file.write(value) diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index 2fd7ee8f18c7..f3ca7798b3d0 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -66,6 +66,7 @@ struct tables { PyObject *comm_thread_handler; PyObject *dso_handler; PyObject *symbol_handler; + PyObject *branch_type_handler; PyObject *sample_handler; bool db_export_mode; }; @@ -664,13 +665,31 @@ static int python_export_symbol(struct db_export *dbe, struct symbol *sym, return 0; } +static int python_export_branch_type(struct db_export *dbe, u32 branch_type, + const char *name) +{ + struct tables *tables = container_of(dbe, struct tables, dbe); + PyObject *t; + + t = tuple_new(2); + + tuple_set_s32(t, 0, branch_type); + tuple_set_string(t, 1, name); + + call_object(tables->branch_type_handler, t, "branch_type_table"); + + Py_DECREF(t); + + return 0; +} + static int python_export_sample(struct db_export *dbe, struct export_sample *es) { struct tables *tables = container_of(dbe, struct tables, dbe); PyObject *t; - t = tuple_new(19); + t = tuple_new(21); tuple_set_u64(t, 0, es->db_id); tuple_set_u64(t, 1, es->evsel->db_id); @@ -691,6 +710,8 @@ static int python_export_sample(struct db_export *dbe, tuple_set_u64(t, 16, es->sample->weight); tuple_set_u64(t, 17, es->sample->transaction); tuple_set_u64(t, 18, es->sample->data_src); + tuple_set_s32(t, 19, es->sample->flags & PERF_BRANCH_MASK); + tuple_set_s32(t, 20, !!(es->sample->flags & PERF_IP_FLAG_IN_TX)); call_object(tables->sample_handler, t, "sample_table"); @@ -861,6 +882,7 @@ static void set_table_handlers(struct tables *tables) SET_TABLE_HANDLER(comm_thread); SET_TABLE_HANDLER(dso); SET_TABLE_HANDLER(symbol); + SET_TABLE_HANDLER(branch_type); SET_TABLE_HANDLER(sample); } @@ -910,6 +932,12 @@ static int python_start_script(const char *script, int argc, const char **argv) set_table_handlers(tables); + if (tables->db_export_mode) { + err = db_export__branch_types(&tables->dbe); + if (err) + goto error; + } + return err; error: Py_Finalize(); -- cgit v1.2.3-59-g8ed1b From 6a70307ddcd9999598c399d55dc44c07816a575f Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 30 Oct 2014 16:09:47 +0200 Subject: perf tools: Add call information to Python export Add the ability to export detailed information about paired calls and returns to Python db export and the export-to-postgresql.py script. Signed-off-by: Adrian Hunter Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lkml.kernel.org/r/1414678188-14946-7-git-send-email-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- .../scripts/python/bin/export-to-postgresql-report | 15 ++-- tools/perf/scripts/python/export-to-postgresql.py | 66 ++++++++++++++++- .../util/scripting-engines/trace-event-python.c | 84 +++++++++++++++++++++- 3 files changed, 158 insertions(+), 7 deletions(-) (limited to 'tools/perf/scripts/python/export-to-postgresql.py') diff --git a/tools/perf/scripts/python/bin/export-to-postgresql-report b/tools/perf/scripts/python/bin/export-to-postgresql-report index a8fdd15f85bf..cd335b6e2a01 100644 --- a/tools/perf/scripts/python/bin/export-to-postgresql-report +++ b/tools/perf/scripts/python/bin/export-to-postgresql-report @@ -1,6 +1,6 @@ #!/bin/bash # description: export perf data to a postgresql database -# args: [database name] [columns] +# args: [database name] [columns] [calls] n_args=0 for i in "$@" do @@ -9,11 +9,16 @@ do fi n_args=$(( $n_args + 1 )) done -if [ "$n_args" -gt 2 ] ; then - echo "usage: export-to-postgresql-report [database name] [columns]" +if [ "$n_args" -gt 3 ] ; then + echo "usage: export-to-postgresql-report [database name] [columns] [calls]" exit fi -if [ "$n_args" -gt 1 ] ; then +if [ "$n_args" -gt 2 ] ; then + dbname=$1 + columns=$2 + calls=$3 + shift 3 +elif [ "$n_args" -gt 1 ] ; then dbname=$1 columns=$2 shift 2 @@ -21,4 +26,4 @@ elif [ "$n_args" -gt 0 ] ; then dbname=$1 shift fi -perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/export-to-postgresql.py $dbname $columns +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/export-to-postgresql.py $dbname $columns $calls diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py index bb79aecccf58..4cdafd880074 100644 --- a/tools/perf/scripts/python/export-to-postgresql.py +++ b/tools/perf/scripts/python/export-to-postgresql.py @@ -40,10 +40,12 @@ sys.path.append(os.environ['PERF_EXEC_PATH'] + \ #from Core import * perf_db_export_mode = True +perf_db_export_calls = False def usage(): - print >> sys.stderr, "Usage is: export-to-postgresql.py []" + print >> sys.stderr, "Usage is: export-to-postgresql.py [] []" print >> sys.stderr, "where: columns 'all' or 'branches'" + print >> sys.stderr, " calls 'calls' => create calls table" raise Exception("Too few arguments") if (len(sys.argv) < 2): @@ -61,6 +63,12 @@ if columns not in ("all", "branches"): branches = (columns == "branches") +if (len(sys.argv) >= 4): + if (sys.argv[3] == "calls"): + perf_db_export_calls = True + else: + usage() + output_dir_name = os.getcwd() + "/" + dbname + "-perf-data" os.mkdir(output_dir_name) @@ -170,6 +178,25 @@ else: 'branch_type integer,' 'in_tx boolean)') +if perf_db_export_calls: + do_query(query, 'CREATE TABLE call_paths (' + 'id bigint NOT NULL,' + 'parent_id bigint,' + 'symbol_id bigint,' + 'ip bigint)') + do_query(query, 'CREATE TABLE calls (' + 'id bigint NOT NULL,' + 'thread_id bigint,' + 'comm_id bigint,' + 'call_path_id bigint,' + 'call_time bigint,' + 'return_time bigint,' + 'branch_count bigint,' + 'call_id bigint,' + 'return_id bigint,' + 'parent_call_path_id bigint,' + 'flags integer)') + do_query(query, 'CREATE VIEW samples_view AS ' 'SELECT ' 'id,' @@ -246,6 +273,9 @@ dso_file = open_output_file("dso_table.bin") symbol_file = open_output_file("symbol_table.bin") branch_type_file = open_output_file("branch_type_table.bin") sample_file = open_output_file("sample_table.bin") +if perf_db_export_calls: + call_path_file = open_output_file("call_path_table.bin") + call_file = open_output_file("call_table.bin") def trace_begin(): print datetime.datetime.today(), "Writing to intermediate files..." @@ -256,6 +286,9 @@ def trace_begin(): comm_table(0, "unknown") dso_table(0, 0, "unknown", "unknown", "") symbol_table(0, 0, 0, 0, 0, "unknown") + sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + if perf_db_export_calls: + call_path_table(0, 0, 0, 0) unhandled_count = 0 @@ -270,6 +303,9 @@ def trace_end(): copy_output_file(symbol_file, "symbols") copy_output_file(branch_type_file, "branch_types") copy_output_file(sample_file, "samples") + if perf_db_export_calls: + copy_output_file(call_path_file, "call_paths") + copy_output_file(call_file, "calls") print datetime.datetime.today(), "Removing intermediate files..." remove_output_file(evsel_file) @@ -281,6 +317,9 @@ def trace_end(): remove_output_file(symbol_file) remove_output_file(branch_type_file) remove_output_file(sample_file) + if perf_db_export_calls: + remove_output_file(call_path_file) + remove_output_file(call_file) os.rmdir(output_dir_name) print datetime.datetime.today(), "Adding primary keys" do_query(query, 'ALTER TABLE selected_events ADD PRIMARY KEY (id)') @@ -292,6 +331,9 @@ def trace_end(): do_query(query, 'ALTER TABLE symbols ADD PRIMARY KEY (id)') do_query(query, 'ALTER TABLE branch_types ADD PRIMARY KEY (id)') do_query(query, 'ALTER TABLE samples ADD PRIMARY KEY (id)') + if perf_db_export_calls: + do_query(query, 'ALTER TABLE call_paths ADD PRIMARY KEY (id)') + do_query(query, 'ALTER TABLE calls ADD PRIMARY KEY (id)') print datetime.datetime.today(), "Adding foreign keys" do_query(query, 'ALTER TABLE threads ' @@ -313,6 +355,18 @@ def trace_end(): 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),' 'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),' 'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) REFERENCES symbols (id)') + if perf_db_export_calls: + do_query(query, 'ALTER TABLE call_paths ' + 'ADD CONSTRAINT parentfk FOREIGN KEY (parent_id) REFERENCES call_paths (id),' + 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id)') + do_query(query, 'ALTER TABLE calls ' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT call_pathfk FOREIGN KEY (call_path_id) REFERENCES call_paths (id),' + 'ADD CONSTRAINT callfk FOREIGN KEY (call_id) REFERENCES samples (id),' + 'ADD CONSTRAINT returnfk FOREIGN KEY (return_id) REFERENCES samples (id),' + 'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) REFERENCES call_paths (id)') + do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)') if (unhandled_count): print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events" @@ -378,3 +432,13 @@ def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, sy else: value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiB", 21, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src, 4, branch_type, 1, in_tx) sample_file.write(value) + +def call_path_table(cp_id, parent_id, symbol_id, ip, *x): + fmt = "!hiqiqiqiq" + value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip) + call_path_file.write(value) + +def call_return_table(cr_id, thread_id, comm_id, call_path_id, call_time, return_time, branch_count, call_id, return_id, parent_call_path_id, flags, *x): + fmt = "!hiqiqiqiqiqiqiqiqiqiqii" + value = struct.pack(fmt, 11, 8, cr_id, 8, thread_id, 8, comm_id, 8, call_path_id, 8, call_time, 8, return_time, 8, branch_count, 8, call_id, 8, return_id, 8, parent_call_path_id, 4, flags) + call_file.write(value) diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index f3ca7798b3d0..cb1d9602f418 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -37,6 +37,7 @@ #include "../comm.h" #include "../machine.h" #include "../db-export.h" +#include "../thread-stack.h" #include "../trace-event.h" #include "../machine.h" @@ -68,6 +69,8 @@ struct tables { PyObject *symbol_handler; PyObject *branch_type_handler; PyObject *sample_handler; + PyObject *call_path_handler; + PyObject *call_return_handler; bool db_export_mode; }; @@ -720,6 +723,64 @@ static int python_export_sample(struct db_export *dbe, return 0; } +static int python_export_call_path(struct db_export *dbe, struct call_path *cp) +{ + struct tables *tables = container_of(dbe, struct tables, dbe); + PyObject *t; + u64 parent_db_id, sym_db_id; + + parent_db_id = cp->parent ? cp->parent->db_id : 0; + sym_db_id = cp->sym ? *(u64 *)symbol__priv(cp->sym) : 0; + + t = tuple_new(4); + + tuple_set_u64(t, 0, cp->db_id); + tuple_set_u64(t, 1, parent_db_id); + tuple_set_u64(t, 2, sym_db_id); + tuple_set_u64(t, 3, cp->ip); + + call_object(tables->call_path_handler, t, "call_path_table"); + + Py_DECREF(t); + + return 0; +} + +static int python_export_call_return(struct db_export *dbe, + struct call_return *cr) +{ + struct tables *tables = container_of(dbe, struct tables, dbe); + u64 comm_db_id = cr->comm ? cr->comm->db_id : 0; + PyObject *t; + + t = tuple_new(11); + + tuple_set_u64(t, 0, cr->db_id); + tuple_set_u64(t, 1, cr->thread->db_id); + tuple_set_u64(t, 2, comm_db_id); + tuple_set_u64(t, 3, cr->cp->db_id); + tuple_set_u64(t, 4, cr->call_time); + tuple_set_u64(t, 5, cr->return_time); + tuple_set_u64(t, 6, cr->branch_count); + tuple_set_u64(t, 7, cr->call_ref); + tuple_set_u64(t, 8, cr->return_ref); + tuple_set_u64(t, 9, cr->cp->parent->db_id); + tuple_set_s32(t, 10, cr->flags); + + call_object(tables->call_return_handler, t, "call_return_table"); + + Py_DECREF(t); + + return 0; +} + +static int python_process_call_return(struct call_return *cr, void *data) +{ + struct db_export *dbe = data; + + return db_export__call_return(dbe, cr); +} + static void python_process_general_event(struct perf_sample *sample, struct perf_evsel *evsel, struct thread *thread, @@ -852,7 +913,9 @@ error: static void set_table_handlers(struct tables *tables) { const char *perf_db_export_mode = "perf_db_export_mode"; - PyObject *db_export_mode; + const char *perf_db_export_calls = "perf_db_export_calls"; + PyObject *db_export_mode, *db_export_calls; + bool export_calls = false; int ret; memset(tables, 0, sizeof(struct tables)); @@ -869,6 +932,23 @@ static void set_table_handlers(struct tables *tables) if (!ret) return; + tables->dbe.crp = NULL; + db_export_calls = PyDict_GetItemString(main_dict, perf_db_export_calls); + if (db_export_calls) { + ret = PyObject_IsTrue(db_export_calls); + if (ret == -1) + handler_call_die(perf_db_export_calls); + export_calls = !!ret; + } + + if (export_calls) { + tables->dbe.crp = + call_return_processor__new(python_process_call_return, + &tables->dbe); + if (!tables->dbe.crp) + Py_FatalError("failed to create calls processor"); + } + tables->db_export_mode = true; /* * Reserve per symbol space for symbol->db_id via symbol__priv() @@ -884,6 +964,8 @@ static void set_table_handlers(struct tables *tables) SET_TABLE_HANDLER(symbol); SET_TABLE_HANDLER(branch_type); SET_TABLE_HANDLER(sample); + SET_TABLE_HANDLER(call_path); + SET_TABLE_HANDLER(call_return); } /* -- cgit v1.2.3-59-g8ed1b From 4b715d24f4f14731c7b553cbb8604fe865cb8d3c Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Fri, 17 Jul 2015 19:33:45 +0300 Subject: perf tools: Add example call-graph script Add a script to produce a call-graph from data exported to a postgresql database and derived from a processor trace event like intel_pt or intel_bts. Refer to comments in the scripts call-graph-from-postgresql.py and export-to-postgresql.py for more details on how to set up the environment, install the required packages, etc. Committer note: From the scripts, for convenience while reading 'git log': An example of using this script with Intel PT: $ perf record -e intel_pt//u ls $ perf script -s ~/libexec/perf-core/scripts/python/export-to-postgresql.py pt_example branches calls 2015-05-29 12:49:23.464364 Creating database... 2015-05-29 12:49:26.281717 Writing to intermediate files... 2015-05-29 12:49:27.190383 Copying to database... 2015-05-29 12:49:28.140451 Removing intermediate files... 2015-05-29 12:49:28.147451 Adding primary keys 2015-05-29 12:49:28.655683 Adding foreign keys 2015-05-29 12:49:29.365350 Done $ python tools/perf/scripts/python/call-graph-from-postgresql.py pt_example # The result is a GUI window with a tree representing a context-sensitive # call-graph. Expanding a couple of levels of the tree and adjusting column # widths to suit will display something like: Call Graph: pt_example Call Path |Object |Count|Time(ns)|Time(%)|Branch Count|Branch Count(%) v- ls v- 2638:2638 v- _start ld-2.19.so 1 10074071 100.0 211135 100.0 |- unknown unknown 1 13198 0.1 1 0.0 >- _dl_start ld-2.19.so 1 1400980 13.9 19637 9.3 >- _d_linit_internal ld-2.19.so 1 448152 4.4 11094 5.3 v-__libc_start_main@plt ls 1 8211741 81.5 180397 85.4 >- _dl_fixup ld-2.19.so 1 7607 0.1 108 0.1 >- __cxa_atexit libc-2.19.so 1 11737 0.1 10 0.0 >- __libc_csu_init ls 1 10354 0.1 10 0.0 |- _setjmp libc-2.19.so 1 0 0.0 4 0.0 v- main ls 1 8182043 99.6 180254 99.9 Signed-off-by: Adrian Hunter Tested-by: Arnaldo Carvalho de Melo Cc: Jiri Olsa Link: http://lkml.kernel.org/r/1437150840-31811-11-git-send-email-adrian.hunter@intel.com [ Added 'python-pyside qt-postgresql' to the yum cmdline installing required packages ] Signed-off-by: Arnaldo Carvalho de Melo --- .../scripts/python/call-graph-from-postgresql.py | 327 +++++++++++++++++++++ tools/perf/scripts/python/export-to-postgresql.py | 47 +++ 2 files changed, 374 insertions(+) create mode 100644 tools/perf/scripts/python/call-graph-from-postgresql.py (limited to 'tools/perf/scripts/python/export-to-postgresql.py') diff --git a/tools/perf/scripts/python/call-graph-from-postgresql.py b/tools/perf/scripts/python/call-graph-from-postgresql.py new file mode 100644 index 000000000000..e78fdc2a5a9d --- /dev/null +++ b/tools/perf/scripts/python/call-graph-from-postgresql.py @@ -0,0 +1,327 @@ +#!/usr/bin/python2 +# call-graph-from-postgresql.py: create call-graph from postgresql database +# Copyright (c) 2014, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# To use this script you will need to have exported data using the +# export-to-postgresql.py script. Refer to that script for details. +# +# Following on from the example in the export-to-postgresql.py script, a +# call-graph can be displayed for the pt_example database like this: +# +# python tools/perf/scripts/python/call-graph-from-postgresql.py pt_example +# +# Note this script supports connecting to remote databases by setting hostname, +# port, username, password, and dbname e.g. +# +# python tools/perf/scripts/python/call-graph-from-postgresql.py "hostname=myhost username=myuser password=mypassword dbname=pt_example" +# +# The result is a GUI window with a tree representing a context-sensitive +# call-graph. Expanding a couple of levels of the tree and adjusting column +# widths to suit will display something like: +# +# Call Graph: pt_example +# Call Path Object Count Time(ns) Time(%) Branch Count Branch Count(%) +# v- ls +# v- 2638:2638 +# v- _start ld-2.19.so 1 10074071 100.0 211135 100.0 +# |- unknown unknown 1 13198 0.1 1 0.0 +# >- _dl_start ld-2.19.so 1 1400980 13.9 19637 9.3 +# >- _d_linit_internal ld-2.19.so 1 448152 4.4 11094 5.3 +# v-__libc_start_main@plt ls 1 8211741 81.5 180397 85.4 +# >- _dl_fixup ld-2.19.so 1 7607 0.1 108 0.1 +# >- __cxa_atexit libc-2.19.so 1 11737 0.1 10 0.0 +# >- __libc_csu_init ls 1 10354 0.1 10 0.0 +# |- _setjmp libc-2.19.so 1 0 0.0 4 0.0 +# v- main ls 1 8182043 99.6 180254 99.9 +# +# Points to note: +# The top level is a command name (comm) +# The next level is a thread (pid:tid) +# Subsequent levels are functions +# 'Count' is the number of calls +# 'Time' is the elapsed time until the function returns +# Percentages are relative to the level above +# 'Branch Count' is the total number of branches for that function and all +# functions that it calls + +import sys +from PySide.QtCore import * +from PySide.QtGui import * +from PySide.QtSql import * +from decimal import * + +class TreeItem(): + + def __init__(self, db, row, parent_item): + self.db = db + self.row = row + self.parent_item = parent_item + self.query_done = False; + self.child_count = 0 + self.child_items = [] + self.data = ["", "", "", "", "", "", ""] + self.comm_id = 0 + self.thread_id = 0 + self.call_path_id = 1 + self.branch_count = 0 + self.time = 0 + if not parent_item: + self.setUpRoot() + + def setUpRoot(self): + self.query_done = True + query = QSqlQuery(self.db) + ret = query.exec_('SELECT id, comm FROM comms') + if not ret: + raise Exception("Query failed: " + query.lastError().text()) + while query.next(): + if not query.value(0): + continue + child_item = TreeItem(self.db, self.child_count, self) + self.child_items.append(child_item) + self.child_count += 1 + child_item.setUpLevel1(query.value(0), query.value(1)) + + def setUpLevel1(self, comm_id, comm): + self.query_done = True; + self.comm_id = comm_id + self.data[0] = comm + self.child_items = [] + self.child_count = 0 + query = QSqlQuery(self.db) + ret = query.exec_('SELECT thread_id, ( SELECT pid FROM threads WHERE id = thread_id ), ( SELECT tid FROM threads WHERE id = thread_id ) FROM comm_threads WHERE comm_id = ' + str(comm_id)) + if not ret: + raise Exception("Query failed: " + query.lastError().text()) + while query.next(): + child_item = TreeItem(self.db, self.child_count, self) + self.child_items.append(child_item) + self.child_count += 1 + child_item.setUpLevel2(comm_id, query.value(0), query.value(1), query.value(2)) + + def setUpLevel2(self, comm_id, thread_id, pid, tid): + self.comm_id = comm_id + self.thread_id = thread_id + self.data[0] = str(pid) + ":" + str(tid) + + def getChildItem(self, row): + return self.child_items[row] + + def getParentItem(self): + return self.parent_item + + def getRow(self): + return self.row + + def timePercent(self, b): + if not self.time: + return "0.0" + x = (b * Decimal(100)) / self.time + return str(x.quantize(Decimal('.1'), rounding=ROUND_HALF_UP)) + + def branchPercent(self, b): + if not self.branch_count: + return "0.0" + x = (b * Decimal(100)) / self.branch_count + return str(x.quantize(Decimal('.1'), rounding=ROUND_HALF_UP)) + + def addChild(self, call_path_id, name, dso, count, time, branch_count): + child_item = TreeItem(self.db, self.child_count, self) + child_item.comm_id = self.comm_id + child_item.thread_id = self.thread_id + child_item.call_path_id = call_path_id + child_item.branch_count = branch_count + child_item.time = time + child_item.data[0] = name + if dso == "[kernel.kallsyms]": + dso = "[kernel]" + child_item.data[1] = dso + child_item.data[2] = str(count) + child_item.data[3] = str(time) + child_item.data[4] = self.timePercent(time) + child_item.data[5] = str(branch_count) + child_item.data[6] = self.branchPercent(branch_count) + self.child_items.append(child_item) + self.child_count += 1 + + def selectCalls(self): + self.query_done = True; + query = QSqlQuery(self.db) + ret = query.exec_('SELECT id, call_path_id, branch_count, call_time, return_time, ' + '( SELECT name FROM symbols WHERE id = ( SELECT symbol_id FROM call_paths WHERE id = call_path_id ) ), ' + '( SELECT short_name FROM dsos WHERE id = ( SELECT dso_id FROM symbols WHERE id = ( SELECT symbol_id FROM call_paths WHERE id = call_path_id ) ) ), ' + '( SELECT ip FROM call_paths where id = call_path_id ) ' + 'FROM calls WHERE parent_call_path_id = ' + str(self.call_path_id) + ' AND comm_id = ' + str(self.comm_id) + ' AND thread_id = ' + str(self.thread_id) + + 'ORDER BY call_path_id') + if not ret: + raise Exception("Query failed: " + query.lastError().text()) + last_call_path_id = 0 + name = "" + dso = "" + count = 0 + branch_count = 0 + total_branch_count = 0 + time = 0 + total_time = 0 + while query.next(): + if query.value(1) == last_call_path_id: + count += 1 + branch_count += query.value(2) + time += query.value(4) - query.value(3) + else: + if count: + self.addChild(last_call_path_id, name, dso, count, time, branch_count) + last_call_path_id = query.value(1) + name = query.value(5) + dso = query.value(6) + count = 1 + total_branch_count += branch_count + total_time += time + branch_count = query.value(2) + time = query.value(4) - query.value(3) + if count: + self.addChild(last_call_path_id, name, dso, count, time, branch_count) + total_branch_count += branch_count + total_time += time + # Top level does not have time or branch count, so fix that here + if total_branch_count > self.branch_count: + self.branch_count = total_branch_count + if self.branch_count: + for child_item in self.child_items: + child_item.data[6] = self.branchPercent(child_item.branch_count) + if total_time > self.time: + self.time = total_time + if self.time: + for child_item in self.child_items: + child_item.data[4] = self.timePercent(child_item.time) + + def childCount(self): + if not self.query_done: + self.selectCalls() + return self.child_count + + def columnCount(self): + return 7 + + def columnHeader(self, column): + headers = ["Call Path", "Object", "Count ", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "] + return headers[column] + + def getData(self, column): + return self.data[column] + +class TreeModel(QAbstractItemModel): + + def __init__(self, db, parent=None): + super(TreeModel, self).__init__(parent) + self.db = db + self.root = TreeItem(db, 0, None) + + def columnCount(self, parent): + return self.root.columnCount() + + def rowCount(self, parent): + if parent.isValid(): + parent_item = parent.internalPointer() + else: + parent_item = self.root + return parent_item.childCount() + + def headerData(self, section, orientation, role): + if role == Qt.TextAlignmentRole: + if section > 1: + return Qt.AlignRight + if role != Qt.DisplayRole: + return None + if orientation != Qt.Horizontal: + return None + return self.root.columnHeader(section) + + def parent(self, child): + child_item = child.internalPointer() + if child_item is self.root: + return QModelIndex() + parent_item = child_item.getParentItem() + return self.createIndex(parent_item.getRow(), 0, parent_item) + + def index(self, row, column, parent): + if parent.isValid(): + parent_item = parent.internalPointer() + else: + parent_item = self.root + child_item = parent_item.getChildItem(row) + return self.createIndex(row, column, child_item) + + def data(self, index, role): + if role == Qt.TextAlignmentRole: + if index.column() > 1: + return Qt.AlignRight + if role != Qt.DisplayRole: + return None + index_item = index.internalPointer() + return index_item.getData(index.column()) + +class MainWindow(QMainWindow): + + def __init__(self, db, dbname, parent=None): + super(MainWindow, self).__init__(parent) + + self.setObjectName("MainWindow") + self.setWindowTitle("Call Graph: " + dbname) + self.move(100, 100) + self.resize(800, 600) + style = self.style() + icon = style.standardIcon(QStyle.SP_MessageBoxInformation) + self.setWindowIcon(icon); + + self.model = TreeModel(db) + + self.view = QTreeView() + self.view.setModel(self.model) + + self.setCentralWidget(self.view) + +if __name__ == '__main__': + if (len(sys.argv) < 2): + print >> sys.stderr, "Usage is: call-graph-from-postgresql.py " + raise Exception("Too few arguments") + + dbname = sys.argv[1] + + db = QSqlDatabase.addDatabase('QPSQL') + + opts = dbname.split() + for opt in opts: + if '=' in opt: + opt = opt.split('=') + if opt[0] == 'hostname': + db.setHostName(opt[1]) + elif opt[0] == 'port': + db.setPort(int(opt[1])) + elif opt[0] == 'username': + db.setUserName(opt[1]) + elif opt[0] == 'password': + db.setPassword(opt[1]) + elif opt[0] == 'dbname': + dbname = opt[1] + else: + dbname = opt + + db.setDatabaseName(dbname) + if not db.open(): + raise Exception("Failed to open database " + dbname + " error: " + db.lastError().text()) + + app = QApplication(sys.argv) + window = MainWindow(db, dbname) + window.show() + err = app.exec_() + db.close() + sys.exit(err) diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py index 4cdafd880074..84a32037a80f 100644 --- a/tools/perf/scripts/python/export-to-postgresql.py +++ b/tools/perf/scripts/python/export-to-postgresql.py @@ -15,6 +15,53 @@ import sys import struct import datetime +# To use this script you will need to have installed package python-pyside which +# provides LGPL-licensed Python bindings for Qt. You will also need the package +# libqt4-sql-psql for Qt postgresql support. +# +# The script assumes postgresql is running on the local machine and that the +# user has postgresql permissions to create databases. Examples of installing +# postgresql and adding such a user are: +# +# fedora: +# +# $ sudo yum install postgresql postgresql-server python-pyside qt-postgresql +# $ sudo su - postgres -c initdb +# $ sudo service postgresql start +# $ sudo su - postgres +# $ createuser +# Shall the new role be a superuser? (y/n) y +# +# ubuntu: +# +# $ sudo apt-get install postgresql +# $ sudo su - postgres +# $ createuser +# Shall the new role be a superuser? (y/n) y +# +# An example of using this script with Intel PT: +# +# $ perf record -e intel_pt//u ls +# $ perf script -s ~/libexec/perf-core/scripts/python/export-to-postgresql.py pt_example branches calls +# 2015-05-29 12:49:23.464364 Creating database... +# 2015-05-29 12:49:26.281717 Writing to intermediate files... +# 2015-05-29 12:49:27.190383 Copying to database... +# 2015-05-29 12:49:28.140451 Removing intermediate files... +# 2015-05-29 12:49:28.147451 Adding primary keys +# 2015-05-29 12:49:28.655683 Adding foreign keys +# 2015-05-29 12:49:29.365350 Done +# +# To browse the database, psql can be used e.g. +# +# $ psql pt_example +# pt_example=# select * from samples_view where id < 100; +# pt_example=# \d+ +# pt_example=# \d+ samples_view +# pt_example=# \q +# +# An example of using the database is provided by the script +# call-graph-from-postgresql.py. Refer to that script for details. + from PySide.QtSql import * # Need to access PostgreSQL C library directly to use COPY FROM STDIN -- cgit v1.2.3-59-g8ed1b From 35ca01c117da9b8e5b60204f730cdde414735596 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Fri, 25 Sep 2015 16:15:38 +0300 Subject: perf tools: Add more documentation to export-to-postgresql.py script Add some comments to the script and some 'views' to the created database that better illustrate the database structure and how it can be used. Signed-off-by: Adrian Hunter Cc: Jiri Olsa Link: http://lkml.kernel.org/r/1443186956-18718-8-git-send-email-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-postgresql.py | 221 ++++++++++++++++++++++ 1 file changed, 221 insertions(+) (limited to 'tools/perf/scripts/python/export-to-postgresql.py') diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py index 84a32037a80f..1b02cdc0cab6 100644 --- a/tools/perf/scripts/python/export-to-postgresql.py +++ b/tools/perf/scripts/python/export-to-postgresql.py @@ -61,6 +61,142 @@ import datetime # # An example of using the database is provided by the script # call-graph-from-postgresql.py. Refer to that script for details. +# +# Tables: +# +# The tables largely correspond to perf tools' data structures. They are largely self-explanatory. +# +# samples +# +# 'samples' is the main table. It represents what instruction was executing at a point in time +# when something (a selected event) happened. The memory address is the instruction pointer or 'ip'. +# +# calls +# +# 'calls' represents function calls and is related to 'samples' by 'call_id' and 'return_id'. +# 'calls' is only created when the 'calls' option to this script is specified. +# +# call_paths +# +# 'call_paths' represents all the call stacks. Each 'call' has an associated record in 'call_paths'. +# 'calls_paths' is only created when the 'calls' option to this script is specified. +# +# branch_types +# +# 'branch_types' provides descriptions for each type of branch. +# +# comm_threads +# +# 'comm_threads' shows how 'comms' relates to 'threads'. +# +# comms +# +# 'comms' contains a record for each 'comm' - the name given to the executable that is running. +# +# dsos +# +# 'dsos' contains a record for each executable file or library. +# +# machines +# +# 'machines' can be used to distinguish virtual machines if virtualization is supported. +# +# selected_events +# +# 'selected_events' contains a record for each kind of event that has been sampled. +# +# symbols +# +# 'symbols' contains a record for each symbol. Only symbols that have samples are present. +# +# threads +# +# 'threads' contains a record for each thread. +# +# Views: +# +# Most of the tables have views for more friendly display. The views are: +# +# calls_view +# call_paths_view +# comm_threads_view +# dsos_view +# machines_view +# samples_view +# symbols_view +# threads_view +# +# More examples of browsing the database with psql: +# Note that some of the examples are not the most optimal SQL query. +# Note that call information is only available if the script's 'calls' option has been used. +# +# Top 10 function calls (not aggregated by symbol): +# +# SELECT * FROM calls_view ORDER BY elapsed_time DESC LIMIT 10; +# +# Top 10 function calls (aggregated by symbol): +# +# SELECT symbol_id,(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, +# SUM(elapsed_time) AS tot_elapsed_time,SUM(branch_count) AS tot_branch_count +# FROM calls_view GROUP BY symbol_id ORDER BY tot_elapsed_time DESC LIMIT 10; +# +# Note that the branch count gives a rough estimation of cpu usage, so functions +# that took a long time but have a relatively low branch count must have spent time +# waiting. +# +# Find symbols by pattern matching on part of the name (e.g. names containing 'alloc'): +# +# SELECT * FROM symbols_view WHERE name LIKE '%alloc%'; +# +# Top 10 function calls for a specific symbol (e.g. whose symbol_id is 187): +# +# SELECT * FROM calls_view WHERE symbol_id = 187 ORDER BY elapsed_time DESC LIMIT 10; +# +# Show function calls made by function in the same context (i.e. same call path) (e.g. one with call_path_id 254): +# +# SELECT * FROM calls_view WHERE parent_call_path_id = 254; +# +# Show branches made during a function call (e.g. where call_id is 29357 and return_id is 29370 and tid is 29670) +# +# SELECT * FROM samples_view WHERE id >= 29357 AND id <= 29370 AND tid = 29670 AND event LIKE 'branches%'; +# +# Show transactions: +# +# SELECT * FROM samples_view WHERE event = 'transactions'; +# +# Note transaction start has 'in_tx' true whereas, transaction end has 'in_tx' false. +# Transaction aborts have branch_type_name 'transaction abort' +# +# Show transaction aborts: +# +# SELECT * FROM samples_view WHERE event = 'transactions' AND branch_type_name = 'transaction abort'; +# +# To print a call stack requires walking the call_paths table. For example this python script: +# #!/usr/bin/python2 +# +# import sys +# from PySide.QtSql import * +# +# if __name__ == '__main__': +# if (len(sys.argv) < 3): +# print >> sys.stderr, "Usage is: printcallstack.py " +# raise Exception("Too few arguments") +# dbname = sys.argv[1] +# call_path_id = sys.argv[2] +# db = QSqlDatabase.addDatabase('QPSQL') +# db.setDatabaseName(dbname) +# if not db.open(): +# raise Exception("Failed to open database " + dbname + " error: " + db.lastError().text()) +# query = QSqlQuery(db) +# print " id ip symbol_id symbol dso_id dso_short_name" +# while call_path_id != 0 and call_path_id != 1: +# ret = query.exec_('SELECT * FROM call_paths_view WHERE id = ' + str(call_path_id)) +# if not ret: +# raise Exception("Query failed: " + query.lastError().text()) +# if not query.next(): +# raise Exception("Query failed") +# print "{0:>6} {1:>10} {2:>9} {3:<30} {4:>6} {5:<30}".format(query.value(0), query.value(1), query.value(2), query.value(3), query.value(4), query.value(5)) +# call_path_id = query.value(6) from PySide.QtSql import * @@ -244,6 +380,91 @@ if perf_db_export_calls: 'parent_call_path_id bigint,' 'flags integer)') +do_query(query, 'CREATE VIEW machines_view AS ' + 'SELECT ' + 'id,' + 'pid,' + 'root_dir,' + 'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest' + ' FROM machines') + +do_query(query, 'CREATE VIEW dsos_view AS ' + 'SELECT ' + 'id,' + 'machine_id,' + '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,' + 'short_name,' + 'long_name,' + 'build_id' + ' FROM dsos') + +do_query(query, 'CREATE VIEW symbols_view AS ' + 'SELECT ' + 'id,' + 'name,' + '(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,' + 'dso_id,' + 'sym_start,' + 'sym_end,' + 'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding' + ' FROM symbols') + +do_query(query, 'CREATE VIEW threads_view AS ' + 'SELECT ' + 'id,' + 'machine_id,' + '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,' + 'process_id,' + 'pid,' + 'tid' + ' FROM threads') + +do_query(query, 'CREATE VIEW comm_threads_view AS ' + 'SELECT ' + 'comm_id,' + '(SELECT comm FROM comms WHERE id = comm_id) AS command,' + 'thread_id,' + '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' + '(SELECT tid FROM threads WHERE id = thread_id) AS tid' + ' FROM comm_threads') + +if perf_db_export_calls: + do_query(query, 'CREATE VIEW call_paths_view AS ' + 'SELECT ' + 'c.id,' + 'to_hex(c.ip) AS ip,' + 'c.symbol_id,' + '(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,' + '(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,' + '(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name,' + 'c.parent_id,' + 'to_hex(p.ip) AS parent_ip,' + 'p.symbol_id AS parent_symbol_id,' + '(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,' + '(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,' + '(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name' + ' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id') + do_query(query, 'CREATE VIEW calls_view AS ' + 'SELECT ' + 'calls.id,' + 'thread_id,' + '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' + '(SELECT tid FROM threads WHERE id = thread_id) AS tid,' + '(SELECT comm FROM comms WHERE id = comm_id) AS command,' + 'call_path_id,' + 'to_hex(ip) AS ip,' + 'symbol_id,' + '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,' + 'call_time,' + 'return_time,' + 'return_time - call_time AS elapsed_time,' + 'branch_count,' + 'call_id,' + 'return_id,' + 'CASE WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' ELSE \'\' END AS flags,' + 'parent_call_path_id' + ' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id') + do_query(query, 'CREATE VIEW samples_view AS ' 'SELECT ' 'id,' -- cgit v1.2.3-59-g8ed1b