diff options
Diffstat (limited to 'tools/perf/builtin-c2c.c')
| -rw-r--r-- | tools/perf/builtin-c2c.c | 962 |
1 files changed, 656 insertions, 306 deletions
diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index d617d5682c68..16b40f5d43db 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -41,7 +41,9 @@ #include "symbol.h" #include "ui/ui.h" #include "ui/progress.h" -#include "../perf.h" +#include "pmus.h" +#include "string2.h" +#include "util/util.h" struct c2c_hists { struct hists hists; @@ -52,6 +54,8 @@ struct c2c_hists { struct compute_stats { struct stats lcl_hitm; struct stats rmt_hitm; + struct stats lcl_peer; + struct stats rmt_peer; struct stats load; }; @@ -97,8 +101,8 @@ struct perf_c2c { bool symbol_full; bool stitch_lbr; - /* HITM shared clines stats */ - struct c2c_stats hitm_stats; + /* Shared cache line stats */ + struct c2c_stats shared_clines_stats; int shared_clines; int display; @@ -110,16 +114,18 @@ struct perf_c2c { }; enum { - DISPLAY_LCL, - DISPLAY_RMT, - DISPLAY_TOT, + DISPLAY_LCL_HITM, + DISPLAY_RMT_HITM, + DISPLAY_TOT_HITM, + DISPLAY_SNP_PEER, DISPLAY_MAX, }; static const char *display_str[DISPLAY_MAX] = { - [DISPLAY_LCL] = "Local", - [DISPLAY_RMT] = "Remote", - [DISPLAY_TOT] = "Total", + [DISPLAY_LCL_HITM] = "Local HITMs", + [DISPLAY_RMT_HITM] = "Remote HITMs", + [DISPLAY_TOT_HITM] = "Total HITMs", + [DISPLAY_SNP_PEER] = "Peer Snoop", }; static const struct option c2c_options[] = { @@ -137,23 +143,31 @@ static void *c2c_he_zalloc(size_t size) if (!c2c_he) return NULL; - c2c_he->cpuset = bitmap_alloc(c2c.cpus_cnt); + c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt); if (!c2c_he->cpuset) - return NULL; + goto out_free; - c2c_he->nodeset = bitmap_alloc(c2c.nodes_cnt); + c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt); if (!c2c_he->nodeset) - return NULL; + goto out_free; c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats)); if (!c2c_he->node_stats) - return NULL; + goto out_free; init_stats(&c2c_he->cstats.lcl_hitm); init_stats(&c2c_he->cstats.rmt_hitm); + init_stats(&c2c_he->cstats.lcl_peer); + init_stats(&c2c_he->cstats.rmt_peer); init_stats(&c2c_he->cstats.load); return &c2c_he->he; + +out_free: + zfree(&c2c_he->nodeset); + zfree(&c2c_he->cpuset); + free(c2c_he); + return NULL; } static void c2c_he_free(void *he) @@ -163,13 +177,13 @@ static void c2c_he_free(void *he) c2c_he = container_of(he, struct c2c_hist_entry, he); if (c2c_he->hists) { hists__delete_entries(&c2c_he->hists->hists); - free(c2c_he->hists); + zfree(&c2c_he->hists); } - free(c2c_he->cpuset); - free(c2c_he->nodeset); - free(c2c_he->nodestr); - free(c2c_he->node_stats); + zfree(&c2c_he->cpuset); + zfree(&c2c_he->nodeset); + zfree(&c2c_he->nodestr); + zfree(&c2c_he->node_stats); free(c2c_he); } @@ -215,7 +229,7 @@ static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he, "WARNING: no sample cpu value")) return; - set_bit(sample->cpu, c2c_he->cpuset); + __set_bit(sample->cpu, c2c_he->cpuset); } static void c2c_he__set_node(struct c2c_hist_entry *c2c_he, @@ -232,7 +246,7 @@ static void c2c_he__set_node(struct c2c_hist_entry *c2c_he, if (WARN_ONCE(node < 0, "WARNING: failed to find node\n")) return; - set_bit(node, c2c_he->nodeset); + __set_bit(node, c2c_he->nodeset); if (c2c_he->paddr != sample->phys_addr) { c2c_he->paddr_cnt++; @@ -250,6 +264,10 @@ static void compute_stats(struct c2c_hist_entry *c2c_he, update_stats(&cstats->rmt_hitm, weight); else if (stats->lcl_hitm) update_stats(&cstats->lcl_hitm, weight); + else if (stats->rmt_peer) + update_stats(&cstats->rmt_peer, weight); + else if (stats->lcl_peer) + update_stats(&cstats->lcl_peer, weight); else if (stats->load) update_stats(&cstats->load, weight); } @@ -266,25 +284,31 @@ static int process_sample_event(struct perf_tool *tool __maybe_unused, struct hist_entry *he; struct addr_location al; struct mem_info *mi, *mi_dup; + struct callchain_cursor *cursor; int ret; + addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { pr_debug("problem processing %d event, skipping it.\n", event->header.type); - return -1; + ret = -1; + goto out; } if (c2c.stitch_lbr) - al.thread->lbr_stitch_enable = true; + thread__set_lbr_stitch_enable(al.thread, true); - ret = sample__resolve_callchain(sample, &callchain_cursor, NULL, + cursor = get_tls_callchain_cursor(); + ret = sample__resolve_callchain(sample, cursor, NULL, evsel, &al, sysctl_perf_event_max_stack); if (ret) goto out; mi = sample__resolve_mem(sample, &al); - if (mi == NULL) - return -ENOMEM; + if (mi == NULL) { + ret = -ENOMEM; + goto out; + } /* * The mi object is released in hists__add_entry_ops, @@ -296,7 +320,7 @@ static int process_sample_event(struct perf_tool *tool __maybe_unused, c2c_decode_stats(&stats, mi); he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops, - &al, NULL, NULL, mi, + &al, NULL, NULL, mi, NULL, sample, true); if (he == NULL) goto free_mi; @@ -330,7 +354,7 @@ static int process_sample_event(struct perf_tool *tool __maybe_unused, goto free_mi; he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops, - &al, NULL, NULL, mi, + &al, NULL, NULL, mi, NULL, sample, true); if (he == NULL) goto free_mi; @@ -350,7 +374,7 @@ static int process_sample_event(struct perf_tool *tool __maybe_unused, } out: - addr_location__put(&al); + addr_location__exit(&al); return ret; free_mi: @@ -369,6 +393,10 @@ static struct perf_c2c c2c = { .exit = perf_event__process_exit, .fork = perf_event__process_fork, .lost = perf_event__process_lost, + .attr = perf_event__process_attr, + .auxtrace_info = perf_event__process_auxtrace_info, + .auxtrace = perf_event__process_auxtrace, + .auxtrace_error = perf_event__process_auxtrace_error, .ordered_events = true, .ordering_requires_timestamps = true, }, @@ -501,7 +529,7 @@ static int dcacheline_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, char buf[20]; if (he->mem_info) - addr = cl_address(he->mem_info->daddr.addr); + addr = cl_address(he->mem_info->daddr.addr, chk_double_cl); return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr)); } @@ -539,7 +567,7 @@ static int offset_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, char buf[20]; if (he->mem_info) - addr = cl_offset(he->mem_info->daddr.al_addr); + addr = cl_offset(he->mem_info->daddr.al_addr, chk_double_cl); return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr)); } @@ -551,9 +579,10 @@ offset_cmp(struct perf_hpp_fmt *fmt __maybe_unused, uint64_t l = 0, r = 0; if (left->mem_info) - l = cl_offset(left->mem_info->daddr.addr); + l = cl_offset(left->mem_info->daddr.addr, chk_double_cl); + if (right->mem_info) - r = cl_offset(right->mem_info->daddr.addr); + r = cl_offset(right->mem_info->daddr.addr, chk_double_cl); return (int64_t)(r - l); } @@ -643,75 +672,48 @@ __f ## _cmp(struct perf_hpp_fmt *fmt __maybe_unused, \ STAT_FN(rmt_hitm) STAT_FN(lcl_hitm) +STAT_FN(rmt_peer) +STAT_FN(lcl_peer) +STAT_FN(tot_peer) STAT_FN(store) STAT_FN(st_l1hit) STAT_FN(st_l1miss) +STAT_FN(st_na) STAT_FN(ld_fbhit) STAT_FN(ld_l1hit) STAT_FN(ld_l2hit) STAT_FN(ld_llchit) STAT_FN(rmt_hit) -static uint64_t llc_miss(struct c2c_stats *stats) +static uint64_t get_load_llc_misses(struct c2c_stats *stats) { - uint64_t llcmiss; - - llcmiss = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hitm + - stats->rmt_hit; - - return llcmiss; + return stats->lcl_dram + + stats->rmt_dram + + stats->rmt_hitm + + stats->rmt_hit; } -static int -ld_llcmiss_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, - struct hist_entry *he) +static uint64_t get_load_cache_hits(struct c2c_stats *stats) { - struct c2c_hist_entry *c2c_he; - int width = c2c_width(fmt, hpp, he->hists); - - c2c_he = container_of(he, struct c2c_hist_entry, he); - - return scnprintf(hpp->buf, hpp->size, "%*lu", width, - llc_miss(&c2c_he->stats)); + return stats->ld_fbhit + + stats->ld_l1hit + + stats->ld_l2hit + + stats->ld_llchit + + stats->lcl_hitm; } -static int64_t -ld_llcmiss_cmp(struct perf_hpp_fmt *fmt __maybe_unused, - struct hist_entry *left, struct hist_entry *right) +static uint64_t get_stores(struct c2c_stats *stats) { - struct c2c_hist_entry *c2c_left; - struct c2c_hist_entry *c2c_right; - - c2c_left = container_of(left, struct c2c_hist_entry, he); - c2c_right = container_of(right, struct c2c_hist_entry, he); - - return (uint64_t) llc_miss(&c2c_left->stats) - - (uint64_t) llc_miss(&c2c_right->stats); + return stats->st_l1hit + + stats->st_l1miss + + stats->st_na; } static uint64_t total_records(struct c2c_stats *stats) { - uint64_t lclmiss, ldcnt, total; - - lclmiss = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hitm + - stats->rmt_hit; - - ldcnt = lclmiss + - stats->ld_fbhit + - stats->ld_l1hit + - stats->ld_l2hit + - stats->ld_llchit + - stats->lcl_hitm; - - total = ldcnt + - stats->st_l1hit + - stats->st_l1miss; - - return total; + return get_load_llc_misses(stats) + + get_load_cache_hits(stats) + + get_stores(stats); } static int @@ -748,21 +750,8 @@ tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused, static uint64_t total_loads(struct c2c_stats *stats) { - uint64_t lclmiss, ldcnt; - - lclmiss = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hitm + - stats->rmt_hit; - - ldcnt = lclmiss + - stats->ld_fbhit + - stats->ld_l1hit + - stats->ld_l2hit + - stats->ld_llchit + - stats->lcl_hitm; - - return ldcnt; + return get_load_llc_misses(stats) + + get_load_cache_hits(stats); } static int @@ -817,7 +806,7 @@ percent_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, per); } -static double percent_hitm(struct c2c_hist_entry *c2c_he) +static double percent_costly_snoop(struct c2c_hist_entry *c2c_he) { struct c2c_hists *hists; struct c2c_stats *stats; @@ -830,17 +819,22 @@ static double percent_hitm(struct c2c_hist_entry *c2c_he) total = &hists->stats; switch (c2c.display) { - case DISPLAY_RMT: + case DISPLAY_RMT_HITM: st = stats->rmt_hitm; tot = total->rmt_hitm; break; - case DISPLAY_LCL: + case DISPLAY_LCL_HITM: st = stats->lcl_hitm; tot = total->lcl_hitm; break; - case DISPLAY_TOT: + case DISPLAY_TOT_HITM: st = stats->tot_hitm; tot = total->tot_hitm; + break; + case DISPLAY_SNP_PEER: + st = stats->tot_peer; + tot = total->tot_peer; + break; default: break; } @@ -857,8 +851,8 @@ static double percent_hitm(struct c2c_hist_entry *c2c_he) }) static int -percent_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, - struct hist_entry *he) +percent_costly_snoop_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) { struct c2c_hist_entry *c2c_he; int width = c2c_width(fmt, hpp, he->hists); @@ -866,20 +860,20 @@ percent_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, double per; c2c_he = container_of(he, struct c2c_hist_entry, he); - per = percent_hitm(c2c_he); + per = percent_costly_snoop(c2c_he); return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per)); } static int -percent_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, - struct hist_entry *he) +percent_costly_snoop_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) { - return percent_color(fmt, hpp, he, percent_hitm); + return percent_color(fmt, hpp, he, percent_costly_snoop); } static int64_t -percent_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused, - struct hist_entry *left, struct hist_entry *right) +percent_costly_snoop_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) { struct c2c_hist_entry *c2c_left; struct c2c_hist_entry *c2c_right; @@ -889,8 +883,8 @@ percent_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused, c2c_left = container_of(left, struct c2c_hist_entry, he); c2c_right = container_of(right, struct c2c_hist_entry, he); - per_left = percent_hitm(c2c_left); - per_right = percent_hitm(c2c_right); + per_left = percent_costly_snoop(c2c_left); + per_right = percent_costly_snoop(c2c_right); return per_left - per_right; } @@ -911,7 +905,7 @@ static struct c2c_stats *total_stats(struct hist_entry *he) return &hists->stats; } -static double percent(int st, int tot) +static double percent(u32 st, u32 tot) { return tot ? 100. * (double) st / (double) tot : 0; } @@ -929,8 +923,11 @@ static double percent_ ## __f(struct c2c_hist_entry *c2c_he) \ PERCENT_FN(rmt_hitm) PERCENT_FN(lcl_hitm) +PERCENT_FN(rmt_peer) +PERCENT_FN(lcl_peer) PERCENT_FN(st_l1hit) PERCENT_FN(st_l1miss) +PERCENT_FN(st_na) static int percent_rmt_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, @@ -957,8 +954,8 @@ percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused, double per_left; double per_right; - per_left = PERCENT(left, lcl_hitm); - per_right = PERCENT(right, lcl_hitm); + per_left = PERCENT(left, rmt_hitm); + per_right = PERCENT(right, rmt_hitm); return per_left - per_right; } @@ -995,6 +992,68 @@ percent_lcl_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused, } static int +percent_lcl_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + int width = c2c_width(fmt, hpp, he->hists); + double per = PERCENT(he, lcl_peer); + char buf[10]; + + return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per)); +} + +static int +percent_lcl_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + return percent_color(fmt, hpp, he, percent_lcl_peer); +} + +static int64_t +percent_lcl_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + double per_left; + double per_right; + + per_left = PERCENT(left, lcl_peer); + per_right = PERCENT(right, lcl_peer); + + return per_left - per_right; +} + +static int +percent_rmt_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + int width = c2c_width(fmt, hpp, he->hists); + double per = PERCENT(he, rmt_peer); + char buf[10]; + + return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per)); +} + +static int +percent_rmt_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + return percent_color(fmt, hpp, he, percent_rmt_peer); +} + +static int64_t +percent_rmt_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + double per_left; + double per_right; + + per_left = PERCENT(left, rmt_peer); + per_right = PERCENT(right, rmt_peer); + + return per_left - per_right; +} + +static int percent_stores_l1hit_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he) { @@ -1056,6 +1115,37 @@ percent_stores_l1miss_cmp(struct perf_hpp_fmt *fmt __maybe_unused, return per_left - per_right; } +static int +percent_stores_na_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + int width = c2c_width(fmt, hpp, he->hists); + double per = PERCENT(he, st_na); + char buf[10]; + + return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per)); +} + +static int +percent_stores_na_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + return percent_color(fmt, hpp, he, percent_st_na); +} + +static int64_t +percent_stores_na_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + double per_left; + double per_right; + + per_left = PERCENT(left, st_na); + per_right = PERCENT(right, st_na); + + return per_left - per_right; +} + STAT_FN(lcl_dram) STAT_FN(rmt_dram) @@ -1065,14 +1155,14 @@ pid_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, { int width = c2c_width(fmt, hpp, he->hists); - return scnprintf(hpp->buf, hpp->size, "%*d", width, he->thread->pid_); + return scnprintf(hpp->buf, hpp->size, "%*d", width, thread__pid(he->thread)); } static int64_t pid_cmp(struct perf_hpp_fmt *fmt __maybe_unused, struct hist_entry *left, struct hist_entry *right) { - return left->thread->pid_ - right->thread->pid_; + return thread__pid(left->thread) - thread__pid(right->thread); } static int64_t @@ -1083,6 +1173,19 @@ empty_cmp(struct perf_hpp_fmt *fmt __maybe_unused, return 0; } +static int display_metrics(struct perf_hpp *hpp, u32 val, u32 sum) +{ + int ret; + + if (sum != 0) + ret = scnprintf(hpp->buf, hpp->size, "%5.1f%% ", + percent(val, sum)); + else + ret = scnprintf(hpp->buf, hpp->size, "%6s ", "n/a"); + + return ret; +} + static int node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp, struct hist_entry *he) @@ -1100,7 +1203,7 @@ node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp, bitmap_zero(set, c2c.cpus_cnt); bitmap_and(set, c2c_he->cpuset, c2c.nodes[node], c2c.cpus_cnt); - if (!bitmap_weight(set, c2c.cpus_cnt)) { + if (bitmap_empty(set, c2c.cpus_cnt)) { if (c2c.node_info == 1) { ret = scnprintf(hpp->buf, hpp->size, "%21s", " "); advance_hpp(hpp, ret); @@ -1126,29 +1229,27 @@ node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp, ret = scnprintf(hpp->buf, hpp->size, "%2d{%2d ", node, num); advance_hpp(hpp, ret); - #define DISPLAY_HITM(__h) \ - if (c2c_he->stats.__h> 0) { \ - ret = scnprintf(hpp->buf, hpp->size, "%5.1f%% ", \ - percent(stats->__h, c2c_he->stats.__h));\ - } else { \ - ret = scnprintf(hpp->buf, hpp->size, "%6s ", "n/a"); \ - } - switch (c2c.display) { - case DISPLAY_RMT: - DISPLAY_HITM(rmt_hitm); + case DISPLAY_RMT_HITM: + ret = display_metrics(hpp, stats->rmt_hitm, + c2c_he->stats.rmt_hitm); + break; + case DISPLAY_LCL_HITM: + ret = display_metrics(hpp, stats->lcl_hitm, + c2c_he->stats.lcl_hitm); + break; + case DISPLAY_TOT_HITM: + ret = display_metrics(hpp, stats->tot_hitm, + c2c_he->stats.tot_hitm); break; - case DISPLAY_LCL: - DISPLAY_HITM(lcl_hitm); + case DISPLAY_SNP_PEER: + ret = display_metrics(hpp, stats->tot_peer, + c2c_he->stats.tot_peer); break; - case DISPLAY_TOT: - DISPLAY_HITM(tot_hitm); default: break; } - #undef DISPLAY_HITM - advance_hpp(hpp, ret); if (c2c_he->stats.store > 0) { @@ -1204,6 +1305,8 @@ __func(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he) \ MEAN_ENTRY(mean_rmt_entry, rmt_hitm); MEAN_ENTRY(mean_lcl_entry, lcl_hitm); MEAN_ENTRY(mean_load_entry, load); +MEAN_ENTRY(mean_rmt_peer_entry, rmt_peer); +MEAN_ENTRY(mean_lcl_peer_entry, lcl_peer); static int cpucnt_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, @@ -1328,7 +1431,7 @@ static struct c2c_dimension dim_iaddr = { }; static struct c2c_dimension dim_tot_hitm = { - .header = HEADER_SPAN("----- LLC Load Hitm -----", "Total", 2), + .header = HEADER_SPAN("------- Load Hitm -------", "Total", 2), .name = "tot_hitm", .cmp = tot_hitm_cmp, .entry = tot_hitm_entry, @@ -1336,7 +1439,7 @@ static struct c2c_dimension dim_tot_hitm = { }; static struct c2c_dimension dim_lcl_hitm = { - .header = HEADER_SPAN_LOW("Lcl"), + .header = HEADER_SPAN_LOW("LclHitm"), .name = "lcl_hitm", .cmp = lcl_hitm_cmp, .entry = lcl_hitm_entry, @@ -1344,13 +1447,37 @@ static struct c2c_dimension dim_lcl_hitm = { }; static struct c2c_dimension dim_rmt_hitm = { - .header = HEADER_SPAN_LOW("Rmt"), + .header = HEADER_SPAN_LOW("RmtHitm"), .name = "rmt_hitm", .cmp = rmt_hitm_cmp, .entry = rmt_hitm_entry, .width = 7, }; +static struct c2c_dimension dim_tot_peer = { + .header = HEADER_SPAN("------- Load Peer -------", "Total", 2), + .name = "tot_peer", + .cmp = tot_peer_cmp, + .entry = tot_peer_entry, + .width = 7, +}; + +static struct c2c_dimension dim_lcl_peer = { + .header = HEADER_SPAN_LOW("Local"), + .name = "lcl_peer", + .cmp = lcl_peer_cmp, + .entry = lcl_peer_entry, + .width = 7, +}; + +static struct c2c_dimension dim_rmt_peer = { + .header = HEADER_SPAN_LOW("Remote"), + .name = "rmt_peer", + .cmp = rmt_peer_cmp, + .entry = rmt_peer_entry, + .width = 7, +}; + static struct c2c_dimension dim_cl_rmt_hitm = { .header = HEADER_SPAN("----- HITM -----", "Rmt", 1), .name = "cl_rmt_hitm", @@ -1367,16 +1494,32 @@ static struct c2c_dimension dim_cl_lcl_hitm = { .width = 7, }; -static struct c2c_dimension dim_stores = { - .header = HEADER_SPAN("---- Store Reference ----", "Total", 2), - .name = "stores", +static struct c2c_dimension dim_cl_rmt_peer = { + .header = HEADER_SPAN("----- Peer -----", "Rmt", 1), + .name = "cl_rmt_peer", + .cmp = rmt_peer_cmp, + .entry = rmt_peer_entry, + .width = 7, +}; + +static struct c2c_dimension dim_cl_lcl_peer = { + .header = HEADER_SPAN_LOW("Lcl"), + .name = "cl_lcl_peer", + .cmp = lcl_peer_cmp, + .entry = lcl_peer_entry, + .width = 7, +}; + +static struct c2c_dimension dim_tot_stores = { + .header = HEADER_BOTH("Total", "Stores"), + .name = "tot_stores", .cmp = store_cmp, .entry = store_entry, .width = 7, }; static struct c2c_dimension dim_stores_l1hit = { - .header = HEADER_SPAN_LOW("L1Hit"), + .header = HEADER_SPAN("--------- Stores --------", "L1Hit", 2), .name = "stores_l1hit", .cmp = st_l1hit_cmp, .entry = st_l1hit_entry, @@ -1391,8 +1534,16 @@ static struct c2c_dimension dim_stores_l1miss = { .width = 7, }; +static struct c2c_dimension dim_stores_na = { + .header = HEADER_SPAN_LOW("N/A"), + .name = "stores_na", + .cmp = st_na_cmp, + .entry = st_na_entry, + .width = 7, +}; + static struct c2c_dimension dim_cl_stores_l1hit = { - .header = HEADER_SPAN("-- Store Refs --", "L1 Hit", 1), + .header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2), .name = "cl_stores_l1hit", .cmp = st_l1hit_cmp, .entry = st_l1hit_entry, @@ -1407,6 +1558,14 @@ static struct c2c_dimension dim_cl_stores_l1miss = { .width = 7, }; +static struct c2c_dimension dim_cl_stores_na = { + .header = HEADER_SPAN_LOW("N/A"), + .name = "cl_stores_na", + .cmp = st_na_cmp, + .entry = st_na_entry, + .width = 7, +}; + static struct c2c_dimension dim_ld_fbhit = { .header = HEADER_SPAN("----- Core Load Hit -----", "FB", 2), .name = "ld_fbhit", @@ -1432,7 +1591,7 @@ static struct c2c_dimension dim_ld_l2hit = { }; static struct c2c_dimension dim_ld_llchit = { - .header = HEADER_SPAN("-- LLC Load Hit --", "Llc", 1), + .header = HEADER_SPAN("- LLC Load Hit --", "LclHit", 1), .name = "ld_lclhit", .cmp = ld_llchit_cmp, .entry = ld_llchit_entry, @@ -1440,21 +1599,13 @@ static struct c2c_dimension dim_ld_llchit = { }; static struct c2c_dimension dim_ld_rmthit = { - .header = HEADER_SPAN_LOW("Rmt"), + .header = HEADER_SPAN("- RMT Load Hit --", "RmtHit", 1), .name = "ld_rmthit", .cmp = rmt_hit_cmp, .entry = rmt_hit_entry, .width = 8, }; -static struct c2c_dimension dim_ld_llcmiss = { - .header = HEADER_BOTH("LLC", "Ld Miss"), - .name = "ld_llcmiss", - .cmp = ld_llcmiss_cmp, - .entry = ld_llcmiss_entry, - .width = 7, -}; - static struct c2c_dimension dim_tot_recs = { .header = HEADER_BOTH("Total", "records"), .name = "tot_recs", @@ -1471,22 +1622,23 @@ static struct c2c_dimension dim_tot_loads = { .width = 7, }; -static struct c2c_header percent_hitm_header[] = { - [DISPLAY_LCL] = HEADER_BOTH("Lcl", "Hitm"), - [DISPLAY_RMT] = HEADER_BOTH("Rmt", "Hitm"), - [DISPLAY_TOT] = HEADER_BOTH("Tot", "Hitm"), +static struct c2c_header percent_costly_snoop_header[] = { + [DISPLAY_LCL_HITM] = HEADER_BOTH("Lcl", "Hitm"), + [DISPLAY_RMT_HITM] = HEADER_BOTH("Rmt", "Hitm"), + [DISPLAY_TOT_HITM] = HEADER_BOTH("Tot", "Hitm"), + [DISPLAY_SNP_PEER] = HEADER_BOTH("Peer", "Snoop"), }; -static struct c2c_dimension dim_percent_hitm = { - .name = "percent_hitm", - .cmp = percent_hitm_cmp, - .entry = percent_hitm_entry, - .color = percent_hitm_color, +static struct c2c_dimension dim_percent_costly_snoop = { + .name = "percent_costly_snoop", + .cmp = percent_costly_snoop_cmp, + .entry = percent_costly_snoop_entry, + .color = percent_costly_snoop_color, .width = 7, }; static struct c2c_dimension dim_percent_rmt_hitm = { - .header = HEADER_SPAN("----- HITM -----", "Rmt", 1), + .header = HEADER_SPAN("----- HITM -----", "RmtHitm", 1), .name = "percent_rmt_hitm", .cmp = percent_rmt_hitm_cmp, .entry = percent_rmt_hitm_entry, @@ -1495,7 +1647,7 @@ static struct c2c_dimension dim_percent_rmt_hitm = { }; static struct c2c_dimension dim_percent_lcl_hitm = { - .header = HEADER_SPAN_LOW("Lcl"), + .header = HEADER_SPAN_LOW("LclHitm"), .name = "percent_lcl_hitm", .cmp = percent_lcl_hitm_cmp, .entry = percent_lcl_hitm_entry, @@ -1503,8 +1655,26 @@ static struct c2c_dimension dim_percent_lcl_hitm = { .width = 7, }; +static struct c2c_dimension dim_percent_rmt_peer = { + .header = HEADER_SPAN("-- Peer Snoop --", "Rmt", 1), + .name = "percent_rmt_peer", + .cmp = percent_rmt_peer_cmp, + .entry = percent_rmt_peer_entry, + .color = percent_rmt_peer_color, + .width = 7, +}; + +static struct c2c_dimension dim_percent_lcl_peer = { + .header = HEADER_SPAN_LOW("Lcl"), + .name = "percent_lcl_peer", + .cmp = percent_lcl_peer_cmp, + .entry = percent_lcl_peer_entry, + .color = percent_lcl_peer_color, + .width = 7, +}; + static struct c2c_dimension dim_percent_stores_l1hit = { - .header = HEADER_SPAN("-- Store Refs --", "L1 Hit", 1), + .header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2), .name = "percent_stores_l1hit", .cmp = percent_stores_l1hit_cmp, .entry = percent_stores_l1hit_entry, @@ -1521,6 +1691,15 @@ static struct c2c_dimension dim_percent_stores_l1miss = { .width = 7, }; +static struct c2c_dimension dim_percent_stores_na = { + .header = HEADER_SPAN_LOW("N/A"), + .name = "percent_stores_na", + .cmp = percent_stores_na_cmp, + .entry = percent_stores_na_entry, + .color = percent_stores_na_color, + .width = 7, +}; + static struct c2c_dimension dim_dram_lcl = { .header = HEADER_SPAN("--- Load Dram ----", "Lcl", 1), .name = "dram_lcl", @@ -1562,12 +1741,6 @@ static struct c2c_dimension dim_dso = { .se = &sort_dso, }; -static struct c2c_header header_node[3] = { - HEADER_LOW("Node"), - HEADER_LOW("Node{cpus %hitms %stores}"), - HEADER_LOW("Node{cpu list}"), -}; - static struct c2c_dimension dim_node = { .name = "node", .cmp = empty_cmp, @@ -1599,6 +1772,22 @@ static struct c2c_dimension dim_mean_load = { .width = 8, }; +static struct c2c_dimension dim_mean_rmt_peer = { + .header = HEADER_SPAN("---------- cycles ----------", "rmt peer", 2), + .name = "mean_rmt_peer", + .cmp = empty_cmp, + .entry = mean_rmt_peer_entry, + .width = 8, +}; + +static struct c2c_dimension dim_mean_lcl_peer = { + .header = HEADER_SPAN_LOW("lcl peer"), + .name = "mean_lcl_peer", + .cmp = empty_cmp, + .entry = mean_lcl_peer_entry, + .width = 8, +}; + static struct c2c_dimension dim_cpucnt = { .header = HEADER_BOTH("cpu", "cnt"), .name = "cpucnt", @@ -1646,26 +1835,35 @@ static struct c2c_dimension *dimensions[] = { &dim_tot_hitm, &dim_lcl_hitm, &dim_rmt_hitm, + &dim_tot_peer, + &dim_lcl_peer, + &dim_rmt_peer, &dim_cl_lcl_hitm, &dim_cl_rmt_hitm, - &dim_stores, + &dim_cl_lcl_peer, + &dim_cl_rmt_peer, + &dim_tot_stores, &dim_stores_l1hit, &dim_stores_l1miss, + &dim_stores_na, &dim_cl_stores_l1hit, &dim_cl_stores_l1miss, + &dim_cl_stores_na, &dim_ld_fbhit, &dim_ld_l1hit, &dim_ld_l2hit, &dim_ld_llchit, &dim_ld_rmthit, - &dim_ld_llcmiss, &dim_tot_recs, &dim_tot_loads, - &dim_percent_hitm, + &dim_percent_costly_snoop, &dim_percent_rmt_hitm, &dim_percent_lcl_hitm, + &dim_percent_rmt_peer, + &dim_percent_lcl_peer, &dim_percent_stores_l1hit, &dim_percent_stores_l1miss, + &dim_percent_stores_na, &dim_dram_lcl, &dim_dram_rmt, &dim_pid, @@ -1675,6 +1873,8 @@ static struct c2c_dimension *dimensions[] = { &dim_node, &dim_mean_rmt, &dim_mean_lcl, + &dim_mean_rmt_peer, + &dim_mean_lcl_peer, &dim_mean_load, &dim_cpucnt, &dim_srcline, @@ -1895,53 +2095,75 @@ static int c2c_hists__reinit(struct c2c_hists *c2c_hists, #define DISPLAY_LINE_LIMIT 0.001 +static u8 filter_display(u32 val, u32 sum) +{ + if (sum == 0 || ((double)val / sum) < DISPLAY_LINE_LIMIT) + return HIST_FILTER__C2C; + + return 0; +} + static bool he__display(struct hist_entry *he, struct c2c_stats *stats) { struct c2c_hist_entry *c2c_he; - double ld_dist; if (c2c.show_all) return true; c2c_he = container_of(he, struct c2c_hist_entry, he); -#define FILTER_HITM(__h) \ - if (stats->__h) { \ - ld_dist = ((double)c2c_he->stats.__h / stats->__h); \ - if (ld_dist < DISPLAY_LINE_LIMIT) \ - he->filtered = HIST_FILTER__C2C; \ - } else { \ - he->filtered = HIST_FILTER__C2C; \ - } - switch (c2c.display) { - case DISPLAY_LCL: - FILTER_HITM(lcl_hitm); + case DISPLAY_LCL_HITM: + he->filtered = filter_display(c2c_he->stats.lcl_hitm, + stats->lcl_hitm); break; - case DISPLAY_RMT: - FILTER_HITM(rmt_hitm); + case DISPLAY_RMT_HITM: + he->filtered = filter_display(c2c_he->stats.rmt_hitm, + stats->rmt_hitm); + break; + case DISPLAY_TOT_HITM: + he->filtered = filter_display(c2c_he->stats.tot_hitm, + stats->tot_hitm); + break; + case DISPLAY_SNP_PEER: + he->filtered = filter_display(c2c_he->stats.tot_peer, + stats->tot_peer); break; - case DISPLAY_TOT: - FILTER_HITM(tot_hitm); default: break; } -#undef FILTER_HITM - return he->filtered == 0; } -static inline int valid_hitm_or_store(struct hist_entry *he) +static inline bool is_valid_hist_entry(struct hist_entry *he) { struct c2c_hist_entry *c2c_he; - bool has_hitm; + bool has_record = false; c2c_he = container_of(he, struct c2c_hist_entry, he); - has_hitm = c2c.display == DISPLAY_TOT ? c2c_he->stats.tot_hitm : - c2c.display == DISPLAY_LCL ? c2c_he->stats.lcl_hitm : - c2c_he->stats.rmt_hitm; - return has_hitm || c2c_he->stats.store; + + /* It's a valid entry if contains stores */ + if (c2c_he->stats.store) + return true; + + switch (c2c.display) { + case DISPLAY_LCL_HITM: + has_record = !!c2c_he->stats.lcl_hitm; + break; + case DISPLAY_RMT_HITM: + has_record = !!c2c_he->stats.rmt_hitm; + break; + case DISPLAY_TOT_HITM: + has_record = !!c2c_he->stats.tot_hitm; + break; + case DISPLAY_SNP_PEER: + has_record = !!c2c_he->stats.tot_peer; + default: + break; + } + + return has_record; } static void set_node_width(struct c2c_hist_entry *c2c_he, int len) @@ -1963,7 +2185,7 @@ static int set_nodestr(struct c2c_hist_entry *c2c_he) if (c2c_he->nodestr) return 0; - if (bitmap_weight(c2c_he->nodeset, c2c.nodes_cnt)) { + if (!bitmap_empty(c2c_he->nodeset, c2c.nodes_cnt)) { len = bitmap_scnprintf(c2c_he->nodeset, c2c.nodes_cnt, buf, sizeof(buf)); } else { @@ -1995,7 +2217,7 @@ static int filter_cb(struct hist_entry *he, void *arg __maybe_unused) calc_width(c2c_he); - if (!valid_hitm_or_store(he)) + if (!is_valid_hist_entry(he)) he->filtered = HIST_FILTER__C2C; return 0; @@ -2005,7 +2227,7 @@ static int resort_cl_cb(struct hist_entry *he, void *arg __maybe_unused) { struct c2c_hist_entry *c2c_he; struct c2c_hists *c2c_hists; - bool display = he__display(he, &c2c.hitm_stats); + bool display = he__display(he, &c2c.shared_clines_stats); c2c_he = container_of(he, struct c2c_hist_entry, he); c2c_hists = c2c_he->hists; @@ -2025,16 +2247,41 @@ static int resort_cl_cb(struct hist_entry *he, void *arg __maybe_unused) return 0; } +static struct c2c_header header_node_0 = HEADER_LOW("Node"); +static struct c2c_header header_node_1_hitms_stores = + HEADER_LOW("Node{cpus %hitms %stores}"); +static struct c2c_header header_node_1_peers_stores = + HEADER_LOW("Node{cpus %peers %stores}"); +static struct c2c_header header_node_2 = HEADER_LOW("Node{cpu list}"); + static void setup_nodes_header(void) { - dim_node.header = header_node[c2c.node_info]; + switch (c2c.node_info) { + case 0: + dim_node.header = header_node_0; + break; + case 1: + if (c2c.display == DISPLAY_SNP_PEER) + dim_node.header = header_node_1_peers_stores; + else + dim_node.header = header_node_1_hitms_stores; + break; + case 2: + dim_node.header = header_node_2; + break; + default: + break; + } + + return; } static int setup_nodes(struct perf_session *session) { struct numa_node *n; unsigned long **nodes; - int node, cpu; + int node, idx; + struct perf_cpu cpu; int *cpu2node; if (c2c.node_info > 2) @@ -2057,8 +2304,8 @@ static int setup_nodes(struct perf_session *session) if (!cpu2node) return -ENOMEM; - for (cpu = 0; cpu < c2c.cpus_cnt; cpu++) - cpu2node[cpu] = -1; + for (idx = 0; idx < c2c.cpus_cnt; idx++) + cpu2node[idx] = -1; c2c.cpu2node = cpu2node; @@ -2066,23 +2313,23 @@ static int setup_nodes(struct perf_session *session) struct perf_cpu_map *map = n[node].map; unsigned long *set; - set = bitmap_alloc(c2c.cpus_cnt); + set = bitmap_zalloc(c2c.cpus_cnt); if (!set) return -ENOMEM; nodes[node] = set; /* empty node, skip */ - if (perf_cpu_map__empty(map)) + if (perf_cpu_map__has_any_cpu_or_is_empty(map)) continue; - for (cpu = 0; cpu < map->nr; cpu++) { - set_bit(map->map[cpu], set); + perf_cpu_map__for_each_cpu(cpu, idx, map) { + __set_bit(cpu.cpu, set); - if (WARN_ONCE(cpu2node[map->map[cpu]] != -1, "node/cpu topology bug")) + if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug")) return -EINVAL; - cpu2node[map->map[cpu]] = node; + cpu2node[cpu.cpu] = node; } } @@ -2091,15 +2338,16 @@ static int setup_nodes(struct perf_session *session) } #define HAS_HITMS(__h) ((__h)->stats.lcl_hitm || (__h)->stats.rmt_hitm) +#define HAS_PEER(__h) ((__h)->stats.lcl_peer || (__h)->stats.rmt_peer) -static int resort_hitm_cb(struct hist_entry *he, void *arg __maybe_unused) +static int resort_shared_cl_cb(struct hist_entry *he, void *arg __maybe_unused) { struct c2c_hist_entry *c2c_he; c2c_he = container_of(he, struct c2c_hist_entry, he); - if (HAS_HITMS(c2c_he)) { + if (HAS_HITMS(c2c_he) || HAS_PEER(c2c_he)) { c2c.shared_clines++; - c2c_add_stats(&c2c.hitm_stats, &c2c_he->stats); + c2c_add_stats(&c2c.shared_clines_stats, &c2c_he->stats); } return 0; @@ -2128,10 +2376,7 @@ static void print_c2c__display_stats(FILE *out) int llc_misses; struct c2c_stats *stats = &c2c.hists.stats; - llc_misses = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hit + - stats->rmt_hitm; + llc_misses = get_load_llc_misses(stats); fprintf(out, "=================================================\n"); fprintf(out, " Trace Event Information \n"); @@ -2155,6 +2400,10 @@ static void print_c2c__display_stats(FILE *out) fprintf(out, " Load MESI State Exclusive : %10d\n", stats->ld_excl); fprintf(out, " Load MESI State Shared : %10d\n", stats->ld_shared); fprintf(out, " Load LLC Misses : %10d\n", llc_misses); + fprintf(out, " Load access blocked by data : %10d\n", stats->blk_data); + fprintf(out, " Load access blocked by address : %10d\n", stats->blk_addr); + fprintf(out, " Load HIT Local Peer : %10d\n", stats->lcl_peer); + fprintf(out, " Load HIT Remote Peer : %10d\n", stats->rmt_peer); fprintf(out, " LLC Misses to Local DRAM : %10.1f%%\n", ((double)stats->lcl_dram/(double)llc_misses) * 100.); fprintf(out, " LLC Misses to Remote DRAM : %10.1f%%\n", ((double)stats->rmt_dram/(double)llc_misses) * 100.); fprintf(out, " LLC Misses to Remote cache (HIT) : %10.1f%%\n", ((double)stats->rmt_hit /(double)llc_misses) * 100.); @@ -2164,13 +2413,14 @@ static void print_c2c__display_stats(FILE *out) fprintf(out, " Store - no mapping : %10d\n", stats->st_noadrs); fprintf(out, " Store L1D Hit : %10d\n", stats->st_l1hit); fprintf(out, " Store L1D Miss : %10d\n", stats->st_l1miss); + fprintf(out, " Store No available memory level : %10d\n", stats->st_na); fprintf(out, " No Page Map Rejects : %10d\n", stats->nomap); fprintf(out, " Unable to parse data source : %10d\n", stats->noparse); } static void print_shared_cacheline_info(FILE *out) { - struct c2c_stats *stats = &c2c.hitm_stats; + struct c2c_stats *stats = &c2c.shared_clines_stats; int hitm_cnt = stats->lcl_hitm + stats->rmt_hitm; fprintf(out, "=================================================\n"); @@ -2182,9 +2432,12 @@ static void print_shared_cacheline_info(FILE *out) fprintf(out, " L1D hits on shared lines : %10d\n", stats->ld_l1hit); fprintf(out, " L2D hits on shared lines : %10d\n", stats->ld_l2hit); fprintf(out, " LLC hits on shared lines : %10d\n", stats->ld_llchit + stats->lcl_hitm); + fprintf(out, " Load hits on peer cache or nodes : %10d\n", stats->lcl_peer + stats->rmt_peer); fprintf(out, " Locked Access on shared lines : %10d\n", stats->locks); + fprintf(out, " Blocked Access on shared lines : %10d\n", stats->blk_data + stats->blk_addr); fprintf(out, " Store HITs on shared lines : %10d\n", stats->store); fprintf(out, " Store L1D hits on shared lines : %10d\n", stats->st_l1hit); + fprintf(out, " Store No available memory level : %10d\n", stats->st_na); fprintf(out, " Total Merged records : %10d\n", hitm_cnt + stats->store); } @@ -2207,10 +2460,10 @@ static void print_cacheline(struct c2c_hists *c2c_hists, fprintf(out, "\n"); } - fprintf(out, " -------------------------------------------------------------\n"); + fprintf(out, " ----------------------------------------------------------------------\n"); __hist_entry__snprintf(he_cl, &hpp, hpp_list); fprintf(out, "%s\n", bf); - fprintf(out, " -------------------------------------------------------------\n"); + fprintf(out, " ----------------------------------------------------------------------\n"); hists__fprintf(&c2c_hists->hists, false, 0, 0, 0, out, false); } @@ -2220,16 +2473,27 @@ static void print_pareto(FILE *out) struct perf_hpp_list hpp_list; struct rb_node *nd; int ret; + const char *cl_output; + + if (c2c.display != DISPLAY_SNP_PEER) + cl_output = "cl_num," + "cl_rmt_hitm," + "cl_lcl_hitm," + "cl_stores_l1hit," + "cl_stores_l1miss," + "cl_stores_na," + "dcacheline"; + else + cl_output = "cl_num," + "cl_rmt_peer," + "cl_lcl_peer," + "cl_stores_l1hit," + "cl_stores_l1miss," + "cl_stores_na," + "dcacheline"; perf_hpp_list__init(&hpp_list); - ret = hpp_list__parse(&hpp_list, - "cl_num," - "cl_rmt_hitm," - "cl_lcl_hitm," - "cl_stores_l1hit," - "cl_stores_l1miss," - "dcacheline", - NULL); + ret = hpp_list__parse(&hpp_list, cl_output, NULL); if (WARN_ONCE(ret, "failed to setup sort entries\n")) return; @@ -2262,7 +2526,7 @@ static void print_c2c_info(FILE *out, struct perf_session *session) fprintf(out, "%-36s: %s\n", first ? " Events" : "", evsel__name(evsel)); first = false; } - fprintf(out, " Cachelines sort on : %s HITMs\n", + fprintf(out, " Cachelines sort on : %s\n", display_str[c2c.display]); fprintf(out, " Cacheline data grouping : %s\n", c2c.cl_sort); } @@ -2332,7 +2596,7 @@ perf_c2c_cacheline_browser__title(struct hist_browser *browser, he = cl_browser->he; if (he->mem_info) - addr = cl_address(he->mem_info->daddr.addr); + addr = cl_address(he->mem_info->daddr.addr, chk_double_cl); scnprintf(bf, size, "Cacheline 0x%lx", addr); return 0; @@ -2419,7 +2683,7 @@ static int perf_c2c_browser__title(struct hist_browser *browser, { scnprintf(bf, size, "Shared Data Cache Line Table " - "(%lu entries, sorted on %s HITMs)", + "(%lu entries, sorted on %s)", browser->nr_non_filtered_entries, display_str[c2c.display]); return 0; @@ -2530,15 +2794,16 @@ static int ui_quirks(void) if (!c2c.use_stdio) { dim_offset.width = 5; dim_offset.header = header_offset_tui; - nodestr = "CL"; + nodestr = chk_double_cl ? "Double-CL" : "CL"; } - dim_percent_hitm.header = percent_hitm_header[c2c.display]; + dim_percent_costly_snoop.header = percent_costly_snoop_header[c2c.display]; /* Fix the zero line for dcacheline column. */ - buf = fill_line("Cacheline", dim_dcacheline.width + - dim_dcacheline_node.width + - dim_dcacheline_count.width + 4); + buf = fill_line(chk_double_cl ? "Double-Cacheline" : "Cacheline", + dim_dcacheline.width + + dim_dcacheline_node.width + + dim_dcacheline_count.width + 4); if (!buf) return -ENOMEM; @@ -2582,7 +2847,7 @@ parse_callchain_opt(const struct option *opt, const char *arg, int unset) static int setup_callchain(struct evlist *evlist) { - u64 sample_type = perf_evlist__combined_sample_type(evlist); + u64 sample_type = evlist__combined_sample_type(evlist); enum perf_call_graph_mode mode = CALLCHAIN_NONE; if ((sample_type & PERF_SAMPLE_REGS_USER) && @@ -2617,14 +2882,16 @@ static int setup_callchain(struct evlist *evlist) static int setup_display(const char *str) { - const char *display = str ?: "tot"; + const char *display = str; if (!strcmp(display, "tot")) - c2c.display = DISPLAY_TOT; + c2c.display = DISPLAY_TOT_HITM; else if (!strcmp(display, "rmt")) - c2c.display = DISPLAY_RMT; + c2c.display = DISPLAY_RMT_HITM; else if (!strcmp(display, "lcl")) - c2c.display = DISPLAY_LCL; + c2c.display = DISPLAY_LCL_HITM; + else if (!strcmp(display, "peer")) + c2c.display = DISPLAY_SNP_PEER; else { pr_err("failed: unknown display type: %s\n", str); return -1; @@ -2671,18 +2938,23 @@ static int build_cl_output(char *cl_sort, bool no_source) } if (asprintf(&c2c.cl_output, - "%s%s%s%s%s%s%s%s%s%s", + "%s%s%s%s%s%s%s%s%s%s%s%s", c2c.use_stdio ? "cl_num_empty," : "", - "percent_rmt_hitm," - "percent_lcl_hitm," + c2c.display == DISPLAY_SNP_PEER ? "percent_rmt_peer," + "percent_lcl_peer," : + "percent_rmt_hitm," + "percent_lcl_hitm,", "percent_stores_l1hit," "percent_stores_l1miss," + "percent_stores_na," "offset,offset_node,dcacheline_count,", add_pid ? "pid," : "", add_tid ? "tid," : "", add_iaddr ? "iaddr," : "", - "mean_rmt," - "mean_lcl," + c2c.display == DISPLAY_SNP_PEER ? "mean_rmt_peer," + "mean_lcl_peer," : + "mean_rmt," + "mean_lcl,", "mean_load," "tot_recs," "cpucnt,", @@ -2703,6 +2975,7 @@ err: static int setup_coalesce(const char *coalesce, bool no_source) { const char *c = coalesce ?: coalesce_default; + const char *sort_str = NULL; if (asprintf(&c2c.cl_sort, "offset,%s", c) < 0) return -ENOMEM; @@ -2710,12 +2983,16 @@ static int setup_coalesce(const char *coalesce, bool no_source) if (build_cl_output(c2c.cl_sort, no_source)) return -1; - if (asprintf(&c2c.cl_resort, "offset,%s", - c2c.display == DISPLAY_TOT ? - "tot_hitm" : - c2c.display == DISPLAY_RMT ? - "rmt_hitm,lcl_hitm" : - "lcl_hitm,rmt_hitm") < 0) + if (c2c.display == DISPLAY_TOT_HITM) + sort_str = "tot_hitm"; + else if (c2c.display == DISPLAY_RMT_HITM) + sort_str = "rmt_hitm,lcl_hitm"; + else if (c2c.display == DISPLAY_LCL_HITM) + sort_str = "lcl_hitm,rmt_hitm"; + else if (c2c.display == DISPLAY_SNP_PEER) + sort_str = "tot_peer"; + + if (asprintf(&c2c.cl_resort, "offset,%s", sort_str) < 0) return -ENOMEM; pr_debug("coalesce sort fields: %s\n", c2c.cl_sort); @@ -2726,6 +3003,12 @@ static int setup_coalesce(const char *coalesce, bool no_source) static int perf_c2c__report(int argc, const char **argv) { + struct itrace_synth_opts itrace_synth_opts = { + .set = true, + .mem = true, /* Only enable memory event */ + .default_no_sample = true, + }; + struct perf_session *session; struct ui_progress prog; struct perf_data data = { @@ -2742,9 +3025,7 @@ static int perf_c2c__report(int argc, const char **argv) "the input file to process"), OPT_INCR('N', "node-info", &c2c.node_info, "show extra node info in report (repeat for more info)"), -#ifdef HAVE_SLANG_SUPPORT OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"), -#endif OPT_BOOLEAN(0, "stats", &c2c.stats_only, "Display only statistic tables (implies --stdio)"), OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full, @@ -2757,58 +3038,82 @@ static int perf_c2c__report(int argc, const char **argv) "print_type,threshold[,print_limit],order,sort_key[,branch],value", callchain_help, &parse_callchain_opt, callchain_default_opt), - OPT_STRING('d', "display", &display, "Switch HITM output type", "lcl,rmt"), + OPT_STRING('d', "display", &display, "Switch HITM output type", "tot,lcl,rmt,peer"), OPT_STRING('c', "coalesce", &coalesce, "coalesce fields", "coalesce fields: pid,tid,iaddr,dso"), OPT_BOOLEAN('f', "force", &symbol_conf.force, "don't complain, do it"), OPT_BOOLEAN(0, "stitch-lbr", &c2c.stitch_lbr, "Enable LBR callgraph stitching approach"), + OPT_BOOLEAN(0, "double-cl", &chk_double_cl, "Detect adjacent cacheline false sharing"), OPT_PARENT(c2c_options), OPT_END() }; int err = 0; + const char *output_str, *sort_str = NULL; argc = parse_options(argc, argv, options, report_c2c_usage, PARSE_OPT_STOP_AT_NON_OPTION); if (argc) usage_with_options(report_c2c_usage, options); +#ifndef HAVE_SLANG_SUPPORT + c2c.use_stdio = true; +#endif + if (c2c.stats_only) c2c.use_stdio = true; + err = symbol__validate_sym_arguments(); + if (err) + goto out; + if (!input_name || !strlen(input_name)) input_name = "perf.data"; data.path = input_name; data.force = symbol_conf.force; + session = perf_session__new(&data, &c2c.tool); + if (IS_ERR(session)) { + err = PTR_ERR(session); + pr_debug("Error creating perf session\n"); + goto out; + } + + /* + * Use the 'tot' as default display type if user doesn't specify it; + * since Arm64 platform doesn't support HITMs flag, use 'peer' as the + * default display type. + */ + if (!display) { + if (!strcmp(perf_env__arch(&session->header.env), "arm64")) + display = "peer"; + else + display = "tot"; + } + err = setup_display(display); if (err) - goto out; + goto out_session; err = setup_coalesce(coalesce, no_source); if (err) { pr_debug("Failed to initialize hists\n"); - goto out; + goto out_session; } err = c2c_hists__init(&c2c.hists, "dcacheline", 2); if (err) { pr_debug("Failed to initialize hists\n"); - goto out; + goto out_session; } - session = perf_session__new(&data, 0, &c2c.tool); - if (IS_ERR(session)) { - err = PTR_ERR(session); - pr_debug("Error creating perf session\n"); - goto out; - } + session->itrace_synth_opts = &itrace_synth_opts; err = setup_nodes(session); if (err) { pr_err("Failed setup nodes\n"); - goto out; + goto out_session; } err = mem2node__init(&c2c.mem2node, &session->header.env); @@ -2841,28 +3146,52 @@ static int perf_c2c__report(int argc, const char **argv) goto out_mem2node; } - c2c_hists__reinit(&c2c.hists, - "cl_idx," - "dcacheline," - "dcacheline_node," - "dcacheline_count," - "tot_recs," - "percent_hitm," - "tot_hitm,lcl_hitm,rmt_hitm," - "stores,stores_l1hit,stores_l1miss," - "dram_lcl,dram_rmt," - "ld_llcmiss," - "tot_loads," - "ld_fbhit,ld_l1hit,ld_l2hit," - "ld_lclhit,ld_rmthit", - c2c.display == DISPLAY_TOT ? "tot_hitm" : - c2c.display == DISPLAY_LCL ? "lcl_hitm" : "rmt_hitm" - ); + if (c2c.display != DISPLAY_SNP_PEER) + output_str = "cl_idx," + "dcacheline," + "dcacheline_node," + "dcacheline_count," + "percent_costly_snoop," + "tot_hitm,lcl_hitm,rmt_hitm," + "tot_recs," + "tot_loads," + "tot_stores," + "stores_l1hit,stores_l1miss,stores_na," + "ld_fbhit,ld_l1hit,ld_l2hit," + "ld_lclhit,lcl_hitm," + "ld_rmthit,rmt_hitm," + "dram_lcl,dram_rmt"; + else + output_str = "cl_idx," + "dcacheline," + "dcacheline_node," + "dcacheline_count," + "percent_costly_snoop," + "tot_peer,lcl_peer,rmt_peer," + "tot_recs," + "tot_loads," + "tot_stores," + "stores_l1hit,stores_l1miss,stores_na," + "ld_fbhit,ld_l1hit,ld_l2hit," + "ld_lclhit,lcl_hitm," + "ld_rmthit,rmt_hitm," + "dram_lcl,dram_rmt"; + + if (c2c.display == DISPLAY_TOT_HITM) + sort_str = "tot_hitm"; + else if (c2c.display == DISPLAY_RMT_HITM) + sort_str = "rmt_hitm"; + else if (c2c.display == DISPLAY_LCL_HITM) + sort_str = "lcl_hitm"; + else if (c2c.display == DISPLAY_SNP_PEER) + sort_str = "tot_peer"; + + c2c_hists__reinit(&c2c.hists, output_str, sort_str); ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting..."); hists__collapse_resort(&c2c.hists.hists, NULL); - hists__output_resort_cb(&c2c.hists.hists, &prog, resort_hitm_cb); + hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb); hists__iterate_cb(&c2c.hists.hists, resort_cl_cb); ui_progress__finish(); @@ -2886,12 +3215,19 @@ static int parse_record_events(const struct option *opt, const char *str, int unset __maybe_unused) { bool *event_set = (bool *) opt->value; + struct perf_pmu *pmu; + + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: there is no PMU that supports perf c2c\n"); + exit(-1); + } if (!strcmp(str, "list")) { - perf_mem_events__list(); + perf_pmu__mem_events_list(pmu); exit(0); } - if (perf_mem_events__parse(str)) + if (perf_pmu__mem_events_parse(pmu, str)) exit(-1); *event_set = true; @@ -2914,9 +3250,11 @@ static int perf_c2c__record(int argc, const char **argv) int ret; bool all_user = false, all_kernel = false; bool event_set = false; + struct perf_mem_event *e; + struct perf_pmu *pmu; struct option options[] = { OPT_CALLBACK('e', "event", &event_set, "event", - "event selector. Use 'perf mem record -e list' to list available events", + "event selector. Use 'perf c2c record -e list' to list available events", parse_record_events), OPT_BOOLEAN('u', "all-user", &all_user, "collect only user level data"), OPT_BOOLEAN('k', "all-kernel", &all_kernel, "collect only kernel level data"), @@ -2925,7 +3263,13 @@ static int perf_c2c__record(int argc, const char **argv) OPT_END() }; - if (perf_mem_events__init()) { + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: no PMU supports the memory events\n"); + return -1; + } + + if (perf_pmu__mem_events_init(pmu)) { pr_err("failed: memory events not supported\n"); return -1; } @@ -2933,7 +3277,9 @@ static int perf_c2c__record(int argc, const char **argv) argc = parse_options(argc, argv, options, record_mem_usage, PARSE_OPT_KEEP_UNKNOWN); - rec_argc = argc + 11; /* max number of arguments */ + /* Max number of arguments multiplied by number of PMUs that can support them. */ + rec_argc = argc + 11 * (perf_pmu__mem_events_num_mem_pmus(pmu) + 1); + rec_argv = calloc(rec_argc + 1, sizeof(char *)); if (!rec_argv) return -1; @@ -2941,31 +3287,34 @@ static int perf_c2c__record(int argc, const char **argv) rec_argv[i++] = "record"; if (!event_set) { - perf_mem_events[PERF_MEM_EVENTS__LOAD].record = true; - perf_mem_events[PERF_MEM_EVENTS__STORE].record = true; + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE); + /* + * The load and store operations are required, use the event + * PERF_MEM_EVENTS__LOAD_STORE if it is supported. + */ + if (e->tag) { + e->record = true; + rec_argv[i++] = "-W"; + } else { + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); + e->record = true; + + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__STORE); + e->record = true; + } } - if (perf_mem_events[PERF_MEM_EVENTS__LOAD].record) + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); + if (e->record) rec_argv[i++] = "-W"; rec_argv[i++] = "-d"; rec_argv[i++] = "--phys-data"; rec_argv[i++] = "--sample-cpu"; - for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - if (!perf_mem_events[j].record) - continue; - - if (!perf_mem_events[j].supported) { - pr_err("failed: event '%s' not supported\n", - perf_mem_events[j].name); - free(rec_argv); - return -1; - } - - rec_argv[i++] = "-e"; - rec_argv[i++] = perf_mem_events__name(j); - } + ret = perf_mem_events__record_args(rec_argv, &i); + if (ret) + goto out; if (all_user) rec_argv[i++] = "--all-user"; @@ -2989,6 +3338,7 @@ static int perf_c2c__record(int argc, const char **argv) } ret = cmd_record(i, rec_argv); +out: free(rec_argv); return ret; } @@ -3001,9 +3351,9 @@ int cmd_c2c(int argc, const char **argv) if (!argc) usage_with_options(c2c_usage, c2c_options); - if (!strncmp(argv[0], "rec", 3)) { + if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) { return perf_c2c__record(argc, argv); - } else if (!strncmp(argv[0], "rep", 3)) { + } else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) { return perf_c2c__report(argc, argv); } else { usage_with_options(c2c_usage, c2c_options); |