diff options
Diffstat (limited to 'tools/lib')
-rw-r--r-- | tools/lib/bpf/Build | 2 | ||||
-rw-r--r-- | tools/lib/bpf/bpf.c | 93 | ||||
-rw-r--r-- | tools/lib/bpf/bpf.h | 3 | ||||
-rw-r--r-- | tools/lib/bpf/bpf_prog_linfo.c | 253 | ||||
-rw-r--r-- | tools/lib/bpf/btf.c | 342 | ||||
-rw-r--r-- | tools/lib/bpf/btf.h | 25 | ||||
-rw-r--r-- | tools/lib/bpf/libbpf.c | 159 | ||||
-rw-r--r-- | tools/lib/bpf/libbpf.h | 13 | ||||
-rw-r--r-- | tools/lib/bpf/libbpf.map | 4 |
9 files changed, 657 insertions, 237 deletions
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index 7bc31c905018..197b40f5b5c6 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1 +1 @@ -libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o +libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o bpf_prog_linfo.o diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 5c3be06bf0dd..3caaa3428774 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name, -1); } +static void * +alloc_zero_tailing_info(const void *orecord, __u32 cnt, + __u32 actual_rec_size, __u32 expected_rec_size) +{ + __u64 info_len = actual_rec_size * cnt; + void *info, *nrecord; + int i; + + info = malloc(info_len); + if (!info) + return NULL; + + /* zero out bytes kernel does not understand */ + nrecord = info; + for (i = 0; i < cnt; i++) { + memcpy(nrecord, orecord, expected_rec_size); + memset(nrecord + expected_rec_size, 0, + actual_rec_size - expected_rec_size); + orecord += actual_rec_size; + nrecord += actual_rec_size; + } + + return info; +} + int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, char *log_buf, size_t log_buf_sz) { + void *finfo = NULL, *linfo = NULL; union bpf_attr attr; - void *finfo = NULL; __u32 name_len; int fd; @@ -201,53 +226,58 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, attr.func_info_rec_size = load_attr->func_info_rec_size; attr.func_info_cnt = load_attr->func_info_cnt; attr.func_info = ptr_to_u64(load_attr->func_info); + attr.line_info_rec_size = load_attr->line_info_rec_size; + attr.line_info_cnt = load_attr->line_info_cnt; + attr.line_info = ptr_to_u64(load_attr->line_info); memcpy(attr.prog_name, load_attr->name, min(name_len, BPF_OBJ_NAME_LEN - 1)); fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); - if (fd >= 0 || !log_buf || !log_buf_sz) + if (fd >= 0) return fd; /* After bpf_prog_load, the kernel may modify certain attributes * to give user space a hint how to deal with loading failure. * Check to see whether we can make some changes and load again. */ - if (errno == E2BIG && attr.func_info_cnt && - attr.func_info_rec_size < load_attr->func_info_rec_size) { - __u32 actual_rec_size = load_attr->func_info_rec_size; - __u32 expected_rec_size = attr.func_info_rec_size; - __u32 finfo_cnt = load_attr->func_info_cnt; - __u64 finfo_len = actual_rec_size * finfo_cnt; - const void *orecord; - void *nrecord; - int i; - - finfo = malloc(finfo_len); - if (!finfo) - /* further try with log buffer won't help */ - return fd; - - /* zero out bytes kernel does not understand */ - orecord = load_attr->func_info; - nrecord = finfo; - for (i = 0; i < load_attr->func_info_cnt; i++) { - memcpy(nrecord, orecord, expected_rec_size); - memset(nrecord + expected_rec_size, 0, - actual_rec_size - expected_rec_size); - orecord += actual_rec_size; - nrecord += actual_rec_size; + while (errno == E2BIG && (!finfo || !linfo)) { + if (!finfo && attr.func_info_cnt && + attr.func_info_rec_size < load_attr->func_info_rec_size) { + /* try with corrected func info records */ + finfo = alloc_zero_tailing_info(load_attr->func_info, + load_attr->func_info_cnt, + load_attr->func_info_rec_size, + attr.func_info_rec_size); + if (!finfo) + goto done; + + attr.func_info = ptr_to_u64(finfo); + attr.func_info_rec_size = load_attr->func_info_rec_size; + } else if (!linfo && attr.line_info_cnt && + attr.line_info_rec_size < + load_attr->line_info_rec_size) { + linfo = alloc_zero_tailing_info(load_attr->line_info, + load_attr->line_info_cnt, + load_attr->line_info_rec_size, + attr.line_info_rec_size); + if (!linfo) + goto done; + + attr.line_info = ptr_to_u64(linfo); + attr.line_info_rec_size = load_attr->line_info_rec_size; + } else { + break; } - /* try with corrected func info records */ - attr.func_info = ptr_to_u64(finfo); - attr.func_info_rec_size = load_attr->func_info_rec_size; - fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); - if (fd >= 0 || !log_buf || !log_buf_sz) + if (fd >= 0) goto done; } + if (!log_buf || !log_buf_sz) + goto done; + /* Try again with log */ attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_buf_sz; @@ -256,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); done: free(finfo); + free(linfo); return fd; } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 098e6f793b76..8f09de482839 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -82,6 +82,9 @@ struct bpf_load_program_attr { __u32 func_info_rec_size; const void *func_info; __u32 func_info_cnt; + __u32 line_info_rec_size; + const void *line_info; + __u32 line_info_cnt; }; /* Flags to direct loading requirements */ diff --git a/tools/lib/bpf/bpf_prog_linfo.c b/tools/lib/bpf/bpf_prog_linfo.c new file mode 100644 index 000000000000..b8af65145408 --- /dev/null +++ b/tools/lib/bpf/bpf_prog_linfo.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2018 Facebook */ + +#include <string.h> +#include <stdlib.h> +#include <linux/err.h> +#include <linux/bpf.h> +#include "libbpf.h" + +#ifndef min +#define min(x, y) ((x) < (y) ? (x) : (y)) +#endif + +struct bpf_prog_linfo { + void *raw_linfo; + void *raw_jited_linfo; + __u32 *nr_jited_linfo_per_func; + __u32 *jited_linfo_func_idx; + __u32 nr_linfo; + __u32 nr_jited_func; + __u32 rec_size; + __u32 jited_rec_size; +}; + +static int dissect_jited_func(struct bpf_prog_linfo *prog_linfo, + const __u64 *ksym_func, const __u32 *ksym_len) +{ + __u32 nr_jited_func, nr_linfo; + const void *raw_jited_linfo; + const __u64 *jited_linfo; + __u64 last_jited_linfo; + /* + * Index to raw_jited_linfo: + * i: Index for searching the next ksym_func + * prev_i: Index to the last found ksym_func + */ + __u32 i, prev_i; + __u32 f; /* Index to ksym_func */ + + raw_jited_linfo = prog_linfo->raw_jited_linfo; + jited_linfo = raw_jited_linfo; + if (ksym_func[0] != *jited_linfo) + goto errout; + + prog_linfo->jited_linfo_func_idx[0] = 0; + nr_jited_func = prog_linfo->nr_jited_func; + nr_linfo = prog_linfo->nr_linfo; + + for (prev_i = 0, i = 1, f = 1; + i < nr_linfo && f < nr_jited_func; + i++) { + raw_jited_linfo += prog_linfo->jited_rec_size; + last_jited_linfo = *jited_linfo; + jited_linfo = raw_jited_linfo; + + if (ksym_func[f] == *jited_linfo) { + prog_linfo->jited_linfo_func_idx[f] = i; + + /* Sanity check */ + if (last_jited_linfo - ksym_func[f - 1] + 1 > + ksym_len[f - 1]) + goto errout; + + prog_linfo->nr_jited_linfo_per_func[f - 1] = + i - prev_i; + prev_i = i; + + /* + * The ksym_func[f] is found in jited_linfo. + * Look for the next one. + */ + f++; + } else if (*jited_linfo <= last_jited_linfo) { + /* Ensure the addr is increasing _within_ a func */ + goto errout; + } + } + + if (f != nr_jited_func) + goto errout; + + prog_linfo->nr_jited_linfo_per_func[nr_jited_func - 1] = + nr_linfo - prev_i; + + return 0; + +errout: + return -EINVAL; +} + +void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo) +{ + if (!prog_linfo) + return; + + free(prog_linfo->raw_linfo); + free(prog_linfo->raw_jited_linfo); + free(prog_linfo->nr_jited_linfo_per_func); + free(prog_linfo->jited_linfo_func_idx); + free(prog_linfo); +} + +struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info) +{ + struct bpf_prog_linfo *prog_linfo; + __u32 nr_linfo, nr_jited_func; + + nr_linfo = info->line_info_cnt; + + /* + * Test !info->line_info because the kernel may NULL + * the ptr if kernel.kptr_restrict is set. + */ + if (!nr_linfo || !info->line_info) + return NULL; + + /* + * The min size that bpf_prog_linfo has to access for + * searching purpose. + */ + if (info->line_info_rec_size < + offsetof(struct bpf_line_info, file_name_off)) + return NULL; + + prog_linfo = calloc(1, sizeof(*prog_linfo)); + if (!prog_linfo) + return NULL; + + /* Copy xlated line_info */ + prog_linfo->nr_linfo = nr_linfo; + prog_linfo->rec_size = info->line_info_rec_size; + prog_linfo->raw_linfo = malloc(nr_linfo * prog_linfo->rec_size); + if (!prog_linfo->raw_linfo) + goto err_free; + memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info, + nr_linfo * prog_linfo->rec_size); + + nr_jited_func = info->nr_jited_ksyms; + if (!nr_jited_func || + !info->jited_line_info || + info->jited_line_info_cnt != nr_linfo || + info->jited_line_info_rec_size < sizeof(__u64) || + info->nr_jited_func_lens != nr_jited_func || + !info->jited_ksyms || + !info->jited_func_lens) + /* Not enough info to provide jited_line_info */ + return prog_linfo; + + /* Copy jited_line_info */ + prog_linfo->nr_jited_func = nr_jited_func; + prog_linfo->jited_rec_size = info->jited_line_info_rec_size; + prog_linfo->raw_jited_linfo = malloc(nr_linfo * + prog_linfo->jited_rec_size); + if (!prog_linfo->raw_jited_linfo) + goto err_free; + memcpy(prog_linfo->raw_jited_linfo, + (void *)(long)info->jited_line_info, + nr_linfo * prog_linfo->jited_rec_size); + + /* Number of jited_line_info per jited func */ + prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func * + sizeof(__u32)); + if (!prog_linfo->nr_jited_linfo_per_func) + goto err_free; + + /* + * For each jited func, + * the start idx to the "linfo" and "jited_linfo" array, + */ + prog_linfo->jited_linfo_func_idx = malloc(nr_jited_func * + sizeof(__u32)); + if (!prog_linfo->jited_linfo_func_idx) + goto err_free; + + if (dissect_jited_func(prog_linfo, + (__u64 *)(long)info->jited_ksyms, + (__u32 *)(long)info->jited_func_lens)) + goto err_free; + + return prog_linfo; + +err_free: + bpf_prog_linfo__free(prog_linfo); + return NULL; +} + +const struct bpf_line_info * +bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, + __u64 addr, __u32 func_idx, __u32 nr_skip) +{ + __u32 jited_rec_size, rec_size, nr_linfo, start, i; + const void *raw_jited_linfo, *raw_linfo; + const __u64 *jited_linfo; + + if (func_idx >= prog_linfo->nr_jited_func) + return NULL; + + nr_linfo = prog_linfo->nr_jited_linfo_per_func[func_idx]; + if (nr_skip >= nr_linfo) + return NULL; + + start = prog_linfo->jited_linfo_func_idx[func_idx] + nr_skip; + jited_rec_size = prog_linfo->jited_rec_size; + raw_jited_linfo = prog_linfo->raw_jited_linfo + + (start * jited_rec_size); + jited_linfo = raw_jited_linfo; + if (addr < *jited_linfo) + return NULL; + + nr_linfo -= nr_skip; + rec_size = prog_linfo->rec_size; + raw_linfo = prog_linfo->raw_linfo + (start * rec_size); + for (i = 0; i < nr_linfo; i++) { + if (addr < *jited_linfo) + break; + + raw_linfo += rec_size; + raw_jited_linfo += jited_rec_size; + jited_linfo = raw_jited_linfo; + } + + return raw_linfo - rec_size; +} + +const struct bpf_line_info * +bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, + __u32 insn_off, __u32 nr_skip) +{ + const struct bpf_line_info *linfo; + __u32 rec_size, nr_linfo, i; + const void *raw_linfo; + + nr_linfo = prog_linfo->nr_linfo; + if (nr_skip >= nr_linfo) + return NULL; + + rec_size = prog_linfo->rec_size; + raw_linfo = prog_linfo->raw_linfo + (nr_skip * rec_size); + linfo = raw_linfo; + if (insn_off < linfo->insn_off) + return NULL; + + nr_linfo -= nr_skip; + for (i = 0; i < nr_linfo; i++) { + if (insn_off < linfo->insn_off) + break; + + raw_linfo += rec_size; + linfo = raw_linfo; + } + + return raw_linfo - rec_size; +} diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 85d6446cf832..d682d3b8f7b9 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -37,10 +37,27 @@ struct btf { int fd; }; +struct btf_ext_info { + /* + * info points to a deep copy of the individual info section + * (e.g. func_info and line_info) from the .BTF.ext. + * It does not include the __u32 rec_size. + */ + void *info; + __u32 rec_size; + __u32 len; +}; + struct btf_ext { - void *func_info; - __u32 func_info_rec_size; - __u32 func_info_len; + struct btf_ext_info func_info; + struct btf_ext_info line_info; +}; + +struct btf_ext_info_sec { + __u32 sec_name_off; + __u32 num_info; + /* Followed by num_info * record_size number of bytes */ + __u8 data[0]; }; /* The minimum bpf_func_info checked by the loader */ @@ -49,6 +66,14 @@ struct bpf_func_info_min { __u32 type_id; }; +/* The minimum bpf_line_info checked by the loader */ +struct bpf_line_info_min { + __u32 insn_off; + __u32 file_name_off; + __u32 line_off; + __u32 line_col; +}; + static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; @@ -479,71 +504,147 @@ exit_free: return err; } -static int btf_ext_validate_func_info(const void *finfo, __u32 size, - btf_print_fn_t err_log) +struct btf_ext_sec_copy_param { + __u32 off; + __u32 len; + __u32 min_rec_size; + struct btf_ext_info *ext_info; + const char *desc; +}; + +static int btf_ext_copy_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + struct btf_ext_sec_copy_param *ext_sec, + btf_print_fn_t err_log) { - int sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 size_left, num_records, record_size; - const struct btf_sec_func_info *sinfo; - __u64 total_record_size; - - /* At least a func_info record size */ - if (size < sizeof(__u32)) { - elog("BTF.ext func_info record size not found"); + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + const struct btf_ext_info_sec *sinfo; + struct btf_ext_info *ext_info; + __u32 info_left, record_size; + /* The start of the info sec (including the __u32 record_size). */ + const void *info; + + /* data and data_size do not include btf_ext_header from now on */ + data = data + hdr->hdr_len; + data_size -= hdr->hdr_len; + + if (ext_sec->off & 0x03) { + elog(".BTF.ext %s section is not aligned to 4 bytes\n", + ext_sec->desc); return -EINVAL; } - /* The record size needs to meet below minimum standard */ - record_size = *(__u32 *)finfo; - if (record_size < sizeof(struct bpf_func_info_min) || - record_size % sizeof(__u32)) { - elog("BTF.ext func_info invalid record size"); + if (data_size < ext_sec->off || + ext_sec->len > data_size - ext_sec->off) { + elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", + ext_sec->desc, ext_sec->off, ext_sec->len); return -EINVAL; } - sinfo = finfo + sizeof(__u32); - size_left = size - sizeof(__u32); + info = data + ext_sec->off; + info_left = ext_sec->len; - /* If no func_info records, return failure now so .BTF.ext - * won't be used. - */ - if (!size_left) { - elog("BTF.ext no func info records"); + /* At least a record size */ + if (info_left < sizeof(__u32)) { + elog(".BTF.ext %s record size not found\n", ext_sec->desc); + return -EINVAL; + } + + /* The record size needs to meet the minimum standard */ + record_size = *(__u32 *)info; + if (record_size < ext_sec->min_rec_size || + record_size & 0x03) { + elog("%s section in .BTF.ext has invalid record size %u\n", + ext_sec->desc, record_size); return -EINVAL; } - while (size_left) { - if (size_left < sec_hdrlen) { - elog("BTF.ext func_info header not found"); + sinfo = info + sizeof(__u32); + info_left -= sizeof(__u32); + + /* If no records, return failure now so .BTF.ext won't be used. */ + if (!info_left) { + elog("%s section in .BTF.ext has no records", ext_sec->desc); + return -EINVAL; + } + + while (info_left) { + unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); + __u64 total_record_size; + __u32 num_records; + + if (info_left < sec_hdrlen) { + elog("%s section header is not found in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } - num_records = sinfo->num_func_info; + num_records = sinfo->num_info; if (num_records == 0) { - elog("incorrect BTF.ext num_func_info"); + elog("%s section has incorrect num_records in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } total_record_size = sec_hdrlen + (__u64)num_records * record_size; - if (size_left < total_record_size) { - elog("incorrect BTF.ext num_func_info"); + if (info_left < total_record_size) { + elog("%s section has incorrect num_records in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } - size_left -= total_record_size; + info_left -= total_record_size; sinfo = (void *)sinfo + total_record_size; } + ext_info = ext_sec->ext_info; + ext_info->len = ext_sec->len - sizeof(__u32); + ext_info->rec_size = record_size; + ext_info->info = malloc(ext_info->len); + if (!ext_info->info) + return -ENOMEM; + memcpy(ext_info->info, info + sizeof(__u32), ext_info->len); + return 0; } +static int btf_ext_copy_func_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + struct btf_ext_sec_copy_param param = { + .off = hdr->func_info_off, + .len = hdr->func_info_len, + .min_rec_size = sizeof(struct bpf_func_info_min), + .ext_info = &btf_ext->func_info, + .desc = "func_info" + }; + + return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); +} + +static int btf_ext_copy_line_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + struct btf_ext_sec_copy_param param = { + .off = hdr->line_info_off, + .len = hdr->line_info_len, + .min_rec_size = sizeof(struct bpf_line_info_min), + .ext_info = &btf_ext->line_info, + .desc = "line_info", + }; + + return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); +} + static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, btf_print_fn_t err_log) { const struct btf_ext_header *hdr = (struct btf_ext_header *)data; - __u32 meta_left, last_func_info_pos; - void *finfo; if (data_size < offsetof(struct btf_ext_header, func_info_off) || data_size < hdr->hdr_len) { @@ -566,34 +667,12 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, return -ENOTSUP; } - meta_left = data_size - hdr->hdr_len; - if (!meta_left) { + if (data_size == hdr->hdr_len) { elog("BTF.ext has no data\n"); return -EINVAL; } - if (meta_left < hdr->func_info_off) { - elog("Invalid BTF.ext func_info section offset:%u\n", - hdr->func_info_off); - return -EINVAL; - } - - if (hdr->func_info_off & 0x03) { - elog("BTF.ext func_info section is not aligned to 4 bytes\n"); - return -EINVAL; - } - - last_func_info_pos = hdr->hdr_len + hdr->func_info_off + - hdr->func_info_len; - if (last_func_info_pos > data_size) { - elog("Invalid BTF.ext func_info section size:%u\n", - hdr->func_info_len); - return -EINVAL; - } - - finfo = data + hdr->hdr_len + hdr->func_info_off; - return btf_ext_validate_func_info(finfo, hdr->func_info_len, - err_log); + return 0; } void btf_ext__free(struct btf_ext *btf_ext) @@ -601,16 +680,14 @@ void btf_ext__free(struct btf_ext *btf_ext) if (!btf_ext) return; - free(btf_ext->func_info); + free(btf_ext->func_info.info); + free(btf_ext->line_info.info); free(btf_ext); } struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) { - const struct btf_ext_header *hdr; struct btf_ext *btf_ext; - void *org_fdata, *fdata; - __u32 hdrlen, size_u32; int err; err = btf_ext_parse_hdr(data, size, err_log); @@ -621,42 +698,38 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) if (!btf_ext) return ERR_PTR(-ENOMEM); - hdr = (const struct btf_ext_header *)data; - hdrlen = hdr->hdr_len; - size_u32 = sizeof(__u32); - fdata = malloc(hdr->func_info_len - size_u32); - if (!fdata) { - free(btf_ext); - return ERR_PTR(-ENOMEM); + err = btf_ext_copy_func_info(btf_ext, data, size, err_log); + if (err) { + btf_ext__free(btf_ext); + return ERR_PTR(err); } - /* remember record size and copy rest of func_info data */ - org_fdata = data + hdrlen + hdr->func_info_off; - btf_ext->func_info_rec_size = *(__u32 *)org_fdata; - memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32); - btf_ext->func_info = fdata; - btf_ext->func_info_len = hdr->func_info_len - size_u32; + err = btf_ext_copy_line_info(btf_ext, data, size, err_log); + if (err) { + btf_ext__free(btf_ext); + return ERR_PTR(err); + } return btf_ext; } -int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, void **func_info, - __u32 *func_info_rec_size, __u32 *func_info_len) +static int btf_ext_reloc_info(const struct btf *btf, + const struct btf_ext_info *ext_info, + const char *sec_name, __u32 insns_cnt, + void **info, __u32 *cnt) { - __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 i, record_size, records_len; - struct btf_sec_func_info *sinfo; + __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec); + __u32 i, record_size, existing_len, records_len; + struct btf_ext_info_sec *sinfo; const char *info_sec_name; - __s64 remain_len; + __u64 remain_len; void *data; - record_size = btf_ext->func_info_rec_size; - sinfo = btf_ext->func_info; - remain_len = btf_ext->func_info_len; - + record_size = ext_info->rec_size; + sinfo = ext_info->info; + remain_len = ext_info->len; while (remain_len > 0) { - records_len = sinfo->num_func_info * record_size; + records_len = sinfo->num_info * record_size; info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); if (strcmp(info_sec_name, sec_name)) { remain_len -= sec_hdrlen + records_len; @@ -664,79 +737,52 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, continue; } - data = malloc(records_len); + existing_len = (*cnt) * record_size; + data = realloc(*info, existing_len + records_len); if (!data) return -ENOMEM; - memcpy(data, sinfo->data, records_len); - - /* adjust the insn_off, the data in .BTF.ext is - * the actual byte offset, and the kernel expects - * the offset in term of bpf_insn. - * - * adjust the insn offset only, the rest data will - * be passed to kernel. + memcpy(data + existing_len, sinfo->data, records_len); + /* adjust insn_off only, the rest data will be passed + * to the kernel. */ - for (i = 0; i < sinfo->num_func_info; i++) { - struct bpf_func_info_min *record; + for (i = 0; i < sinfo->num_info; i++) { + __u32 *insn_off; - record = data + i * record_size; - record->insn_off /= sizeof(struct bpf_insn); + insn_off = data + existing_len + (i * record_size); + *insn_off = *insn_off / sizeof(struct bpf_insn) + + insns_cnt; } - - *func_info = data; - *func_info_len = records_len; - *func_info_rec_size = record_size; + *info = data; + *cnt += sinfo->num_info; return 0; } - return -EINVAL; + return -ENOENT; } -int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *func_info_len) +int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *cnt) { - __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 i, record_size, existing_flen, records_len; - struct btf_sec_func_info *sinfo; - const char *info_sec_name; - __u64 remain_len; - void *data; - - record_size = btf_ext->func_info_rec_size; - sinfo = btf_ext->func_info; - remain_len = btf_ext->func_info_len; - while (remain_len > 0) { - records_len = sinfo->num_func_info * record_size; - info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); - if (strcmp(info_sec_name, sec_name)) { - remain_len -= sec_hdrlen + records_len; - sinfo = (void *)sinfo + sec_hdrlen + records_len; - continue; - } - - existing_flen = *func_info_len; - data = realloc(*func_info, existing_flen + records_len); - if (!data) - return -ENOMEM; + return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, + insns_cnt, func_info, cnt); +} - memcpy(data + existing_flen, sinfo->data, records_len); - /* adjust insn_off only, the rest data will be passed - * to the kernel. - */ - for (i = 0; i < sinfo->num_func_info; i++) { - struct bpf_func_info_min *record; +int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **line_info, __u32 *cnt) +{ + return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, + insns_cnt, line_info, cnt); +} - record = data + existing_flen + i * record_size; - record->insn_off = - record->insn_off / sizeof(struct bpf_insn) + - insns_cnt; - } - *func_info = data; - *func_info_len = existing_flen + records_len; - return 0; - } +__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext) +{ + return btf_ext->func_info.rec_size; +} - return -EINVAL; +__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext) +{ + return btf_ext->line_info.rec_size; } diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 5336b2f37293..b0610dcdae6b 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -51,13 +51,8 @@ struct btf_ext_header { /* All offsets are in bytes relative to the end of this header */ __u32 func_info_off; __u32 func_info_len; -}; - -struct btf_sec_func_info { - __u32 sec_name_off; - __u32 num_func_info; - /* Followed by num_func_info number of bpf func_info records */ - __u8 data[0]; + __u32 line_info_off; + __u32 line_info_len; }; typedef int (*btf_print_fn_t)(const char *, ...) @@ -77,12 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log); void btf_ext__free(struct btf_ext *btf_ext); -int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, void **func_info, - __u32 *func_info_rec_size, __u32 *func_info_len); -int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, void **func_info, - __u32 *func_info_len); +int btf_ext__reloc_func_info(const struct btf *btf, + const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *func_info_len); +int btf_ext__reloc_line_info(const struct btf *btf, + const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **line_info, __u32 *cnt); +__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); +__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); #ifdef __cplusplus } /* extern "C" */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 59b748ebd15f..e2bc75ee1614 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -167,9 +167,13 @@ struct bpf_program { int btf_fd; void *func_info; __u32 func_info_rec_size; - __u32 func_info_len; + __u32 func_info_cnt; struct bpf_capabilities *caps; + + void *line_info; + __u32 line_info_rec_size; + __u32 line_info_cnt; }; struct bpf_map { @@ -779,6 +783,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) { Elf *elf = obj->efile.elf; GElf_Ehdr *ep = &obj->efile.ehdr; + Elf_Data *btf_ext_data = NULL; Elf_Scn *scn = NULL; int idx = 0, err = 0; @@ -841,14 +846,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) obj->btf = NULL; } } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { - obj->btf_ext = btf_ext__new(data->d_buf, data->d_size, - __pr_debug); - if (IS_ERR(obj->btf_ext)) { - pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", - BTF_EXT_ELF_SEC, - PTR_ERR(obj->btf_ext)); - obj->btf_ext = NULL; - } + btf_ext_data = data; } else if (sh.sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warning("bpf: multiple SYMTAB in %s\n", @@ -910,6 +908,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) pr_warning("Corrupted ELF file: index of strtab invalid\n"); return LIBBPF_ERRNO__FORMAT; } + if (btf_ext_data) { + if (!obj->btf) { + pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", + BTF_EXT_ELF_SEC, BTF_ELF_SEC); + } else { + obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, + btf_ext_data->d_size, + __pr_debug); + if (IS_ERR(obj->btf_ext)) { + pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", + BTF_EXT_ELF_SEC, + PTR_ERR(obj->btf_ext)); + obj->btf_ext = NULL; + } + } + } if (obj->efile.maps_shndx >= 0) { err = bpf_object__init_maps(obj, flags); if (err) @@ -1276,6 +1290,82 @@ bpf_object__create_maps(struct bpf_object *obj) } static int +check_btf_ext_reloc_err(struct bpf_program *prog, int err, + void *btf_prog_info, const char *info_name) +{ + if (err != -ENOENT) { + pr_warning("Error in loading %s for sec %s.\n", + info_name, prog->section_name); + return err; + } + + /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ + + if (btf_prog_info) { + /* + * Some info has already been found but has problem + * in the last btf_ext reloc. Must have to error + * out. + */ + pr_warning("Error in relocating %s for sec %s.\n", + info_name, prog->section_name); + return err; + } + + /* + * Have problem loading the very first info. Ignore + * the rest. + */ + pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", + info_name, prog->section_name, info_name); + return 0; +} + +static int +bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, + const char *section_name, __u32 insn_offset) +{ + int err; + + if (!insn_offset || prog->func_info) { + /* + * !insn_offset => main program + * + * For sub prog, the main program's func_info has to + * be loaded first (i.e. prog->func_info != NULL) + */ + err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, + section_name, insn_offset, + &prog->func_info, + &prog->func_info_cnt); + if (err) + return check_btf_ext_reloc_err(prog, err, + prog->func_info, + "bpf_func_info"); + + prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); + } + + if (!insn_offset || prog->line_info) { + err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, + section_name, insn_offset, + &prog->line_info, + &prog->line_info_cnt); + if (err) + return check_btf_ext_reloc_err(prog, err, + prog->line_info, + "bpf_line_info"); + + prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); + } + + if (!insn_offset) + prog->btf_fd = btf__fd(obj->btf); + + return 0; +} + +static int bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, struct reloc_desc *relo) { @@ -1306,17 +1396,12 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, return -ENOMEM; } - if (obj->btf && obj->btf_ext) { - err = btf_ext__reloc(obj->btf, obj->btf_ext, - text->section_name, - prog->insns_cnt, - &prog->func_info, - &prog->func_info_len); - if (err) { - pr_warning("error in btf_ext__reloc for sec %s\n", - text->section_name); + if (obj->btf_ext) { + err = bpf_program_reloc_btf_ext(prog, obj, + text->section_name, + prog->insns_cnt); + if (err) return err; - } } memcpy(new_insn + prog->insns_cnt, text->insns, @@ -1341,18 +1426,11 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) if (!prog) return 0; - if (obj->btf && obj->btf_ext) { - err = btf_ext__reloc_init(obj->btf, obj->btf_ext, - prog->section_name, - &prog->func_info, - &prog->func_info_rec_size, - &prog->func_info_len); - if (err) { - pr_warning("err in btf_ext__reloc_init for sec %s\n", - prog->section_name); + if (obj->btf_ext) { + err = bpf_program_reloc_btf_ext(prog, obj, + prog->section_name, 0); + if (err) return err; - } - prog->btf_fd = btf__fd(obj->btf); } if (!prog->reloc_desc) @@ -1444,8 +1522,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) static int load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, - char *license, __u32 kern_version, int *pfd, - __u32 func_info_cnt) + char *license, __u32 kern_version, int *pfd) { struct bpf_load_program_attr load_attr; char *cp, errmsg[STRERR_BUFSIZE]; @@ -1465,8 +1542,10 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; load_attr.func_info = prog->func_info; load_attr.func_info_rec_size = prog->func_info_rec_size; - load_attr.func_info_cnt = func_info_cnt; - + load_attr.func_info_cnt = prog->func_info_cnt; + load_attr.line_info = prog->line_info; + load_attr.line_info_rec_size = prog->line_info_rec_size; + load_attr.line_info_cnt = prog->line_info_cnt; if (!load_attr.insns || !load_attr.insns_cnt) return -EINVAL; @@ -1523,14 +1602,8 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_version) { - __u32 func_info_cnt; int err = 0, fd, i; - if (prog->func_info_len == 0) - func_info_cnt = 0; - else - func_info_cnt = prog->func_info_len / prog->func_info_rec_size; - if (prog->instances.nr < 0 || !prog->instances.fds) { if (prog->preprocessor) { pr_warning("Internal error: can't load program '%s'\n", @@ -1553,8 +1626,7 @@ bpf_program__load(struct bpf_program *prog, prog->section_name, prog->instances.nr); } err = load_program(prog, prog->insns, prog->insns_cnt, - license, kern_version, &fd, - func_info_cnt); + license, kern_version, &fd); if (!err) prog->instances.fds[0] = fd; goto out; @@ -1584,8 +1656,7 @@ bpf_program__load(struct bpf_program *prog, err = load_program(prog, result.new_insn_ptr, result.new_insn_cnt, - license, kern_version, &fd, - func_info_cnt); + license, kern_version, &fd); if (err) { pr_warning("Loading the %dth instance of program '%s' failed\n", diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index f30c3d07bb7d..5f68d7b75215 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -342,6 +342,19 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex, int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle, libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie); +struct bpf_prog_linfo; +struct bpf_prog_info; + +LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo); +LIBBPF_API struct bpf_prog_linfo * +bpf_prog_linfo__new(const struct bpf_prog_info *info); +LIBBPF_API const struct bpf_line_info * +bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, + __u64 addr, __u32 func_idx, __u32 nr_skip); +LIBBPF_API const struct bpf_line_info * +bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, + __u32 insn_off, __u32 nr_skip); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 8deff22d61bb..cd02cd4e2cc3 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -99,6 +99,10 @@ LIBBPF_0.0.1 { bpf_program__unload; bpf_program__unpin; bpf_program__unpin_instance; + bpf_prog_linfo__free; + bpf_prog_linfo__new; + bpf_prog_linfo__lfind_addr_func; + bpf_prog_linfo__lfind; bpf_raw_tracepoint_open; bpf_set_link_xdp_fd; bpf_task_fd_query; |